Protei/PromucFlow_constructor
2
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

chore: Move all remaining actions into links (#37829)

Browse Source 
The chain+link framework is now used for all the steps in the backup
command.


## Automation

/test sanity

### 🔍 Cypress test results
<!-- This is an auto-generated comment: Cypress test results  -->
> [!TIP]
> 🟢 🟢 🟢 All cypress tests have passed! 🎉 🎉 🎉
> Workflow run:
<https://github.com/appsmithorg/appsmith/actions/runs/12080116829>
> Commit: fcc016fd1fa0bbe9b4cae2478bf4f7750ee5459d
> <a
href="https://internal.appsmith.com/app/cypress-dashboard/rundetails-65890b3c81d7400d08fa9ee5?branch=master&workflowId=12080116829&attempt=1"
target="_blank">Cypress dashboard</a>.
> Tags: `@tag.Sanity`
> Spec:
> <hr>Fri, 29 Nov 2024 07:12:22 UTC
<!-- end of auto-generated comment: Cypress test results  -->


## Communication
Should the DevRel and Marketing teams inform users about this change?
- [ ] Yes
- [x] No


<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->

## Summary by CodeRabbit

## Release Notes

- **New Features**
- Introduced new classes for managing backup processes, including
`BackupFolderLink`, `DiskSpaceLink`, `EnvFileLink`, `GitStorageLink`,
`MongoDumpLink`, and `ManifestLink`.
- Added functionality for checking available disk space and managing
temporary backup folders.
	- Enhanced encryption handling with a new password prompt mechanism.

- **Bug Fixes**
- Streamlined backup process by removing redundant functions and
ensuring proper error handling.

- **Documentation**
- Updated comments and documentation for new classes and methods to
improve clarity.

- **Refactor**
- Restructured backup functionality for improved modularity and
maintainability.

<!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
Shrikant Sharat Kandula 2024-11-29 18:46:11 +05:30 committed by GitHub
parent e8cb73dc68
commit 6773f51d6a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
11 changed files with 299 additions and 249 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
app/client/packages/rts/src/ctl/backup/BackupState.ts
View File

@ -1,25 +1,19 @@
import { getTimeStampInISO } from "./index";
export class BackupState { export class BackupState {
readonly args: string[]; readonly args: readonly string[];
readonly initAt: string = getTimeStampInISO(); readonly initAt: string = new Date().toISOString().replace(/:/g, "-");
readonly errors: string[] = []; readonly errors: string[] = [];
backupRootPath: string = ""; backupRootPath: string = "";
archivePath: string = ""; archivePath: string = "";
encryptionPassword: string = ""; isEncryptionEnabled: boolean = false;
constructor(args: string[]) { constructor(args: string[]) {
this.args = args; this.args = Object.freeze([...args]);
// We seal `this` so that no link in the chain can "add" new properties to the state. This is intentional. If any // We seal `this` so that no link in the chain can "add" new properties to the state. This is intentional. If any
// link wants to save data in the `BackupState`, which shouldn't even be needed in most cases, it should do so by // link wants to save data in the `BackupState`, which shouldn't even be needed in most cases, it should do so by
// explicitly declaring a property in this class. No surprises. // explicitly declaring a property in this class. No surprises.
Object.seal(this); Object.seal(this);
} }
isEncryptionEnabled() {
return !!this.encryptionPassword;
}
} }

50
app/client/packages/rts/src/ctl/backup/backup.test.ts
View File

@ -3,6 +3,16 @@ import * as backup from ".";
import * as Constants from "../constants"; import * as Constants from "../constants";
import * as utils from "../utils"; import * as utils from "../utils";
import readlineSync from "readline-sync"; import readlineSync from "readline-sync";
import {
checkAvailableBackupSpace,
encryptBackupArchive,
executeCopyCMD,
executeMongoDumpCMD,
getAvailableBackupSpaceInBytes,
getEncryptionPasswordFromUser,
getGitRoot,
removeSensitiveEnvData,
} from "./links";
jest.mock("../utils", () => ({ jest.mock("../utils", () => ({
...jest.requireActual("../utils"), ...jest.requireActual("../utils"),
@ -10,15 +20,8 @@ jest.mock("../utils", () => ({
})); }));
describe("Backup Tests", () => { describe("Backup Tests", () => {
test("Timestamp string in ISO format", () => {
console.log(backup.getTimeStampInISO());
expect(backup.getTimeStampInISO()).toMatch(
/(\d{4})-(\d{2})-(\d{2})T(\d{2})-(\d{2})-(\d{2})\.(\d{3})Z/,
);
});
test("Available Space in /appsmith-stacks volume in Bytes", async () => { test("Available Space in /appsmith-stacks volume in Bytes", async () => {
const res = expect(await backup.getAvailableBackupSpaceInBytes("/")); const res = expect(await getAvailableBackupSpaceInBytes("/"));
res.toBeGreaterThan(1024 * 1024); res.toBeGreaterThan(1024 * 1024);
}); });
@ -32,14 +35,12 @@ describe("Backup Tests", () => {
it("Should throw Error when the available size is below MIN_REQUIRED_DISK_SPACE_IN_BYTES", () => { it("Should throw Error when the available size is below MIN_REQUIRED_DISK_SPACE_IN_BYTES", () => {
const size = Constants.MIN_REQUIRED_DISK_SPACE_IN_BYTES - 1; const size = Constants.MIN_REQUIRED_DISK_SPACE_IN_BYTES - 1;
expect(() => backup.checkAvailableBackupSpace(size)).toThrow(); expect(() => checkAvailableBackupSpace(size)).toThrow();
}); });
it("Should not should throw Error when the available size is >= MIN_REQUIRED_DISK_SPACE_IN_BYTES", () => { it("Should not should throw Error when the available size is >= MIN_REQUIRED_DISK_SPACE_IN_BYTES", () => {
expect(() => { expect(() => {
backup.checkAvailableBackupSpace( checkAvailableBackupSpace(Constants.MIN_REQUIRED_DISK_SPACE_IN_BYTES);
Constants.MIN_REQUIRED_DISK_SPACE_IN_BYTES,
);
}).not.toThrow( }).not.toThrow(
"Not enough space available at /appsmith-stacks. Please ensure availability of at least 5GB to backup successfully.", "Not enough space available at /appsmith-stacks. Please ensure availability of at least 5GB to backup successfully.",
); );
@ -59,29 +60,29 @@ describe("Backup Tests", () => {
const appsmithMongoURI = "mongodb://username:password@host/appsmith"; const appsmithMongoURI = "mongodb://username:password@host/appsmith";
const cmd = const cmd =
"mongodump --uri=mongodb://username:password@host/appsmith --archive=/dest/mongodb-data.gz --gzip"; "mongodump --uri=mongodb://username:password@host/appsmith --archive=/dest/mongodb-data.gz --gzip";
const res = await backup.executeMongoDumpCMD(dest, appsmithMongoURI); const res = await executeMongoDumpCMD(dest, appsmithMongoURI);
expect(res).toBe(cmd); expect(res).toBe(cmd);
console.log(res); console.log(res);
}); });
test("Test get gitRoot path when APPSMITH_GIT_ROOT is '' ", () => { test("Test get gitRoot path when APPSMITH_GIT_ROOT is '' ", () => {
expect(backup.getGitRoot("")).toBe("/appsmith-stacks/git-storage"); expect(getGitRoot("")).toBe("/appsmith-stacks/git-storage");
}); });
test("Test get gitRoot path when APPSMITH_GIT_ROOT is null ", () => { test("Test get gitRoot path when APPSMITH_GIT_ROOT is null ", () => {
expect(backup.getGitRoot()).toBe("/appsmith-stacks/git-storage"); expect(getGitRoot()).toBe("/appsmith-stacks/git-storage");
}); });
test("Test get gitRoot path when APPSMITH_GIT_ROOT is defined ", () => { test("Test get gitRoot path when APPSMITH_GIT_ROOT is defined ", () => {
expect(backup.getGitRoot("/my/git/storage")).toBe("/my/git/storage"); expect(getGitRoot("/my/git/storage")).toBe("/my/git/storage");
}); });
test("Test ln command generation", async () => { test("Test ln command generation", async () => {
const gitRoot = "/appsmith-stacks/git-storage"; const gitRoot = "/appsmith-stacks/git-storage";
const dest = "/destdir"; const dest = "/destdir";
const cmd = "ln -s /appsmith-stacks/git-storage /destdir/git-storage"; const cmd = "ln -s /appsmith-stacks/git-storage /destdir/git-storage";
const res = await backup.executeCopyCMD(gitRoot, dest); const res = await executeCopyCMD(gitRoot, dest);
expect(res).toBe(cmd); expect(res).toBe(cmd);
console.log(res); console.log(res);
@ -102,7 +103,7 @@ describe("Backup Tests", () => {
test("If MONGODB and Encryption env values are being removed", () => { test("If MONGODB and Encryption env values are being removed", () => {
expect( expect(
backup.removeSensitiveEnvData(`APPSMITH_REDIS_URL=redis://127.0.0.1:6379\nAPPSMITH_DB_URL=mongodb://appsmith:pass@localhost:27017/appsmith\nAPPSMITH_MONGODB_USER=appsmith\nAPPSMITH_MONGODB_PASSWORD=pass\nAPPSMITH_INSTANCE_NAME=Appsmith\n removeSensitiveEnvData(`APPSMITH_REDIS_URL=redis://127.0.0.1:6379\nAPPSMITH_DB_URL=mongodb://appsmith:pass@localhost:27017/appsmith\nAPPSMITH_MONGODB_USER=appsmith\nAPPSMITH_MONGODB_PASSWORD=pass\nAPPSMITH_INSTANCE_NAME=Appsmith\n
`), `),
).toMatch( ).toMatch(
`APPSMITH_REDIS_URL=redis://127.0.0.1:6379\nAPPSMITH_INSTANCE_NAME=Appsmith\n`, `APPSMITH_REDIS_URL=redis://127.0.0.1:6379\nAPPSMITH_INSTANCE_NAME=Appsmith\n`,
@ -111,7 +112,7 @@ describe("Backup Tests", () => {
test("If MONGODB and Encryption env values are being removed", () => { test("If MONGODB and Encryption env values are being removed", () => {
expect( expect(
backup.removeSensitiveEnvData(`APPSMITH_REDIS_URL=redis://127.0.0.1:6379\nAPPSMITH_ENCRYPTION_PASSWORD=dummy-pass\nAPPSMITH_ENCRYPTION_SALT=dummy-salt\nAPPSMITH_DB_URL=mongodb://appsmith:pass@localhost:27017/appsmith\nAPPSMITH_MONGODB_USER=appsmith\nAPPSMITH_MONGODB_PASSWORD=pass\nAPPSMITH_INSTANCE_NAME=Appsmith\n removeSensitiveEnvData(`APPSMITH_REDIS_URL=redis://127.0.0.1:6379\nAPPSMITH_ENCRYPTION_PASSWORD=dummy-pass\nAPPSMITH_ENCRYPTION_SALT=dummy-salt\nAPPSMITH_DB_URL=mongodb://appsmith:pass@localhost:27017/appsmith\nAPPSMITH_MONGODB_USER=appsmith\nAPPSMITH_MONGODB_PASSWORD=pass\nAPPSMITH_INSTANCE_NAME=Appsmith\n
`), `),
).toMatch( ).toMatch(
`APPSMITH_REDIS_URL=redis://127.0.0.1:6379\nAPPSMITH_INSTANCE_NAME=Appsmith\n`, `APPSMITH_REDIS_URL=redis://127.0.0.1:6379\nAPPSMITH_INSTANCE_NAME=Appsmith\n`,
@ -199,7 +200,7 @@ describe("Backup Tests", () => {
const password = "password#4321"; const password = "password#4321";
readlineSync.question = jest.fn().mockImplementation(() => password); readlineSync.question = jest.fn().mockImplementation(() => password);
const password_res = backup.getEncryptionPasswordFromUser(); const password_res = getEncryptionPasswordFromUser();
expect(password_res).toEqual(password); expect(password_res).toEqual(password);
}); });
@ -215,13 +216,13 @@ describe("Backup Tests", () => {
return password; return password;
}); });
expect(() => backup.getEncryptionPasswordFromUser()).toThrow(); expect(() => getEncryptionPasswordFromUser()).toThrow();
}); });
test("Get encrypted archive path", async () => { test("Get encrypted archive path", async () => {
const archivePath = "/rootDir/appsmith-backup-0000-00-0T00-00-00.00Z"; const archivePath = "/rootDir/appsmith-backup-0000-00-0T00-00-00.00Z";
const encryptionPassword = "password#4321"; const encryptionPassword = "password#4321";
const encArchivePath = await backup.encryptBackupArchive( const encArchivePath = await encryptBackupArchive(
archivePath, archivePath,
encryptionPassword, encryptionPassword,
); );
@ -234,10 +235,7 @@ describe("Backup Tests", () => {
test("Test backup encryption function", async () => { test("Test backup encryption function", async () => {
const archivePath = "/rootDir/appsmith-backup-0000-00-0T00-00-00.00Z"; const archivePath = "/rootDir/appsmith-backup-0000-00-0T00-00-00.00Z";
const encryptionPassword = "password#123"; const encryptionPassword = "password#123";
const res = await backup.encryptBackupArchive( const res = await encryptBackupArchive(archivePath, encryptionPassword);
archivePath,
encryptionPassword,
);
console.log(res); console.log(res);
expect(res).toEqual("/rootDir/appsmith-backup-0000-00-0T00-00-00.00Z.enc"); expect(res).toEqual("/rootDir/appsmith-backup-0000-00-0T00-00-00.00Z.enc");

215
app/client/packages/rts/src/ctl/backup/index.ts
View File

@ -1,14 +1,10 @@
import fsPromises from "fs/promises"; import fsPromises from "fs/promises";
import path from "path";
import os from "os";
import * as utils from "../utils"; import * as utils from "../utils";
import * as Constants from "../constants"; import * as Constants from "../constants";
import * as logger from "../logger"; import * as logger from "../logger";
import * as mailer from "../mailer"; import * as mailer from "../mailer";
import readlineSync from "readline-sync";
import { DiskSpaceLink } from "./links/DiskSpaceLink";
import type { Link } from "./links"; import type { Link } from "./links";
import { EncryptionLink, ManifestLink } from "./links"; import * as linkClasses from "./links";
import { BackupState } from "./BackupState"; import { BackupState } from "./BackupState";
export async function run(args: string[]) { export async function run(args: string[]) {
@ -17,9 +13,16 @@ export async function run(args: string[]) {
const state: BackupState = new BackupState(args); const state: BackupState = new BackupState(args);
const chain: Link[] = [ const chain: Link[] = [
new DiskSpaceLink(), new linkClasses.BackupFolderLink(state),
new ManifestLink(state), new linkClasses.DiskSpaceLink(),
new EncryptionLink(state), new linkClasses.ManifestLink(state),
new linkClasses.MongoDumpLink(state),
new linkClasses.GitStorageLink(state),
new linkClasses.EnvFileLink(state),
// Encryption link is best placed last so if any of the above links fail, we don't ask the user for a password and
// then do nothing with it.
new linkClasses.EncryptionLink(state),
]; ];
try { try {
@ -29,19 +32,6 @@ export async function run(args: string[]) {
} }
// BACKUP // BACKUP
state.backupRootPath = await fsPromises.mkdtemp(
path.join(os.tmpdir(), "appsmithctl-backup-"),
);
await exportDatabase(state.backupRootPath);
await createGitStorageArchive(state.backupRootPath);
await exportDockerEnvFile(
state.backupRootPath,
state.isEncryptionEnabled(),
);
for (const link of chain) { for (const link of chain) {
await link.doBackup?.(); await link.doBackup?.();
} }
@ -58,23 +48,6 @@ export async function run(args: string[]) {
console.log("Post-backup done. Final archive at", state.archivePath); console.log("Post-backup done. Final archive at", state.archivePath);
if (!state.isEncryptionEnabled()) {
console.log(
"********************************************************* IMPORTANT!!! *************************************************************",
);
console.log(
"*** Please ensure you have saved the APPSMITH_ENCRYPTION_SALT and APPSMITH_ENCRYPTION_PASSWORD variables from the docker.env file **",
);
console.log(
"*** These values are not included in the backup export. **",
);
console.log(
"************************************************************************************************************************************",
);
}
await fsPromises.rm(state.backupRootPath, { recursive: true, force: true });
await logger.backup_info( await logger.backup_info(
"Finished taking a backup at " + state.archivePath, "Finished taking a backup at " + state.archivePath,
); );
@ -116,118 +89,6 @@ export async function run(args: string[]) {
} }
} }
export async function encryptBackupArchive(
archivePath: string,
encryptionPassword: string,
) {
const encryptedArchivePath = archivePath + ".enc";
await utils.execCommand([
"openssl",
"enc",
"-aes-256-cbc",
"-pbkdf2",
"-iter",
"100000",
"-in",
archivePath,
"-out",
encryptedArchivePath,
"-k",
encryptionPassword,
]);
return encryptedArchivePath;
}
export function getEncryptionPasswordFromUser(): string {
for (const attempt of [1, 2, 3]) {
if (attempt > 1) {
console.log("Retry attempt", attempt);
}
const encryptionPwd1: string = readlineSync.question(
"Enter a password to encrypt the backup archive: ",
{ hideEchoBack: true },
);
const encryptionPwd2: string = readlineSync.question(
"Enter the above password again: ",
{ hideEchoBack: true },
);
if (encryptionPwd1 === encryptionPwd2) {
if (encryptionPwd1) {
return encryptionPwd1;
}
console.error(
"Invalid input. Empty password is not allowed, please try again.",
);
} else {
console.error("The passwords do not match, please try again.");
}
}
console.error(
"Aborting backup process, failed to obtain valid encryption password.",
);
throw new Error(
"Backup process aborted because a valid encryption password could not be obtained from the user",
);
}
async function exportDatabase(destFolder: string) {
console.log("Exporting database");
await executeMongoDumpCMD(destFolder, utils.getDburl());
console.log("Exporting database done.");
}
async function createGitStorageArchive(destFolder: string) {
console.log("Creating git-storage archive");
const gitRoot = getGitRoot(process.env.APPSMITH_GIT_ROOT);
await executeCopyCMD(gitRoot, destFolder);
console.log("Created git-storage archive");
}
async function exportDockerEnvFile(
destFolder: string,
encryptArchive: boolean,
) {
console.log("Exporting docker environment file");
const content = await fsPromises.readFile(
"/appsmith-stacks/configuration/docker.env",
{ encoding: "utf8" },
);
let cleaned_content = removeSensitiveEnvData(content);
if (encryptArchive) {
cleaned_content +=
"\nAPPSMITH_ENCRYPTION_SALT=" +
process.env.APPSMITH_ENCRYPTION_SALT +
"\nAPPSMITH_ENCRYPTION_PASSWORD=" +
process.env.APPSMITH_ENCRYPTION_PASSWORD;
}
await fsPromises.writeFile(destFolder + "/docker.env", cleaned_content);
console.log("Exporting docker environment file done.");
}
export async function executeMongoDumpCMD(
destFolder: string,
appsmithMongoURI: string,
) {
return await utils.execCommand([
"mongodump",
`--uri=${appsmithMongoURI}`,
`--archive=${destFolder}/mongodb-data.gz`,
"--gzip",
]); // generate cmd
}
async function createFinalArchive(destFolder: string, timestamp: string) { async function createFinalArchive(destFolder: string, timestamp: string) {
console.log("Creating final archive"); console.log("Creating final archive");
@ -260,23 +121,6 @@ async function postBackupCleanup() {
console.log("Cleanup completed."); console.log("Cleanup completed.");
} }
export async function executeCopyCMD(srcFolder: string, destFolder: string) {
return await utils.execCommand([
"ln",
"-s",
srcFolder,
path.join(destFolder, "git-storage"),
]);
}
export function getGitRoot(gitRoot?: string | undefined) {
if (gitRoot == null || gitRoot === "") {
gitRoot = "/appsmith-stacks/git-storage";
}
return gitRoot;
}
export function getBackupContentsPath( export function getBackupContentsPath(
backupRootPath: string, backupRootPath: string,
timestamp: string, timestamp: string,
@ -284,23 +128,6 @@ export function getBackupContentsPath(
return backupRootPath + "/appsmith-backup-" + timestamp; return backupRootPath + "/appsmith-backup-" + timestamp;
} }
export function removeSensitiveEnvData(content: string): string {
// Remove encryption and Mongodb data from docker.env
const output_lines = [];
content.split(/\r?\n/).forEach((line) => {
if (
!line.startsWith("APPSMITH_ENCRYPTION") &&
!line.startsWith("APPSMITH_MONGODB") &&
!line.startsWith("APPSMITH_DB_URL=")
) {
output_lines.push(line);
}
});
return output_lines.join("\n");
}
export function getBackupArchiveLimit(backupArchivesLimit?: number): number { export function getBackupArchiveLimit(backupArchivesLimit?: number): number {
return backupArchivesLimit || Constants.APPSMITH_DEFAULT_BACKUP_ARCHIVE_LIMIT; return backupArchivesLimit || Constants.APPSMITH_DEFAULT_BACKUP_ARCHIVE_LIMIT;
} }
@ -318,23 +145,3 @@ export async function removeOldBackups(
.map(async (file) => fsPromises.rm(file)), .map(async (file) => fsPromises.rm(file)),
); );
} }
export function getTimeStampInISO() {
return new Date().toISOString().replace(/:/g, "-");
}
export async function getAvailableBackupSpaceInBytes(
path: string,
): Promise<number> {
const stat = await fsPromises.statfs(path);
return stat.bsize * stat.bfree;
}
export function checkAvailableBackupSpace(availSpaceInBytes: number) {
if (availSpaceInBytes < Constants.MIN_REQUIRED_DISK_SPACE_IN_BYTES) {
throw new Error(
"Not enough space available at /appsmith-stacks. Please ensure availability of at least 2GB to backup successfully.",
);
}
}

26
app/client/packages/rts/src/ctl/backup/links/BackupFolderLink.ts Normal file
View File

@ -0,0 +1,26 @@
import type { Link } from ".";
import type { BackupState } from "../BackupState";
import fsPromises from "fs/promises";
import path from "path";
import os from "os";
/**
* Creates the backup folder in pre step, and deletes it in post step. The existence of the backup folder should only
* be assumed in the "doBackup" step, and no other.
*/
export class BackupFolderLink implements Link {
constructor(private readonly state: BackupState) {}
async preBackup() {
this.state.backupRootPath = await fsPromises.mkdtemp(
path.join(os.tmpdir(), "appsmithctl-backup-"),
);
}
async postBackup() {
await fsPromises.rm(this.state.backupRootPath, {
recursive: true,
force: true,
});
}
}

22
app/client/packages/rts/src/ctl/backup/links/DiskSpaceLink.ts
View File

@ -1,6 +1,10 @@
import { checkAvailableBackupSpace, getAvailableBackupSpaceInBytes } from "..";
import type { Link } from "."; import type { Link } from ".";
import * as Constants from "../../constants";
import fsPromises from "fs/promises";
/**
* Checks if there is enough space available at the backup location.
*/
export class DiskSpaceLink implements Link { export class DiskSpaceLink implements Link {
async preBackup() { async preBackup() {
const availSpaceInBytes: number = const availSpaceInBytes: number =
@ -9,3 +13,19 @@ export class DiskSpaceLink implements Link {
checkAvailableBackupSpace(availSpaceInBytes); checkAvailableBackupSpace(availSpaceInBytes);
} }
} }
export async function getAvailableBackupSpaceInBytes(
path: string,
): Promise<number> {
const stat = await fsPromises.statfs(path);
return stat.bsize * stat.bfree;
}
export function checkAvailableBackupSpace(availSpaceInBytes: number) {
if (availSpaceInBytes < Constants.MIN_REQUIRED_DISK_SPACE_IN_BYTES) {
throw new Error(
"Not enough space available at /appsmith-stacks. Please ensure availability of at least 2GB to backup successfully.",
);
}
}

80
app/client/packages/rts/src/ctl/backup/links/EncryptionLink.ts
View File

@ -1,10 +1,17 @@
import type { Link } from "./index"; import type { Link } from ".";
import tty from "tty"; import tty from "tty";
import fsPromises from "fs/promises"; import fsPromises from "fs/promises";
import { encryptBackupArchive, getEncryptionPasswordFromUser } from "../index";
import type { BackupState } from "../BackupState"; import type { BackupState } from "../BackupState";
import readlineSync from "readline-sync";
import * as utils from "../../utils";
/**
* Asks the user for a password, and then encrypts the backup archive using openssl, with that password. If a TTY is not
* available to ask for a password, then this feature is gracefully disabled, and encryption is not performed.
*/
export class EncryptionLink implements Link { export class EncryptionLink implements Link {
#password: string = "";
constructor(private readonly state: BackupState) {} constructor(private readonly state: BackupState) {}
async preBackup() { async preBackup() {
@ -12,12 +19,14 @@ export class EncryptionLink implements Link {
!this.state.args.includes("--non-interactive") && !this.state.args.includes("--non-interactive") &&
tty.isatty((process.stdout as any).fd) tty.isatty((process.stdout as any).fd)
) { ) {
this.state.encryptionPassword = getEncryptionPasswordFromUser(); this.#password = getEncryptionPasswordFromUser();
} }
this.state.isEncryptionEnabled = !!this.#password;
} }
async postBackup() { async postBackup() {
if (!this.state.isEncryptionEnabled()) { if (!this.#password) {
return; return;
} }
@ -25,7 +34,7 @@ export class EncryptionLink implements Link {
this.state.archivePath = await encryptBackupArchive( this.state.archivePath = await encryptBackupArchive(
unencryptedArchivePath, unencryptedArchivePath,
this.state.encryptionPassword, this.#password,
); );
await fsPromises.rm(unencryptedArchivePath, { await fsPromises.rm(unencryptedArchivePath, {
@ -34,3 +43,64 @@ export class EncryptionLink implements Link {
}); });
} }
} }
export function getEncryptionPasswordFromUser(): string {
for (const attempt of [1, 2, 3]) {
if (attempt > 1) {
console.log("Retry attempt", attempt);
}
const encryptionPwd1: string = readlineSync.question(
"Enter a password to encrypt the backup archive: ",
{ hideEchoBack: true },
);
const encryptionPwd2: string = readlineSync.question(
"Enter the above password again: ",
{ hideEchoBack: true },
);
if (encryptionPwd1 === encryptionPwd2) {
if (encryptionPwd1) {
return encryptionPwd1;
}
console.error(
"Invalid input. Empty password is not allowed, please try again.",
);
} else {
console.error("The passwords do not match, please try again.");
}
}
console.error(
"Aborting backup process, failed to obtain valid encryption password.",
);
throw new Error(
"Backup process aborted because a valid encryption password could not be obtained from the user",
);
}
export async function encryptBackupArchive(
archivePath: string,
encryptionPassword: string,
) {
const encryptedArchivePath = archivePath + ".enc";
await utils.execCommand([
"openssl",
"enc",
"-aes-256-cbc",
"-pbkdf2",
"-iter",
"100000",
"-in",
archivePath,
"-out",
encryptedArchivePath,
"-k",
encryptionPassword,
]);
return encryptedArchivePath;
}

65
app/client/packages/rts/src/ctl/backup/links/EnvFileLink.ts Normal file
View File

@ -0,0 +1,65 @@
import type { Link } from ".";
import type { BackupState } from "../BackupState";
import fsPromises from "fs/promises";
const SECRETS_WARNING = `
***************************** IMPORTANT!!! *****************************
*** Please ensure you have saved the APPSMITH_ENCRYPTION_SALT and ***
*** APPSMITH_ENCRYPTION_PASSWORD variables from the docker.env file. ***
*** These values are not included in the backup export. ***
************************************************************************
`;
/**
* Exports the docker environment file to the backup folder. If encryption is not enabled, sensitive information is
* not written to the backup folder.
*/
export class EnvFileLink implements Link {
constructor(private readonly state: BackupState) {}
async doBackup() {
console.log("Exporting docker environment file");
const content = await fsPromises.readFile(
"/appsmith-stacks/configuration/docker.env",
{ encoding: "utf8" },
);
let cleanedContent = removeSensitiveEnvData(content);
if (this.state.isEncryptionEnabled) {
cleanedContent +=
"\nAPPSMITH_ENCRYPTION_SALT=" +
process.env.APPSMITH_ENCRYPTION_SALT +
"\nAPPSMITH_ENCRYPTION_PASSWORD=" +
process.env.APPSMITH_ENCRYPTION_PASSWORD;
}
await fsPromises.writeFile(
this.state.backupRootPath + "/docker.env",
cleanedContent,
);
console.log("Exporting docker environment file done.");
}
async postBackup() {
if (!this.state.isEncryptionEnabled) {
console.log(SECRETS_WARNING);
}
}
}
export function removeSensitiveEnvData(content: string): string {
// Remove encryption and Mongodb data from docker.env
const output_lines = [];
content.split(/\r?\n/).forEach((line) => {
if (
!line.startsWith("APPSMITH_ENCRYPTION") &&
!line.startsWith("APPSMITH_MONGODB") &&
!line.startsWith("APPSMITH_DB_URL=")
) {
output_lines.push(line);
}
});
return output_lines.join("\n");
}

37
app/client/packages/rts/src/ctl/backup/links/GitStorageLink.ts Normal file
View File

@ -0,0 +1,37 @@
import type { Link } from ".";
import type { BackupState } from "../BackupState";
import * as utils from "../../utils";
import path from "path";
/**
* Copies the `git-storage` folder to the backup folder.
*/
export class GitStorageLink implements Link {
constructor(private readonly state: BackupState) {}
async doBackup() {
console.log("Creating git-storage archive");
const gitRoot = getGitRoot(process.env.APPSMITH_GIT_ROOT);
await executeCopyCMD(gitRoot, this.state.backupRootPath);
console.log("Created git-storage archive");
}
}
export function getGitRoot(gitRoot?: string | undefined) {
if (gitRoot == null || gitRoot === "") {
gitRoot = "/appsmith-stacks/git-storage";
}
return gitRoot;
}
export async function executeCopyCMD(srcFolder: string, destFolder: string) {
return await utils.execCommand([
"ln",
"-s",
srcFolder,
path.join(destFolder, "git-storage"),
]);
}

5
app/client/packages/rts/src/ctl/backup/links/ManifestLink.ts
View File

@ -1,9 +1,12 @@
import type { Link } from "./index"; import type { Link } from ".";
import type { BackupState } from "../BackupState"; import type { BackupState } from "../BackupState";
import * as utils from "../../utils"; import * as utils from "../../utils";
import fsPromises from "fs/promises"; import fsPromises from "fs/promises";
import path from "path"; import path from "path";
/**
* Creates a manifest file that contains metadata about the backup.
*/
export class ManifestLink implements Link { export class ManifestLink implements Link {
constructor(private readonly state: BackupState) {} constructor(private readonly state: BackupState) {}

25
app/client/packages/rts/src/ctl/backup/links/MongoDumpLink.ts Normal file
View File

@ -0,0 +1,25 @@
import type { Link } from ".";
import type { BackupState } from "../BackupState";
import * as utils from "../../utils";
/**
* Exports the MongoDB database data using mongodump.
*/
export class MongoDumpLink implements Link {
constructor(private readonly state: BackupState) {}
async doBackup() {
console.log("Exporting database");
await executeMongoDumpCMD(this.state.backupRootPath, utils.getDburl());
console.log("Exporting database done.");
}
}
export async function executeMongoDumpCMD(destFolder: string, dbUrl: string) {
return await utils.execCommand([
"mongodump",
`--uri=${dbUrl}`,
`--archive=${destFolder}/mongodb-data.gz`,
"--gzip",
]);
}

9
app/client/packages/rts/src/ctl/backup/links/index.ts
View File

@ -9,5 +9,10 @@ export interface Link {
postBackup?(): Promise<void>; postBackup?(): Promise<void>;
} }
export { EncryptionLink } from "./EncryptionLink"; export * from "./BackupFolderLink";
export { ManifestLink } from "./ManifestLink"; export * from "./DiskSpaceLink";
export * from "./EncryptionLink";
export * from "./EnvFileLink";
export * from "./GitStorageLink";
export * from "./ManifestLink";
export * from "./MongoDumpLink";