feat: appsmith ctl jest with workflow (#17713)
This commit is contained in:
parent
f560fcc4f8
commit
59fc70b36b
102
.github/workflows/appsmithctl.yml
vendored
Normal file
102
.github/workflows/appsmithctl.yml
vendored
Normal file
|
|
@ -0,0 +1,102 @@
|
||||||
|
# This workflow is responsible for building, testing & packaging the Appsmithctl CLI util
|
||||||
|
name: Build Appsmithctl CLI util Workflow
|
||||||
|
|
||||||
|
on:
|
||||||
|
# This line enables manual triggering of this workflow.
|
||||||
|
workflow_dispatch:
|
||||||
|
workflow_call:
|
||||||
|
inputs:
|
||||||
|
pr:
|
||||||
|
description: "This is the PR number in case the workflow is being called in a pull request"
|
||||||
|
required: false
|
||||||
|
type: number
|
||||||
|
|
||||||
|
pull_request:
|
||||||
|
branches: [release, master]
|
||||||
|
paths:
|
||||||
|
- "deploy/docker/utils/**"
|
||||||
|
|
||||||
|
# Change the working directory for all the jobs in this workflow
|
||||||
|
defaults:
|
||||||
|
run:
|
||||||
|
working-directory: deploy/docker/utils/
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
build:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
# Only run this workflow for internally triggered events
|
||||||
|
if: |
|
||||||
|
github.event.pull_request.head.repo.full_name == github.repository ||
|
||||||
|
github.event_name == 'push' ||
|
||||||
|
github.event_name == 'workflow_dispatch' ||
|
||||||
|
github.event_name == 'repository_dispatch'
|
||||||
|
|
||||||
|
steps:
|
||||||
|
# The checkout steps MUST happen first because the default directory is set according to the code base.
|
||||||
|
# Github Action expects all future commands to be executed in the code directory. Hence, we need to check out
|
||||||
|
# the code before doing anything else.
|
||||||
|
|
||||||
|
# Check out merge commit with the base branch in case this workflow is invoked via pull request
|
||||||
|
- name: Checkout the merged commit from PR and base branch
|
||||||
|
if: inputs.pr != 0
|
||||||
|
uses: actions/checkout@v2
|
||||||
|
with:
|
||||||
|
fetch-depth: 0
|
||||||
|
ref: refs/pull/${{ inputs.pr }}/merge
|
||||||
|
|
||||||
|
# Checkout the code in the current branch in case the workflow is called because of a branch push event
|
||||||
|
- name: Checkout the head commit of the branch
|
||||||
|
if: inputs.pr == 0
|
||||||
|
uses: actions/checkout@v2
|
||||||
|
with:
|
||||||
|
fetch-depth: 0
|
||||||
|
|
||||||
|
- name: Figure out the PR number
|
||||||
|
run: echo ${{ inputs.pr }}
|
||||||
|
|
||||||
|
- name: Print the Github event
|
||||||
|
run: echo ${{ github.event_name }}
|
||||||
|
|
||||||
|
# In case this is second attempt try restoring status of the prior attempt from cache
|
||||||
|
- name: Restore the previous run result
|
||||||
|
uses: actions/cache@v2
|
||||||
|
with:
|
||||||
|
path: |
|
||||||
|
~/appsmithctl_run_result
|
||||||
|
key: ${{ github.run_id }}-${{ github.job }}-appsmithctl-util
|
||||||
|
|
||||||
|
# Fetch prior run result
|
||||||
|
- name: Get the previous run result
|
||||||
|
id: appsmithctl_run_result
|
||||||
|
run: cat ~/appsmithctl_run_result 2>/dev/null || echo 'default'
|
||||||
|
|
||||||
|
# Incase of prior failure run the job
|
||||||
|
- if: steps.appsmithctl_run_result.outputs.appsmithctl_run_result != 'success'
|
||||||
|
run: echo "I'm alive!" && exit 0
|
||||||
|
|
||||||
|
- name: Use Node.js 16.14.0
|
||||||
|
if: steps.appsmithctl_run_result.outputs.appsmithctl_run_result != 'success'
|
||||||
|
uses: actions/setup-node@v1
|
||||||
|
with:
|
||||||
|
node-version: "16.14.0"
|
||||||
|
|
||||||
|
# Install all the dependencies
|
||||||
|
- name: Install dependencies
|
||||||
|
if: steps.appsmithctl_run_result.outputs.appsmithctl_run_result != 'success'
|
||||||
|
run: yarn install --frozen-lockfile
|
||||||
|
|
||||||
|
# Run the Jest tests only if the workflow has been invoked in a PR
|
||||||
|
- name: Run the jest tests
|
||||||
|
if: steps.appsmithctl_run_result.outputs.appsmithctl_run_result != 'success'
|
||||||
|
run: yarn run test
|
||||||
|
|
||||||
|
# Set status = failure
|
||||||
|
- name: Set result as failed if there are build failures
|
||||||
|
if: failure()
|
||||||
|
run: |
|
||||||
|
echo "::set-output name=appsmithctl_run_result::failed" > ~/appsmithctl_run_result
|
||||||
|
exit 1;
|
||||||
|
|
||||||
|
|
||||||
|
# Set status = success
|
||||||
|
- run: echo "::set-output name=appsmithctl_run_result::success" > ~/appsmithctl_run_result
|
||||||
|
|
@ -49,6 +49,13 @@ jobs:
|
||||||
with:
|
with:
|
||||||
pr: ${{ github.event.client_payload.pull_request.number }}
|
pr: ${{ github.event.client_payload.pull_request.number }}
|
||||||
|
|
||||||
|
test-appsmithctl:
|
||||||
|
name: appsmithctl
|
||||||
|
uses: ./.github/workflows/appsmithctl.yml
|
||||||
|
secrets: inherit
|
||||||
|
with:
|
||||||
|
pr: ${{ github.event.client_payload.pull_request.number }}
|
||||||
|
|
||||||
fat-container-test:
|
fat-container-test:
|
||||||
needs: [client-build, server-build, rts-build]
|
needs: [client-build, server-build, rts-build]
|
||||||
# Only run if the build step is successful
|
# Only run if the build step is successful
|
||||||
|
|
|
||||||
|
|
@ -1,9 +1,7 @@
|
||||||
const fsPromises = require('fs/promises');
|
const fsPromises = require('fs/promises');
|
||||||
const path = require('path');
|
const path = require('path');
|
||||||
const os = require('os');
|
const os = require('os');
|
||||||
|
|
||||||
const shell = require('shelljs');
|
const shell = require('shelljs');
|
||||||
|
|
||||||
const utils = require('./utils');
|
const utils = require('./utils');
|
||||||
const Constants = require('./constants');
|
const Constants = require('./constants');
|
||||||
const logger = require('./logger');
|
const logger = require('./logger');
|
||||||
|
|
@ -12,8 +10,7 @@ const mailer = require('./mailer');
|
||||||
const command_args = process.argv.slice(3);
|
const command_args = process.argv.slice(3);
|
||||||
|
|
||||||
async function run() {
|
async function run() {
|
||||||
|
const timestamp = getTimeStampInISO();
|
||||||
const timestamp = new Date().toISOString().replace(/:/g, '-')
|
|
||||||
let errorCode = 0;
|
let errorCode = 0;
|
||||||
try {
|
try {
|
||||||
const check_supervisord_status_cmd = '/usr/bin/supervisorctl >/dev/null 2>&1';
|
const check_supervisord_status_cmd = '/usr/bin/supervisorctl >/dev/null 2>&1';
|
||||||
|
|
@ -27,15 +24,13 @@ async function run() {
|
||||||
utils.stop(['backend', 'rts']);
|
utils.stop(['backend', 'rts']);
|
||||||
|
|
||||||
console.log('Available free space at /appsmith-stacks');
|
console.log('Available free space at /appsmith-stacks');
|
||||||
const availSpaceInBytes = parseInt(shell.exec('df --output=avail -B 1 /appsmith-stacks | tail -n 1'), 10);
|
const availSpaceInBytes = getAvailableBackupSpaceInBytes();
|
||||||
console.log('\n');
|
console.log('\n');
|
||||||
|
|
||||||
if (availSpaceInBytes < Constants.MIN_REQUIRED_DISK_SPACE_IN_BYTES) {
|
checkAvailableBackupSpace(availSpaceInBytes);
|
||||||
throw new Error('Not enough space avaliable at /appsmith-stacks. Please ensure availability of atleast 5GB to backup successfully.');
|
|
||||||
}
|
|
||||||
|
|
||||||
const backupRootPath = await fsPromises.mkdtemp(path.join(os.tmpdir(), 'appsmithctl-backup-'));
|
const backupRootPath = await generateBackupRootPath();
|
||||||
const backupContentsPath = backupRootPath + '/appsmith-backup-' + timestamp;
|
const backupContentsPath = getBackupContentsPath(backupRootPath, timestamp);
|
||||||
|
|
||||||
await fsPromises.mkdir(backupContentsPath);
|
await fsPromises.mkdir(backupContentsPath);
|
||||||
|
|
||||||
|
|
@ -50,7 +45,7 @@ async function run() {
|
||||||
|
|
||||||
await fsPromises.rm(backupRootPath, { recursive: true, force: true });
|
await fsPromises.rm(backupRootPath, { recursive: true, force: true });
|
||||||
|
|
||||||
console.log('Finished taking a backup at', archivePath);
|
logger.backup_info('Finished taking a backup at' + archivePath);
|
||||||
await postBackupCleanup();
|
await postBackupCleanup();
|
||||||
|
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
|
|
@ -60,7 +55,7 @@ async function run() {
|
||||||
if (command_args.includes('--error-mail')) {
|
if (command_args.includes('--error-mail')) {
|
||||||
const currentTS = new Date().getTime();
|
const currentTS = new Date().getTime();
|
||||||
const lastMailTS = await utils.getLastBackupErrorMailSentInMilliSec();
|
const lastMailTS = await utils.getLastBackupErrorMailSentInMilliSec();
|
||||||
if ((lastMailTS + Constants.DURATION_BETWEEN_BACKUP_ERROR_MAILS_IN_MILLI_SEC) < currentTS){
|
if ((lastMailTS + Constants.DURATION_BETWEEN_BACKUP_ERROR_MAILS_IN_MILLI_SEC) < currentTS) {
|
||||||
await mailer.sendBackupErrorToAdmins(err, timestamp);
|
await mailer.sendBackupErrorToAdmins(err, timestamp);
|
||||||
await utils.updateLastBackupErrorMailSentInMilliSec(currentTS);
|
await utils.updateLastBackupErrorMailSentInMilliSec(currentTS);
|
||||||
}
|
}
|
||||||
|
|
@ -68,32 +63,27 @@ async function run() {
|
||||||
} finally {
|
} finally {
|
||||||
utils.start(['backend', 'rts']);
|
utils.start(['backend', 'rts']);
|
||||||
process.exit(errorCode);
|
process.exit(errorCode);
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function exportDatabase(destFolder) {
|
async function exportDatabase(destFolder) {
|
||||||
console.log('Exporting database');
|
console.log('Exporting database');
|
||||||
await utils.execCommand(['mongodump', `--uri=${process.env.APPSMITH_MONGODB_URI}`, `--archive=${destFolder}/mongodb-data.gz`, '--gzip']);
|
await executeMongoDumpCMD(destFolder, process.env.APPSMITH_MONGODB_URI)
|
||||||
console.log('Exporting database done.');
|
console.log('Exporting database done.');
|
||||||
}
|
}
|
||||||
|
|
||||||
async function createGitStorageArchive(destFolder) {
|
async function createGitStorageArchive(destFolder) {
|
||||||
console.log('Creating git-storage archive');
|
console.log('Creating git-storage archive');
|
||||||
|
|
||||||
let gitRoot = process.env.APPSMITH_GIT_ROOT;
|
const gitRoot = getGitRoot(process.env.APPSMITH_GIT_ROOT);
|
||||||
if (gitRoot == null || gitRoot === '') {
|
|
||||||
gitRoot = '/appsmith-stacks/git-storage';
|
|
||||||
}
|
|
||||||
|
|
||||||
await utils.execCommand(['ln', '-s', gitRoot, destFolder + '/git-storage'])
|
await executeCopyCMD(gitRoot, destFolder)
|
||||||
|
|
||||||
console.log('Created git-storage archive');
|
console.log('Created git-storage archive');
|
||||||
}
|
}
|
||||||
|
|
||||||
async function createManifestFile(path) {
|
async function createManifestFile(path) {
|
||||||
const content = await fsPromises.readFile('/opt/appsmith/rts/version.js', { encoding: 'utf8' });
|
const version = await getCurrentVersion()
|
||||||
const version = content.match(/\bexports\.VERSION\s*=\s*["']([^"]+)["']/)[1];
|
|
||||||
const manifest_data = { "appsmithVersion": version }
|
const manifest_data = { "appsmithVersion": version }
|
||||||
await fsPromises.writeFile(path + '/manifest.json', JSON.stringify(manifest_data));
|
await fsPromises.writeFile(path + '/manifest.json', JSON.stringify(manifest_data));
|
||||||
}
|
}
|
||||||
|
|
@ -101,20 +91,18 @@ async function createManifestFile(path) {
|
||||||
async function exportDockerEnvFile(destFolder) {
|
async function exportDockerEnvFile(destFolder) {
|
||||||
console.log('Exporting docker environment file');
|
console.log('Exporting docker environment file');
|
||||||
const content = await fsPromises.readFile('/appsmith-stacks/configuration/docker.env', { encoding: 'utf8' });
|
const content = await fsPromises.readFile('/appsmith-stacks/configuration/docker.env', { encoding: 'utf8' });
|
||||||
const output_lines = []
|
const cleaned_content = removeEncryptionEnvData(content)
|
||||||
content.split(/\r?\n/).forEach(line => {
|
await fsPromises.writeFile(destFolder + '/docker.env', cleaned_content);
|
||||||
if (!line.startsWith("APPSMITH_ENCRYPTION")) {
|
|
||||||
output_lines.push(line)
|
|
||||||
}
|
|
||||||
});
|
|
||||||
await fsPromises.writeFile(destFolder + '/docker.env', output_lines.join('\n'));
|
|
||||||
console.log('Exporting docker environment file done.');
|
console.log('Exporting docker environment file done.');
|
||||||
|
|
||||||
console.log('!!!!!!!!!!!!!!!!!!!!!!!!!! Important !!!!!!!!!!!!!!!!!!!!!!!!!!');
|
console.log('!!!!!!!!!!!!!!!!!!!!!!!!!! Important !!!!!!!!!!!!!!!!!!!!!!!!!!');
|
||||||
console.log('!!! Please ensure you have saved the APPSMITH_ENCRYPTION_SALT and APPSMITH_ENCRYPTION_PASSWORD variables from the docker.env file because those values are not included in the backup export.');
|
console.log('!!! Please ensure you have saved the APPSMITH_ENCRYPTION_SALT and APPSMITH_ENCRYPTION_PASSWORD variables from the docker.env file because those values are not included in the backup export.');
|
||||||
console.log('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!');
|
console.log('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function executeMongoDumpCMD(destFolder, appsmithMongoURI) {
|
||||||
|
return await utils.execCommand(['mongodump', `--uri=${appsmithMongoURI}`, `--archive=${destFolder}/mongodb-data.gz`, '--gzip']);// generate cmd
|
||||||
|
}
|
||||||
|
|
||||||
async function createFinalArchive(destFolder, timestamp) {
|
async function createFinalArchive(destFolder, timestamp) {
|
||||||
console.log('Creating final archive');
|
console.log('Creating final archive');
|
||||||
|
|
||||||
|
|
@ -126,20 +114,92 @@ async function createFinalArchive(destFolder, timestamp) {
|
||||||
return archive;
|
return archive;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function postBackupCleanup(){
|
async function postBackupCleanup() {
|
||||||
console.log('Starting the cleanup task after taking a backup.');
|
console.log('Starting the cleanup task after taking a backup.');
|
||||||
let backupArchivesLimit = process.env.APPSMITH_BACKUP_ARCHIVE_LIMIT;
|
let backupArchivesLimit = getBackupArchiveLimit(process.env.APPSMITH_BACKUP_ARCHIVE_LIMIT);
|
||||||
if(!backupArchivesLimit)
|
|
||||||
backupArchivesLimit = 4;
|
|
||||||
const backupFiles = await utils.listLocalBackupFiles();
|
const backupFiles = await utils.listLocalBackupFiles();
|
||||||
while (backupFiles.length > backupArchivesLimit){
|
while (backupFiles.length > backupArchivesLimit) {
|
||||||
const fileName = backupFiles.shift();
|
const fileName = backupFiles.shift();
|
||||||
await fsPromises.rm(Constants.BACKUP_PATH + '/' + fileName);
|
await fsPromises.rm(Constants.BACKUP_PATH + '/' + fileName);
|
||||||
}
|
}
|
||||||
console.log('Cleanup task completed.');
|
console.log('Cleanup task completed.');
|
||||||
|
|
||||||
}
|
}
|
||||||
|
async function executeCopyCMD(srcFolder, destFolder) {
|
||||||
|
return await utils.execCommand(['ln', '-s', srcFolder, destFolder + '/git-storage'])
|
||||||
|
}
|
||||||
|
|
||||||
|
function getGitRoot(gitRoot) {
|
||||||
|
if (gitRoot == null || gitRoot === '') {
|
||||||
|
gitRoot = '/appsmith-stacks/git-storage';
|
||||||
|
}
|
||||||
|
return gitRoot
|
||||||
|
}
|
||||||
|
|
||||||
|
async function generateBackupRootPath() {
|
||||||
|
const backupRootPath = await fsPromises.mkdtemp(path.join(os.tmpdir(), 'appsmithctl-backup-'));
|
||||||
|
return backupRootPath
|
||||||
|
}
|
||||||
|
|
||||||
|
function getBackupContentsPath(backupRootPath, timestamp) {
|
||||||
|
return backupRootPath + '/appsmith-backup-' + timestamp;
|
||||||
|
}
|
||||||
|
|
||||||
|
function removeEncryptionEnvData(content) {
|
||||||
|
const output_lines = []
|
||||||
|
content.split(/\r?\n/).forEach(line => {
|
||||||
|
if (!line.startsWith("APPSMITH_ENCRYPTION")) {
|
||||||
|
output_lines.push(line)
|
||||||
|
}
|
||||||
|
});
|
||||||
|
return output_lines.join('\n')
|
||||||
|
}
|
||||||
|
|
||||||
|
function getBackupArchiveLimit(backupArchivesLimit) {
|
||||||
|
if (!backupArchivesLimit)
|
||||||
|
backupArchivesLimit = 4;
|
||||||
|
return backupArchivesLimit
|
||||||
|
}
|
||||||
|
|
||||||
|
async function removeOldBackups(backupFiles, backupArchivesLimit) {
|
||||||
|
while (backupFiles.length > backupArchivesLimit) {
|
||||||
|
const fileName = backupFiles.shift();
|
||||||
|
await fsPromises.rm(Constants.BACKUP_PATH + '/' + fileName);
|
||||||
|
}
|
||||||
|
return backupFiles
|
||||||
|
}
|
||||||
|
|
||||||
|
function getTimeStampInISO() {
|
||||||
|
return new Date().toISOString().replace(/:/g, '-')
|
||||||
|
}
|
||||||
|
|
||||||
|
function getAvailableBackupSpaceInBytes() {
|
||||||
|
return parseInt(shell.exec('df --output=avail -B 1 /appsmith-stacks | tail -n 1'), 10)
|
||||||
|
}
|
||||||
|
|
||||||
|
function checkAvailableBackupSpace(availSpaceInBytes) {
|
||||||
|
if (availSpaceInBytes < Constants.MIN_REQUIRED_DISK_SPACE_IN_BYTES) {
|
||||||
|
throw new Error('Not enough space avaliable at /appsmith-stacks. Please ensure availability of atleast 5GB to backup successfully.');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function getCurrentVersion() {
|
||||||
|
const content = await fsPromises.readFile('/opt/appsmith/rts/version.js', { encoding: 'utf8' });
|
||||||
|
return content.match(/\bexports\.VERSION\s*=\s*["']([^"]+)["']/)[1];
|
||||||
|
}
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
run,
|
run,
|
||||||
|
getTimeStampInISO,
|
||||||
|
getAvailableBackupSpaceInBytes,
|
||||||
|
checkAvailableBackupSpace,
|
||||||
|
generateBackupRootPath,
|
||||||
|
getBackupContentsPath,
|
||||||
|
executeMongoDumpCMD,
|
||||||
|
getGitRoot,
|
||||||
|
executeCopyCMD,
|
||||||
|
getCurrentVersion,
|
||||||
|
removeEncryptionEnvData,
|
||||||
|
getBackupArchiveLimit,
|
||||||
|
removeOldBackups
|
||||||
};
|
};
|
||||||
|
|
|
||||||
181
deploy/docker/utils/bin/backup.test.js
Normal file
181
deploy/docker/utils/bin/backup.test.js
Normal file
|
|
@ -0,0 +1,181 @@
|
||||||
|
const backup = require('./backup');
|
||||||
|
const Constants = require('./constants');
|
||||||
|
const os = require('os');
|
||||||
|
const fsPromises = require('fs/promises');
|
||||||
|
const utils = require('./utils');
|
||||||
|
const shell = require('shelljs');
|
||||||
|
|
||||||
|
describe('Backup Tests', () => {
|
||||||
|
|
||||||
|
test('Timestamp string in ISO format', () => {
|
||||||
|
console.log(backup.getTimeStampInISO())
|
||||||
|
expect(backup.getTimeStampInISO()).toMatch(/(\d{4})-(\d{2})-(\d{2})T(\d{2})\-(\d{2})\-(\d{2})\.(\d{3})Z/)
|
||||||
|
});
|
||||||
|
|
||||||
|
test('Available Space in /appsmith-stacks volume in Bytes', () => {
|
||||||
|
shell.exec = jest.fn((format) => '20');
|
||||||
|
const res = expect(backup.getAvailableBackupSpaceInBytes())
|
||||||
|
res.toBe(20)
|
||||||
|
|
||||||
|
});
|
||||||
|
it('Checkx the constant is 2 GB', () => {
|
||||||
|
let size = 2 * 1024 * 1024 * 1024
|
||||||
|
expect(Constants.MIN_REQUIRED_DISK_SPACE_IN_BYTES).toBe(size)
|
||||||
|
});
|
||||||
|
it('Should throw Error when the available size is below MIN_REQUIRED_DISK_SPACE_IN_BYTES', () => {
|
||||||
|
let size = Constants.MIN_REQUIRED_DISK_SPACE_IN_BYTES - 1;
|
||||||
|
expect(() => {backup.checkAvailableBackupSpace(size)}).toThrow('Not enough space avaliable at /appsmith-stacks. Please ensure availability of atleast 5GB to backup successfully.');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('Should not hould throw Error when the available size is >= MIN_REQUIRED_DISK_SPACE_IN_BYTES', () => {
|
||||||
|
expect(() => {backup.checkAvailableBackupSpace(Constants.MIN_REQUIRED_DISK_SPACE_IN_BYTES)}).not.toThrow('Not enough space avaliable at /appsmith-stacks. Please ensure availability of atleast 5GB to backup successfully.');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('Generates t', async () => {
|
||||||
|
os.tmpdir = jest.fn().mockReturnValue('temp/dir');
|
||||||
|
fsPromises.mkdtemp = jest.fn().mockImplementation((a) => a);
|
||||||
|
backup.generateBackupRootPath().then((response)=>{console.log(response)})
|
||||||
|
const res = await backup.generateBackupRootPath()
|
||||||
|
expect(res).toBe('temp/dir/appsmithctl-backup-')
|
||||||
|
});
|
||||||
|
|
||||||
|
test('Test backup contents path generation', () => {
|
||||||
|
var root = '/rootDir'
|
||||||
|
var timestamp = '0000-00-0T00-00-00.00Z'
|
||||||
|
expect(backup.getBackupContentsPath(root, timestamp)).toBe('/rootDir/appsmith-backup-0000-00-0T00-00-00.00Z')
|
||||||
|
});
|
||||||
|
|
||||||
|
test('Test mongodump CMD generaton', async () => {
|
||||||
|
var dest = '/dest'
|
||||||
|
var appsmithMongoURI = 'mongodb://username:password@host/appsmith'
|
||||||
|
var cmd = 'mongodump --uri=mongodb://username:password@host/appsmith --archive=/dest/mongodb-data.gz --gzip'
|
||||||
|
utils.execCommand = jest.fn().mockImplementation(async (a) => a.join(' '));
|
||||||
|
const res = await backup.executeMongoDumpCMD(dest, appsmithMongoURI)
|
||||||
|
expect(res).toBe(cmd)
|
||||||
|
console.log(res)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('Test get gitRoot path when APPSMITH_GIT_ROOT is \'\' ', () => {
|
||||||
|
expect(backup.getGitRoot('')).toBe('/appsmith-stacks/git-storage')
|
||||||
|
});
|
||||||
|
|
||||||
|
test('Test get gitRoot path when APPSMITH_GIT_ROOT is null ', () => {
|
||||||
|
expect(backup.getGitRoot()).toBe('/appsmith-stacks/git-storage')
|
||||||
|
});
|
||||||
|
|
||||||
|
test('Test get gitRoot path when APPSMITH_GIT_ROOT is defined ', () => {
|
||||||
|
expect(backup.getGitRoot('/my/git/storage')).toBe('/my/git/storage')
|
||||||
|
});
|
||||||
|
|
||||||
|
test('Test ln command generation', async () => {
|
||||||
|
var gitRoot = '/appsmith-stacks/git-storage'
|
||||||
|
var dest = '/destdir'
|
||||||
|
var cmd = 'ln -s /appsmith-stacks/git-storage /destdir/git-storage'
|
||||||
|
utils.execCommand = jest.fn().mockImplementation(async (a) => a.join(' '));
|
||||||
|
const res = await backup.executeCopyCMD(gitRoot, dest)
|
||||||
|
expect(res).toBe(cmd)
|
||||||
|
console.log(res)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('Checks for the current Appsmith Version.', async () => {
|
||||||
|
|
||||||
|
fsPromises.readFile = jest.fn().mockImplementation(async (a) =>
|
||||||
|
`Object.defineProperty(exports, "__esModule", { value: true });
|
||||||
|
exports.VERSION = void 0;
|
||||||
|
exports.VERSION = "v0.0.0-SNAPSHOT";`);
|
||||||
|
const res = await backup.getCurrentVersion()
|
||||||
|
expect(res).toBe("v0.0.0-SNAPSHOT")
|
||||||
|
console.log(res)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('If encriytpion env values are being removed', () => {
|
||||||
|
expect(backup.removeEncryptionEnvData(`APPSMITH_REDIS_URL=redis://127.0.0.1:6379\nAPPSMITH_ENCRYPTION_PASSWORD=dummy-pass\nAPPSMITH_ENCRYPTION_SALT=dummy-salt\nAPPSMITH_INSTANCE_NAME=Appsmith\n
|
||||||
|
`)).toMatch(`APPSMITH_REDIS_URL=redis://127.0.0.1:6379\nAPPSMITH_INSTANCE_NAME=Appsmith\n`)
|
||||||
|
});
|
||||||
|
|
||||||
|
test('Backup Archive Limit when env APPSMITH_BACKUP_ARCHIVE_LIMIT is null', () => {
|
||||||
|
expect(backup.getBackupArchiveLimit()).toBe(4)
|
||||||
|
});
|
||||||
|
|
||||||
|
test('Backup Archive Limit when env APPSMITH_BACKUP_ARCHIVE_LIMIT is 5', () => {
|
||||||
|
expect(backup.getBackupArchiveLimit(5)).toBe(5)
|
||||||
|
});
|
||||||
|
|
||||||
|
|
||||||
|
test('Cleanup Backups when limit is 4 and there are 5 files', async () => {
|
||||||
|
const backupArchivesLimit = 4;
|
||||||
|
fsPromises.rm = jest.fn().mockImplementation(async (a) => console.log(a));
|
||||||
|
var backupFiles = ['file1','file2','file3','file4','file5']
|
||||||
|
var expectedBackupFiles = ['file2','file3','file4','file5']
|
||||||
|
const res = await backup.removeOldBackups(backupFiles,backupArchivesLimit)
|
||||||
|
console.log(res)
|
||||||
|
|
||||||
|
expect(res).toEqual(expectedBackupFiles)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('Cleanup Backups when limit is 2 and there are 5 files', async () => {
|
||||||
|
const backupArchivesLimit = 2;
|
||||||
|
fsPromises.rm = jest.fn().mockImplementation(async (a) => console.log(a));
|
||||||
|
var backupFiles = ['file1','file2','file3','file4','file5']
|
||||||
|
var expectedBackupFiles = ['file4','file5']
|
||||||
|
const res = await backup.removeOldBackups(backupFiles,backupArchivesLimit)
|
||||||
|
console.log(res)
|
||||||
|
|
||||||
|
expect(res).toEqual(expectedBackupFiles)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('Cleanup Backups when limit is 4 and there are 4 files', async () => {
|
||||||
|
const backupArchivesLimit = 4;
|
||||||
|
fsPromises.rm = jest.fn().mockImplementation(async (a) => console.log(a));
|
||||||
|
var backupFiles = ['file1','file2','file3','file4']
|
||||||
|
var expectedBackupFiles = ['file1','file2','file3','file4']
|
||||||
|
const res = await backup.removeOldBackups(backupFiles,backupArchivesLimit)
|
||||||
|
console.log(res)
|
||||||
|
|
||||||
|
expect(res).toEqual(expectedBackupFiles)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('Cleanup Backups when limit is 4 and there are 2 files', async () => {
|
||||||
|
const backupArchivesLimit = 4;
|
||||||
|
fsPromises.rm = jest.fn().mockImplementation(async (a) => console.log(a));
|
||||||
|
var backupFiles = ['file1','file2']
|
||||||
|
var expectedBackupFiles = ['file1','file2']
|
||||||
|
const res = await backup.removeOldBackups(backupFiles,backupArchivesLimit)
|
||||||
|
console.log(res)
|
||||||
|
|
||||||
|
expect(res).toEqual(expectedBackupFiles)
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
test('Cleanup Backups when limit is 4 and there are 2 files', async () => {
|
||||||
|
const backupArchivesLimit = 4;
|
||||||
|
fsPromises.rm = jest.fn().mockImplementation(async (a) => console.log(a));
|
||||||
|
var backupFiles = ['file1','file2']
|
||||||
|
var expectedBackupFiles = ['file1','file2']
|
||||||
|
const res = await backup.removeOldBackups(backupFiles,backupArchivesLimit)
|
||||||
|
console.log(res)
|
||||||
|
expect(res).toEqual(expectedBackupFiles)
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
test('Cleanup Backups when limit is 2 and there is 1 file', async () => {
|
||||||
|
const backupArchivesLimit = 4;
|
||||||
|
fsPromises.rm = jest.fn().mockImplementation(async (a) => console.log(a));
|
||||||
|
var backupFiles = ['file1']
|
||||||
|
var expectedBackupFiles = ['file1']
|
||||||
|
const res = await backup.removeOldBackups(backupFiles,backupArchivesLimit)
|
||||||
|
console.log(res)
|
||||||
|
expect(res).toEqual(expectedBackupFiles)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('Cleanup Backups when limit is 2 and there is no file', async () => {
|
||||||
|
const backupArchivesLimit = 4;
|
||||||
|
fsPromises.rm = jest.fn().mockImplementation(async (a) => console.log(a));
|
||||||
|
var backupFiles = []
|
||||||
|
var expectedBackupFiles = []
|
||||||
|
const res = await backup.removeOldBackups(backupFiles,backupArchivesLimit)
|
||||||
|
console.log(res)
|
||||||
|
expect(res).toEqual(expectedBackupFiles)
|
||||||
|
})
|
||||||
|
});
|
||||||
|
|
||||||
|
|
@ -5,7 +5,7 @@ const RESTORE_PATH = "/appsmith-stacks/data/restore"
|
||||||
|
|
||||||
const DUMP_FILE_NAME = "appsmith-data.archive"
|
const DUMP_FILE_NAME = "appsmith-data.archive"
|
||||||
|
|
||||||
const BACKUP_ERROR_LOG_PATH = "/appsmith-stacks/logs/backup"
|
const APPSMITHCTL_LOG_PATH = "/appsmith-stacks/logs/appsmithctl"
|
||||||
|
|
||||||
const LAST_ERROR_MAIL_TS = "/appsmith-stacks/data/backup/last-error-mail-ts"
|
const LAST_ERROR_MAIL_TS = "/appsmith-stacks/data/backup/last-error-mail-ts"
|
||||||
|
|
||||||
|
|
@ -18,7 +18,7 @@ module.exports = {
|
||||||
RESTORE_PATH,
|
RESTORE_PATH,
|
||||||
DUMP_FILE_NAME,
|
DUMP_FILE_NAME,
|
||||||
LAST_ERROR_MAIL_TS,
|
LAST_ERROR_MAIL_TS,
|
||||||
BACKUP_ERROR_LOG_PATH,
|
APPSMITHCTL_LOG_PATH,
|
||||||
MIN_REQUIRED_DISK_SPACE_IN_BYTES,
|
MIN_REQUIRED_DISK_SPACE_IN_BYTES,
|
||||||
DURATION_BETWEEN_BACKUP_ERROR_MAILS_IN_MILLI_SEC,
|
DURATION_BETWEEN_BACKUP_ERROR_MAILS_IN_MILLI_SEC,
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -4,13 +4,24 @@ const Constants = require('./constants');
|
||||||
async function backup_error(err) {
|
async function backup_error(err) {
|
||||||
console.error(err);
|
console.error(err);
|
||||||
try {
|
try {
|
||||||
await fsPromises.access(Constants.BACKUP_ERROR_LOG_PATH);
|
await fsPromises.access(Constants.APPSMITHCTL_LOG_PATH);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
await fsPromises.mkdir(Constants.BACKUP_ERROR_LOG_PATH);
|
await fsPromises.mkdir(Constants.APPSMITHCTL_LOG_PATH);
|
||||||
}
|
}
|
||||||
await fsPromises.appendFile(Constants.BACKUP_ERROR_LOG_PATH + '/error.log', new Date().toISOString() + ' ' + err + '\n');
|
await fsPromises.appendFile(Constants.APPSMITHCTL_LOG_PATH + '/backup.log', new Date().toISOString() + ' [ ERROR ] ' + err + '\n');
|
||||||
|
}
|
||||||
|
|
||||||
|
async function backup_info(msg) {
|
||||||
|
console.log(msg);
|
||||||
|
try {
|
||||||
|
await fsPromises.access(Constants.APPSMITHCTL_LOG_PATH);
|
||||||
|
} catch (error) {
|
||||||
|
await fsPromises.mkdir(Constants.APPSMITHCTL_LOG_PATH);
|
||||||
|
}
|
||||||
|
await fsPromises.appendFile(Constants.APPSMITHCTL_LOG_PATH + '/backup.log', new Date().toISOString() + ' [ INFO ] ' + msg + '\n');
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
backup_error,
|
backup_error,
|
||||||
|
backup_info,
|
||||||
};
|
};
|
||||||
|
|
|
||||||
5582
deploy/docker/utils/package-lock.json
generated
5582
deploy/docker/utils/package-lock.json
generated
File diff suppressed because it is too large
Load Diff
|
|
@ -12,16 +12,20 @@
|
||||||
"directory": "deploy/docker"
|
"directory": "deploy/docker"
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"dotenv": "10.0.0",
|
|
||||||
"mongodb": "^4.4.0",
|
|
||||||
"readline-sync": "1.4.10",
|
|
||||||
"shelljs": "0.8.5",
|
|
||||||
"nodemailer": "6.7.5",
|
|
||||||
"cli-progress": "^3.11.2",
|
"cli-progress": "^3.11.2",
|
||||||
|
"dotenv": "10.0.0",
|
||||||
|
"jest": "^29.1.2",
|
||||||
"luxon": "^3.0.1",
|
"luxon": "^3.0.1",
|
||||||
"minimist": "^1.2.6"
|
"minimist": "^1.2.6",
|
||||||
|
"mongodb": "^4.4.0",
|
||||||
|
"nodemailer": "6.7.5",
|
||||||
|
"readline-sync": "1.4.10",
|
||||||
|
"shelljs": "0.8.5"
|
||||||
},
|
},
|
||||||
"bin": {
|
"bin": {
|
||||||
"appsmithctl": "./bin/index.js"
|
"appsmithctl": "./bin/index.js"
|
||||||
|
},
|
||||||
|
"scripts": {
|
||||||
|
"test": "jest"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue
Block a user