Protei/PromucFlow_constructor
2
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

feat: appsmith ctl jest with workflow (#17713)

Browse Source
This commit is contained in:
Sumesh Pradhan 2022-11-01 12:57:41 +05:30 committed by GitHub
parent f560fcc4f8
commit 59fc70b36b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 5991 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

102
.github/workflows/appsmithctl.yml vendored Normal file
View File

@ -0,0 +1,102 @@
# This workflow is responsible for building, testing & packaging the Appsmithctl CLI util
name: Build Appsmithctl CLI util Workflow
on:
# This line enables manual triggering of this workflow.
workflow_dispatch:
workflow_call:
inputs:
pr:
description: "This is the PR number in case the workflow is being called in a pull request"
required: false
type: number
pull_request:
branches: [release, master]
paths:
- "deploy/docker/utils/**"
# Change the working directory for all the jobs in this workflow
defaults:
run:
working-directory: deploy/docker/utils/
jobs:
build:
runs-on: ubuntu-latest
# Only run this workflow for internally triggered events
if: |
github.event.pull_request.head.repo.full_name == github.repository ||
github.event_name == 'push' ||
github.event_name == 'workflow_dispatch' ||
github.event_name == 'repository_dispatch'
steps:
# The checkout steps MUST happen first because the default directory is set according to the code base.
# Github Action expects all future commands to be executed in the code directory. Hence, we need to check out
# the code before doing anything else.
# Check out merge commit with the base branch in case this workflow is invoked via pull request
- name: Checkout the merged commit from PR and base branch
if: inputs.pr != 0
uses: actions/checkout@v2
with:
fetch-depth: 0
ref: refs/pull/${{ inputs.pr }}/merge
# Checkout the code in the current branch in case the workflow is called because of a branch push event
- name: Checkout the head commit of the branch
if: inputs.pr == 0
uses: actions/checkout@v2
with:
fetch-depth: 0
- name: Figure out the PR number
run: echo ${{ inputs.pr }}
- name: Print the Github event
run: echo ${{ github.event_name }}
# In case this is second attempt try restoring status of the prior attempt from cache
- name: Restore the previous run result
uses: actions/cache@v2
with:
path: |
~/appsmithctl_run_result
key: ${{ github.run_id }}-${{ github.job }}-appsmithctl-util
# Fetch prior run result
- name: Get the previous run result
id: appsmithctl_run_result
run: cat ~/appsmithctl_run_result 2>/dev/null || echo 'default'
# Incase of prior failure run the job
- if: steps.appsmithctl_run_result.outputs.appsmithctl_run_result != 'success'
run: echo "I'm alive!" && exit 0
- name: Use Node.js 16.14.0
if: steps.appsmithctl_run_result.outputs.appsmithctl_run_result != 'success'
uses: actions/setup-node@v1
with:
node-version: "16.14.0"
# Install all the dependencies
- name: Install dependencies
if: steps.appsmithctl_run_result.outputs.appsmithctl_run_result != 'success'
run: yarn install --frozen-lockfile
# Run the Jest tests only if the workflow has been invoked in a PR
- name: Run the jest tests
if: steps.appsmithctl_run_result.outputs.appsmithctl_run_result != 'success'
run: yarn run test
# Set status = failure
- name: Set result as failed if there are build failures
if: failure()
run: |
echo "::set-output name=appsmithctl_run_result::failed" > ~/appsmithctl_run_result
exit 1;
# Set status = success
- run: echo "::set-output name=appsmithctl_run_result::success" > ~/appsmithctl_run_result

7
.github/workflows/integration-tests-command.yml vendored
View File

@ -49,6 +49,13 @@ jobs:
with:
pr: ${{ github.event.client_payload.pull_request.number }}
test-appsmithctl:
name: appsmithctl
uses: ./.github/workflows/appsmithctl.yml
secrets: inherit
with:
pr: ${{ github.event.client_payload.pull_request.number }}
fat-container-test:
needs: [client-build, server-build, rts-build]
# Only run if the build step is successful

128
deploy/docker/utils/bin/backup.js
View File

@ -1,9 +1,7 @@
const fsPromises = require('fs/promises');
const path = require('path');
const os = require('os');
const shell = require('shelljs');
const utils = require('./utils');
const Constants = require('./constants');
const logger = require('./logger');
@ -12,8 +10,7 @@ const mailer = require('./mailer');
const command_args = process.argv.slice(3);
async function run() {
const timestamp = new Date().toISOString().replace(/:/g, '-')
const timestamp = getTimeStampInISO();
let errorCode = 0;
try {
const check_supervisord_status_cmd = '/usr/bin/supervisorctl >/dev/null 2>&1';
@ -27,15 +24,13 @@ async function run() {
utils.stop(['backend', 'rts']);
console.log('Available free space at /appsmith-stacks');
const availSpaceInBytes = parseInt(shell.exec('df --output=avail -B 1 /appsmith-stacks | tail -n 1'), 10);
const availSpaceInBytes = getAvailableBackupSpaceInBytes();
console.log('\n');
if (availSpaceInBytes < Constants.MIN_REQUIRED_DISK_SPACE_IN_BYTES) {
throw new Error('Not enough space avaliable at /appsmith-stacks. Please ensure availability of atleast 5GB to backup successfully.');
}
checkAvailableBackupSpace(availSpaceInBytes);
const backupRootPath = await fsPromises.mkdtemp(path.join(os.tmpdir(), 'appsmithctl-backup-'));
const backupContentsPath = backupRootPath + '/appsmith-backup-' + timestamp;
const backupRootPath = await generateBackupRootPath();
const backupContentsPath = getBackupContentsPath(backupRootPath, timestamp);
await fsPromises.mkdir(backupContentsPath);
@ -50,7 +45,7 @@ async function run() {
await fsPromises.rm(backupRootPath, { recursive: true, force: true });
console.log('Finished taking a backup at', archivePath);
logger.backup_info('Finished taking a backup at' + archivePath);
await postBackupCleanup();
} catch (err) {
@ -60,7 +55,7 @@ async function run() {
if (command_args.includes('--error-mail')) {
const currentTS = new Date().getTime();
const lastMailTS = await utils.getLastBackupErrorMailSentInMilliSec();
if ((lastMailTS + Constants.DURATION_BETWEEN_BACKUP_ERROR_MAILS_IN_MILLI_SEC) < currentTS){
if ((lastMailTS + Constants.DURATION_BETWEEN_BACKUP_ERROR_MAILS_IN_MILLI_SEC) < currentTS) {
await mailer.sendBackupErrorToAdmins(err, timestamp);
await utils.updateLastBackupErrorMailSentInMilliSec(currentTS);
}
@ -68,32 +63,27 @@ async function run() {
} finally {
utils.start(['backend', 'rts']);
process.exit(errorCode);
}
}
async function exportDatabase(destFolder) {
console.log('Exporting database');
await utils.execCommand(['mongodump', `--uri=${process.env.APPSMITH_MONGODB_URI}`, `--archive=${destFolder}/mongodb-data.gz`, '--gzip']);
await executeMongoDumpCMD(destFolder, process.env.APPSMITH_MONGODB_URI)
console.log('Exporting database done.');
}
async function createGitStorageArchive(destFolder) {
console.log('Creating git-storage archive');
let gitRoot = process.env.APPSMITH_GIT_ROOT;
if (gitRoot == null || gitRoot === '') {
gitRoot = '/appsmith-stacks/git-storage';
}
const gitRoot = getGitRoot(process.env.APPSMITH_GIT_ROOT);
await utils.execCommand(['ln', '-s', gitRoot, destFolder + '/git-storage'])
await executeCopyCMD(gitRoot, destFolder)
console.log('Created git-storage archive');
}
async function createManifestFile(path) {
const content = await fsPromises.readFile('/opt/appsmith/rts/version.js', { encoding: 'utf8' });
const version = content.match(/\bexports\.VERSION\s*=\s*["']([^"]+)["']/)[1];
const version = await getCurrentVersion()
const manifest_data = { "appsmithVersion": version }
await fsPromises.writeFile(path + '/manifest.json', JSON.stringify(manifest_data));
}
@ -101,20 +91,18 @@ async function createManifestFile(path) {
async function exportDockerEnvFile(destFolder) {
console.log('Exporting docker environment file');
const content = await fsPromises.readFile('/appsmith-stacks/configuration/docker.env', { encoding: 'utf8' });
const output_lines = []
content.split(/\r?\n/).forEach(line => {
if (!line.startsWith("APPSMITH_ENCRYPTION")) {
output_lines.push(line)
}
});
await fsPromises.writeFile(destFolder + '/docker.env', output_lines.join('\n'));
const cleaned_content = removeEncryptionEnvData(content)
await fsPromises.writeFile(destFolder + '/docker.env', cleaned_content);
console.log('Exporting docker environment file done.');
console.log('!!!!!!!!!!!!!!!!!!!!!!!!!! Important !!!!!!!!!!!!!!!!!!!!!!!!!!');
console.log('!!! Please ensure you have saved the APPSMITH_ENCRYPTION_SALT and APPSMITH_ENCRYPTION_PASSWORD variables from the docker.env file because those values are not included in the backup export.');
console.log('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!');
}
async function executeMongoDumpCMD(destFolder, appsmithMongoURI) {
return await utils.execCommand(['mongodump', `--uri=${appsmithMongoURI}`, `--archive=${destFolder}/mongodb-data.gz`, '--gzip']);// generate cmd
}
async function createFinalArchive(destFolder, timestamp) {
console.log('Creating final archive');
@ -126,20 +114,92 @@ async function createFinalArchive(destFolder, timestamp) {
return archive;
}
async function postBackupCleanup(){
async function postBackupCleanup() {
console.log('Starting the cleanup task after taking a backup.');
let backupArchivesLimit = process.env.APPSMITH_BACKUP_ARCHIVE_LIMIT;
if(!backupArchivesLimit)
backupArchivesLimit = 4;
let backupArchivesLimit = getBackupArchiveLimit(process.env.APPSMITH_BACKUP_ARCHIVE_LIMIT);
const backupFiles = await utils.listLocalBackupFiles();
while (backupFiles.length > backupArchivesLimit){
while (backupFiles.length > backupArchivesLimit) {
const fileName = backupFiles.shift();
await fsPromises.rm(Constants.BACKUP_PATH + '/' + fileName);
}
console.log('Cleanup task completed.');
}
async function executeCopyCMD(srcFolder, destFolder) {
return await utils.execCommand(['ln', '-s', srcFolder, destFolder + '/git-storage'])
}
function getGitRoot(gitRoot) {
if (gitRoot == null || gitRoot === '') {
gitRoot = '/appsmith-stacks/git-storage';
}
return gitRoot
}
async function generateBackupRootPath() {
const backupRootPath = await fsPromises.mkdtemp(path.join(os.tmpdir(), 'appsmithctl-backup-'));
return backupRootPath
}
function getBackupContentsPath(backupRootPath, timestamp) {
return backupRootPath + '/appsmith-backup-' + timestamp;
}
function removeEncryptionEnvData(content) {
const output_lines = []
content.split(/\r?\n/).forEach(line => {
if (!line.startsWith("APPSMITH_ENCRYPTION")) {
output_lines.push(line)
}
});
return output_lines.join('\n')
}
function getBackupArchiveLimit(backupArchivesLimit) {
if (!backupArchivesLimit)
backupArchivesLimit = 4;
return backupArchivesLimit
}
async function removeOldBackups(backupFiles, backupArchivesLimit) {
while (backupFiles.length > backupArchivesLimit) {
const fileName = backupFiles.shift();
await fsPromises.rm(Constants.BACKUP_PATH + '/' + fileName);
}
return backupFiles
}
function getTimeStampInISO() {
return new Date().toISOString().replace(/:/g, '-')
}
function getAvailableBackupSpaceInBytes() {
return parseInt(shell.exec('df --output=avail -B 1 /appsmith-stacks | tail -n 1'), 10)
}
function checkAvailableBackupSpace(availSpaceInBytes) {
if (availSpaceInBytes < Constants.MIN_REQUIRED_DISK_SPACE_IN_BYTES) {
throw new Error('Not enough space avaliable at /appsmith-stacks. Please ensure availability of atleast 5GB to backup successfully.');
}
}
async function getCurrentVersion() {
const content = await fsPromises.readFile('/opt/appsmith/rts/version.js', { encoding: 'utf8' });
return content.match(/\bexports\.VERSION\s*=\s*["']([^"]+)["']/)[1];
}
module.exports = {
run,
getTimeStampInISO,
getAvailableBackupSpaceInBytes,
checkAvailableBackupSpace,
generateBackupRootPath,
getBackupContentsPath,
executeMongoDumpCMD,
getGitRoot,
executeCopyCMD,
getCurrentVersion,
removeEncryptionEnvData,
getBackupArchiveLimit,
removeOldBackups
};

181
deploy/docker/utils/bin/backup.test.js Normal file
View File

@ -0,0 +1,181 @@
const backup = require('./backup');
const Constants = require('./constants');
const os = require('os');
const fsPromises = require('fs/promises');
const utils = require('./utils');
const shell = require('shelljs');
describe('Backup Tests', () => {
test('Timestamp string in ISO format', () => {
console.log(backup.getTimeStampInISO())
expect(backup.getTimeStampInISO()).toMatch(/(\d{4})-(\d{2})-(\d{2})T(\d{2})\-(\d{2})\-(\d{2})\.(\d{3})Z/)
});
test('Available Space in /appsmith-stacks volume in Bytes', () => {
shell.exec = jest.fn((format) => '20');
const res = expect(backup.getAvailableBackupSpaceInBytes())
res.toBe(20)
});
it('Checkx the constant is 2 GB', () => {
let size = 2 * 1024 * 1024 * 1024
expect(Constants.MIN_REQUIRED_DISK_SPACE_IN_BYTES).toBe(size)
});
it('Should throw Error when the available size is below MIN_REQUIRED_DISK_SPACE_IN_BYTES', () => {
let size = Constants.MIN_REQUIRED_DISK_SPACE_IN_BYTES - 1;
expect(() => {backup.checkAvailableBackupSpace(size)}).toThrow('Not enough space avaliable at /appsmith-stacks. Please ensure availability of atleast 5GB to backup successfully.');
});
it('Should not hould throw Error when the available size is >= MIN_REQUIRED_DISK_SPACE_IN_BYTES', () => {
expect(() => {backup.checkAvailableBackupSpace(Constants.MIN_REQUIRED_DISK_SPACE_IN_BYTES)}).not.toThrow('Not enough space avaliable at /appsmith-stacks. Please ensure availability of atleast 5GB to backup successfully.');
});
it('Generates t', async () => {
os.tmpdir = jest.fn().mockReturnValue('temp/dir');
fsPromises.mkdtemp = jest.fn().mockImplementation((a) => a);
backup.generateBackupRootPath().then((response)=>{console.log(response)})
const res = await backup.generateBackupRootPath()
expect(res).toBe('temp/dir/appsmithctl-backup-')
});
test('Test backup contents path generation', () => {
var root = '/rootDir'
var timestamp = '0000-00-0T00-00-00.00Z'
expect(backup.getBackupContentsPath(root, timestamp)).toBe('/rootDir/appsmith-backup-0000-00-0T00-00-00.00Z')
});
test('Test mongodump CMD generaton', async () => {
var dest = '/dest'
var appsmithMongoURI = 'mongodb://username:password@host/appsmith'
var cmd = 'mongodump --uri=mongodb://username:password@host/appsmith --archive=/dest/mongodb-data.gz --gzip'
utils.execCommand = jest.fn().mockImplementation(async (a) => a.join(' '));
const res = await backup.executeMongoDumpCMD(dest, appsmithMongoURI)
expect(res).toBe(cmd)
console.log(res)
})
test('Test get gitRoot path when APPSMITH_GIT_ROOT is \'\' ', () => {
expect(backup.getGitRoot('')).toBe('/appsmith-stacks/git-storage')
});
test('Test get gitRoot path when APPSMITH_GIT_ROOT is null ', () => {
expect(backup.getGitRoot()).toBe('/appsmith-stacks/git-storage')
});
test('Test get gitRoot path when APPSMITH_GIT_ROOT is defined ', () => {
expect(backup.getGitRoot('/my/git/storage')).toBe('/my/git/storage')
});
test('Test ln command generation', async () => {
var gitRoot = '/appsmith-stacks/git-storage'
var dest = '/destdir'
var cmd = 'ln -s /appsmith-stacks/git-storage /destdir/git-storage'
utils.execCommand = jest.fn().mockImplementation(async (a) => a.join(' '));
const res = await backup.executeCopyCMD(gitRoot, dest)
expect(res).toBe(cmd)
console.log(res)
})
it('Checks for the current Appsmith Version.', async () => {
fsPromises.readFile = jest.fn().mockImplementation(async (a) =>
`Object.defineProperty(exports, "__esModule", { value: true });
exports.VERSION = void 0;
exports.VERSION = "v0.0.0-SNAPSHOT";`);
const res = await backup.getCurrentVersion()
expect(res).toBe("v0.0.0-SNAPSHOT")
console.log(res)
})
test('If encriytpion env values are being removed', () => {
expect(backup.removeEncryptionEnvData(`APPSMITH_REDIS_URL=redis://127.0.0.1:6379\nAPPSMITH_ENCRYPTION_PASSWORD=dummy-pass\nAPPSMITH_ENCRYPTION_SALT=dummy-salt\nAPPSMITH_INSTANCE_NAME=Appsmith\n
`)).toMatch(`APPSMITH_REDIS_URL=redis://127.0.0.1:6379\nAPPSMITH_INSTANCE_NAME=Appsmith\n`)
});
test('Backup Archive Limit when env APPSMITH_BACKUP_ARCHIVE_LIMIT is null', () => {
expect(backup.getBackupArchiveLimit()).toBe(4)
});
test('Backup Archive Limit when env APPSMITH_BACKUP_ARCHIVE_LIMIT is 5', () => {
expect(backup.getBackupArchiveLimit(5)).toBe(5)
});
test('Cleanup Backups when limit is 4 and there are 5 files', async () => {
const backupArchivesLimit = 4;
fsPromises.rm = jest.fn().mockImplementation(async (a) => console.log(a));
var backupFiles = ['file1','file2','file3','file4','file5']
var expectedBackupFiles = ['file2','file3','file4','file5']
const res = await backup.removeOldBackups(backupFiles,backupArchivesLimit)
console.log(res)
expect(res).toEqual(expectedBackupFiles)
})
test('Cleanup Backups when limit is 2 and there are 5 files', async () => {
const backupArchivesLimit = 2;
fsPromises.rm = jest.fn().mockImplementation(async (a) => console.log(a));
var backupFiles = ['file1','file2','file3','file4','file5']
var expectedBackupFiles = ['file4','file5']
const res = await backup.removeOldBackups(backupFiles,backupArchivesLimit)
console.log(res)
expect(res).toEqual(expectedBackupFiles)
})
test('Cleanup Backups when limit is 4 and there are 4 files', async () => {
const backupArchivesLimit = 4;
fsPromises.rm = jest.fn().mockImplementation(async (a) => console.log(a));
var backupFiles = ['file1','file2','file3','file4']
var expectedBackupFiles = ['file1','file2','file3','file4']
const res = await backup.removeOldBackups(backupFiles,backupArchivesLimit)
console.log(res)
expect(res).toEqual(expectedBackupFiles)
})
test('Cleanup Backups when limit is 4 and there are 2 files', async () => {
const backupArchivesLimit = 4;
fsPromises.rm = jest.fn().mockImplementation(async (a) => console.log(a));
var backupFiles = ['file1','file2']
var expectedBackupFiles = ['file1','file2']
const res = await backup.removeOldBackups(backupFiles,backupArchivesLimit)
console.log(res)
expect(res).toEqual(expectedBackupFiles)
})
test('Cleanup Backups when limit is 4 and there are 2 files', async () => {
const backupArchivesLimit = 4;
fsPromises.rm = jest.fn().mockImplementation(async (a) => console.log(a));
var backupFiles = ['file1','file2']
var expectedBackupFiles = ['file1','file2']
const res = await backup.removeOldBackups(backupFiles,backupArchivesLimit)
console.log(res)
expect(res).toEqual(expectedBackupFiles)
})
test('Cleanup Backups when limit is 2 and there is 1 file', async () => {
const backupArchivesLimit = 4;
fsPromises.rm = jest.fn().mockImplementation(async (a) => console.log(a));
var backupFiles = ['file1']
var expectedBackupFiles = ['file1']
const res = await backup.removeOldBackups(backupFiles,backupArchivesLimit)
console.log(res)
expect(res).toEqual(expectedBackupFiles)
})
test('Cleanup Backups when limit is 2 and there is no file', async () => {
const backupArchivesLimit = 4;
fsPromises.rm = jest.fn().mockImplementation(async (a) => console.log(a));
var backupFiles = []
var expectedBackupFiles = []
const res = await backup.removeOldBackups(backupFiles,backupArchivesLimit)
console.log(res)
expect(res).toEqual(expectedBackupFiles)
})
});

4
deploy/docker/utils/bin/constants.js
View File

@ -5,7 +5,7 @@ const RESTORE_PATH = "/appsmith-stacks/data/restore"
const DUMP_FILE_NAME = "appsmith-data.archive"
const BACKUP_ERROR_LOG_PATH = "/appsmith-stacks/logs/backup"
const APPSMITHCTL_LOG_PATH = "/appsmith-stacks/logs/appsmithctl"
const LAST_ERROR_MAIL_TS = "/appsmith-stacks/data/backup/last-error-mail-ts"
@ -18,7 +18,7 @@ module.exports = {
RESTORE_PATH,
DUMP_FILE_NAME,
LAST_ERROR_MAIL_TS,
BACKUP_ERROR_LOG_PATH,
APPSMITHCTL_LOG_PATH,
MIN_REQUIRED_DISK_SPACE_IN_BYTES,
DURATION_BETWEEN_BACKUP_ERROR_MAILS_IN_MILLI_SEC,
}

17
deploy/docker/utils/bin/logger.js
View File

@ -4,13 +4,24 @@ const Constants = require('./constants');
async function backup_error(err) {
console.error(err);
try {
await fsPromises.access(Constants.BACKUP_ERROR_LOG_PATH);
await fsPromises.access(Constants.APPSMITHCTL_LOG_PATH);
} catch (error) {
await fsPromises.mkdir(Constants.BACKUP_ERROR_LOG_PATH);
await fsPromises.mkdir(Constants.APPSMITHCTL_LOG_PATH);
}
await fsPromises.appendFile(Constants.BACKUP_ERROR_LOG_PATH + '/error.log', new Date().toISOString() + ' ' + err + '\n');
await fsPromises.appendFile(Constants.APPSMITHCTL_LOG_PATH + '/backup.log', new Date().toISOString() + ' [ ERROR ] ' + err + '\n');
}
async function backup_info(msg) {
console.log(msg);
try {
await fsPromises.access(Constants.APPSMITHCTL_LOG_PATH);
} catch (error) {
await fsPromises.mkdir(Constants.APPSMITHCTL_LOG_PATH);
}
await fsPromises.appendFile(Constants.APPSMITHCTL_LOG_PATH + '/backup.log', new Date().toISOString() + ' [ INFO ] ' + msg + '\n');
}
module.exports = {
backup_error,
backup_info,
};

5582
deploy/docker/utils/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

16
deploy/docker/utils/package.json
View File

@ -12,16 +12,20 @@
"directory": "deploy/docker"
},
"dependencies": {
"dotenv": "10.0.0",
"mongodb": "^4.4.0",
"readline-sync": "1.4.10",
"shelljs": "0.8.5",
"nodemailer": "6.7.5",
"cli-progress": "^3.11.2",
"dotenv": "10.0.0",
"jest": "^29.1.2",
"luxon": "^3.0.1",
"minimist": "^1.2.6"
"minimist": "^1.2.6",
"mongodb": "^4.4.0",
"nodemailer": "6.7.5",
"readline-sync": "1.4.10",
"shelljs": "0.8.5"
},
"bin": {
"appsmithctl": "./bin/index.js"
},
"scripts": {
"test": "jest"
}
}