Protei/PromucFlow_constructor
2
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

ci: CI optimisation to reduce ci-test run time and modularise docker image building (#21219)

Browse Source 
## Description
- Modularised the docker image building
- Optimised the `install dependancies` step to reduce the time from 8
mins to 30-40 secs now
- Removed unnecessary steps from the ci-dubugging.yml 
- Added ci-debug.sh file to ease the steps to run ngrok while running
ci-debugging
- Changes made to below files
1.  integration-tests-command.yml
2. test-build-docker-image.yml
3. ci-test.yml
4. build-docker-image.yml

## Type of change
- integration-tests-command.yml
- test-build-docker-image.yml
- ci-test.yml
- build-docker-image.yml

## How Has This Been Tested?
- Manual

## Checklist:
### QA activity:
- [ ] Test plan has been approved by relevant developers
- [ ] Test plan has been peer reviewed by QA
- [ ] Cypress test cases have been added and approved by either SDET or
manual QA
- [ ] Organized project review call with relevant stakeholders after
Round 1/2 of QA
- [ ] Added Test Plan Approved label after reveiwing all Cypress test
This commit is contained in:
Saroj 2023-03-10 12:51:32 +05:30 committed by GitHub
parent e24e3b1d29
commit 4ad92d648a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 115 additions and 120 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
.github/workflows/build-docker-image.yml vendored
View File

@ -83,11 +83,17 @@ jobs:
# Saving the docker image to tar file
- name: Save Docker image to tar file
run: docker save cicontainer -o cicontainer.tar
run: |
docker save cicontainer -o cicontainer.tar
gzip cicontainer.tar
# Uploading the artifact to use it in other subsequent runners
- name: Upload Docker image to artifacts
uses: actions/upload-artifact@v3
with:
name: cicontainer
path: cicontainer.tar
path: cicontainer.tar.gz
- name: Save the status of the run
run: echo "run_result=success" >> $GITHUB_OUTPUT > ~/run_result

95
.github/workflows/ci-debugging.yml vendored
View File

@ -3,44 +3,16 @@ name: CI Debugging
on:
# This line enables manual triggering of this workflow.
workflow_dispatch:
inputs:
run_id:
description: 'Run id to download artifacts:'
required: true
type: number
jobs:
server-build:
name: server-build
uses: ./.github/workflows/server-build.yml
secrets: inherit
with:
pr: ${{ github.event.client_payload.pull_request.number }}
client-build:
name: client-build
uses: ./.github/workflows/client-build.yml
secrets: inherit
with:
pr: ${{ github.event.client_payload.pull_request.number }}
rts-build:
name: rts-build
uses: ./.github/workflows/rts-build.yml
secrets: inherit
with:
pr: ${{ github.event.client_payload.pull_request.number }}
build-docker-image:
needs: [ client-build, server-build, rts-build ]
# Only run if the build step is successful
if: success()
name: build-docker-image
uses: ./.github/workflows/build-docker-image.yml
secrets: inherit
with:
pr: ${{ github.event.client_payload.pull_request.number }}
ci-test:
needs: [ build-docker-image ]
# Only run if the build step is successful
if: success()
ci-debugger:
runs-on: ubuntu-latest
if: github.event_name == 'workflow_dispatch'
defaults:
run:
shell: bash
@ -67,31 +39,21 @@ jobs:
- name: Set up Depot CLI
uses: depot/setup-action@v1
# Check out merge commit
- name: Fork based /ok-to-test checkout
- name: Checkout the head commit of the branch
uses: actions/checkout@v3
with:
ref: "refs/pull/${{ github.event.client_payload.pull_request.number }}/merge"
# Timestamp will be used to create cache key
- id: timestamp
run: echo "timestamp=$(date +'%Y-%m-%dT%H:%M:%S')" >> $GITHUB_OUTPUT
# Setup Java
- name: Set up JDK 17
if: steps.run_result.outputs.run_result != 'success'
uses: actions/setup-java@v3
with:
distribution: 'temurin'
java-version: '17'
fetch-depth: 0
- name: Download Docker image artifact
uses: actions/download-artifact@v3
uses: dawidd6/action-download-artifact@v2
with:
name: cicontainer
run_id: ${{ github.event.inputs.run_id }}
- name: Load Docker image from tar file
run: docker load -i cicontainer.tar
run: |
gunzip cicontainer.tar.gz
docker load -i cicontainer.tar
- name: Create folder
if: steps.run_result.outputs.run_result != 'success'
@ -101,17 +63,7 @@ jobs:
run: |
mkdir -p cicontainerlocal/stacks/configuration/
#Download the oldstacks from appsmithorg/oldstacks repo
- name: Download the oldstack
uses: actions/checkout@v3
with:
ref: main
repository: appsmithorg/ci-oldstack
token: ${{ secrets.APPSMITH_CI_TEST_PAT }}
submodules: 'recursive'
path: cicontainerlocal/oldstack
- name: Load docker image
- name: Run docker image
if: steps.run_result.outputs.run_result != 'success'
env:
APPSMITH_LICENSE_KEY: ${{ secrets.APPSMITH_LICENSE_KEY }}
@ -130,21 +82,14 @@ jobs:
-e APPSMITH_CLOUD_SERVICES_BASE_URL=http://host.docker.internal:5001 \
--add-host=host.docker.internal:host-gateway \
cicontainer
- name: Use Node.js 16.14.0
if: steps.run_result.outputs.run_result != 'success'
uses: actions/setup-node@v3
with:
node-version: "16.14.0"
# Install all the dependencies
- name: Install dependencies
- name: Execute access to ci-debug.sh
if: steps.run_result.outputs.run_result != 'success'
working-directory: "."
run: |
cd app/client
yarn install
chmod +x ci-debug.sh
# Start tmate session for tunnelling
- name: Setup tmate session
if: always()
if: steps.run_result.outputs.run_result != 'success'
uses: mxschmitt/action-tmate@v3

49
.github/workflows/ci-test.yml vendored
View File

@ -116,48 +116,15 @@ jobs:
- name: cat run_result
run: echo ${{ steps.run_result.outputs.run_result }}
# Setup Java
- name: Set up JDK 17
if: steps.run_result.outputs.run_result != 'success'
uses: actions/setup-java@v3
with:
distribution: 'temurin'
java-version: '17'
- name: Download the client build artifact
if: steps.run_result.outputs.run_result != 'success'
- name: Download Docker image artifact
uses: actions/download-artifact@v3
with:
name: client-build
path: app/client/build
name: cicontainer
- name: Download the server build artifact
if: steps.run_result.outputs.run_result != 'success'
uses: actions/download-artifact@v3
with:
name: server-build
path: app/server/dist/
- name: Download the rts build artifact
if: steps.run_result.outputs.run_result != 'success'
uses: actions/download-artifact@v3
with:
name: rts-dist
path: app/rts/dist
- name: Un-tar the rts folder
- name: Load Docker image from tar file
run: |
tar -xvf app/rts/dist/rts-dist.tar -C app/rts/
echo "Cleaning up the tar files"
rm app/rts/dist/rts-dist.tar
# We don't use Depot Docker builds because it's faster for local Docker images to be built locally.
# It's slower and more expensive to build these Docker images on Depot and download it back to the CI node.
- name: Build docker image
if: steps.run_result.outputs.run_result != 'success'
working-directory: "."
run: |
docker build -t cicontainer .
gunzip cicontainer.tar.gz
docker load -i cicontainer.tar
- name: Create folder
if: steps.run_result.outputs.run_result != 'success'
@ -177,7 +144,7 @@ jobs:
submodules: 'recursive'
path: cicontainerlocal/oldstack
- name: Load docker image
- name: Run docker image
if: steps.run_result.outputs.run_result != 'success'
env:
APPSMITH_LICENSE_KEY: ${{ secrets.APPSMITH_LICENSE_KEY }}
@ -208,7 +175,9 @@ jobs:
if: steps.run_result.outputs.run_result != 'success'
run: |
cd app/client
yarn install
yarn install \
--cwd cypress \
--modules-folder ../node_modules
- name: Setting up the cypress tests
if: steps.run_result.outputs.run_result != 'success'

12
.github/workflows/integration-tests-command.yml vendored
View File

@ -56,10 +56,20 @@ jobs:
with:
pr: ${{ github.event.client_payload.pull_request.number }}
ci-test:
build-docker-image:
needs: [ client-build, server-build, rts-build ]
# Only run if the build step is successful
if: success()
name: build-docker-image
uses: ./.github/workflows/build-docker-image.yml
secrets: inherit
with:
pr: ${{ github.event.client_payload.pull_request.number }}
ci-test:
needs: [ build-docker-image ]
# Only run if the build step is successful
if: success()
name: ci-test
uses: ./.github/workflows/ci-test.yml
secrets: inherit

12
.github/workflows/test-build-docker-image.yml vendored
View File

@ -46,10 +46,20 @@ jobs:
with:
pr: 0
ci-test:
build-docker-image:
needs: [ client-build, server-build, rts-build ]
# Only run if the build step is successful
if: success()
name: build-docker-image
uses: ./.github/workflows/build-docker-image.yml
secrets: inherit
with:
pr: 0
ci-test:
needs: [ build-docker-image ]
# Only run if the build step is successful
if: success()
name: ci-test
uses: ./.github/workflows/ci-test.yml
secrets: inherit

2
app/client/cypress/apply-patches.js
View File

@ -5,7 +5,7 @@ const fs = require("fs/promises");
const path = require("path");
async function applyPatches() {
const patchesDir = path.join("cypress", "patches");
const patchesDir = path.join("patches");
const patchesAbsDir = path.join(process.cwd(), patchesDir);
const patches = await fs.readdir(patchesAbsDir);
const installDir = getVersionDir();

37
app/client/cypress/package.json Normal file
View File

@ -0,0 +1,37 @@
{
"name": "appsmith-ci-test",
"version": "0.1.0",
"private": true,
"engines": {
"node": "^16.14.0",
"npm": "^8.5.5"
},
"scripts": {
"install": "node apply-patches.js"
},
"browserslist": [
">0.2%",
"not dead",
"not ie <= 11",
"not op_mini all"
],
"devDependencies": {
"@typescript-eslint/eslint-plugin": "^5.25.0",
"@typescript-eslint/parser": "^5.25.0",
"@faker-js/faker": "^7.4.0",
"cypress-log-to-output": "^1.1.2",
"cy-verify-downloads": "^0.0.5",
"cypress": "9.7.0",
"cypress-file-upload": "^4.1.1",
"cypress-image-snapshot": "^4.0.1",
"cypress-multi-reporters": "^1.2.4",
"cypress-real-events": "^1.7.1",
"cypress-wait-until": "^1.7.2",
"cypress-xpath": "^1.4.0",
"eslint-plugin-cypress": "^2.11.2",
"dotenv": "^8.1.0",
"typescript": "4.5.5",
"diff": "^5.0.0",
"chalk": "^4.1.1"
}
}

18
ci-debug.sh Normal file
View File

@ -0,0 +1,18 @@
#!/bin/bash
cd ~
if [ ! -f "ngrok-v3-stable-linux-amd64.tgz" ]; then
wget https://bin.equinox.io/c/bNyj1mQVY4c/ngrok-v3-stable-linux-amd64.tgz
gunzip ngrok-v3-stable-linux-amd64.tgz
tar -xvf ngrok-v3-stable-linux-amd64.tar
else
echo "Starting ngrok"
fi
if [ -z "$1" ]; then
read -p "Please enter ngrok token: " value
else
value="$1"
fi
./ngrok config add-authtoken $value
./ngrok http 80