From 4ad92d648a4eca3f4ec465449c176ef1fce73a0b Mon Sep 17 00:00:00 2001 From: Saroj <43822041+sarojsarab@users.noreply.github.com> Date: Fri, 10 Mar 2023 12:51:32 +0530 Subject: [PATCH] ci: CI optimisation to reduce ci-test run time and modularise docker image building (#21219) ## Description - Modularised the docker image building - Optimised the `install dependancies` step to reduce the time from 8 mins to 30-40 secs now - Removed unnecessary steps from the ci-dubugging.yml - Added ci-debug.sh file to ease the steps to run ngrok while running ci-debugging - Changes made to below files 1. integration-tests-command.yml 2. test-build-docker-image.yml 3. ci-test.yml 4. build-docker-image.yml ## Type of change - integration-tests-command.yml - test-build-docker-image.yml - ci-test.yml - build-docker-image.yml ## How Has This Been Tested? - Manual ## Checklist: ### QA activity: - [ ] Test plan has been approved by relevant developers - [ ] Test plan has been peer reviewed by QA - [ ] Cypress test cases have been added and approved by either SDET or manual QA - [ ] Organized project review call with relevant stakeholders after Round 1/2 of QA - [ ] Added Test Plan Approved label after reveiwing all Cypress test --- .github/workflows/build-docker-image.yml | 10 +- .github/workflows/ci-debugging.yml | 95 ++++--------------- .github/workflows/ci-test.yml | 49 ++-------- .../workflows/integration-tests-command.yml | 12 ++- .github/workflows/test-build-docker-image.yml | 12 ++- app/client/cypress/apply-patches.js | 2 +- app/client/cypress/package.json | 37 ++++++++ ci-debug.sh | 18 ++++ 8 files changed, 115 insertions(+), 120 deletions(-) create mode 100644 app/client/cypress/package.json create mode 100644 ci-debug.sh diff --git a/.github/workflows/build-docker-image.yml b/.github/workflows/build-docker-image.yml index 99695a6839..db2fdae24c 100644 --- a/.github/workflows/build-docker-image.yml +++ b/.github/workflows/build-docker-image.yml @@ -83,11 +83,17 @@ jobs: # Saving the docker image to tar file - name: Save Docker image to tar file - run: docker save cicontainer -o cicontainer.tar + run: | + docker save cicontainer -o cicontainer.tar + gzip cicontainer.tar # Uploading the artifact to use it in other subsequent runners - name: Upload Docker image to artifacts uses: actions/upload-artifact@v3 with: name: cicontainer - path: cicontainer.tar + path: cicontainer.tar.gz + + - name: Save the status of the run + run: echo "run_result=success" >> $GITHUB_OUTPUT > ~/run_result + \ No newline at end of file diff --git a/.github/workflows/ci-debugging.yml b/.github/workflows/ci-debugging.yml index 54994d17e8..aaea85da11 100644 --- a/.github/workflows/ci-debugging.yml +++ b/.github/workflows/ci-debugging.yml @@ -3,44 +3,16 @@ name: CI Debugging on: # This line enables manual triggering of this workflow. workflow_dispatch: + inputs: + run_id: + description: 'Run id to download artifacts:' + required: true + type: number jobs: - server-build: - name: server-build - uses: ./.github/workflows/server-build.yml - secrets: inherit - with: - pr: ${{ github.event.client_payload.pull_request.number }} - - client-build: - name: client-build - uses: ./.github/workflows/client-build.yml - secrets: inherit - with: - pr: ${{ github.event.client_payload.pull_request.number }} - - rts-build: - name: rts-build - uses: ./.github/workflows/rts-build.yml - secrets: inherit - with: - pr: ${{ github.event.client_payload.pull_request.number }} - - build-docker-image: - needs: [ client-build, server-build, rts-build ] - # Only run if the build step is successful - if: success() - name: build-docker-image - uses: ./.github/workflows/build-docker-image.yml - secrets: inherit - with: - pr: ${{ github.event.client_payload.pull_request.number }} - - ci-test: - needs: [ build-docker-image ] - # Only run if the build step is successful - if: success() + ci-debugger: runs-on: ubuntu-latest + if: github.event_name == 'workflow_dispatch' defaults: run: shell: bash @@ -67,31 +39,21 @@ jobs: - name: Set up Depot CLI uses: depot/setup-action@v1 - # Check out merge commit - - name: Fork based /ok-to-test checkout + - name: Checkout the head commit of the branch uses: actions/checkout@v3 with: - ref: "refs/pull/${{ github.event.client_payload.pull_request.number }}/merge" - - # Timestamp will be used to create cache key - - id: timestamp - run: echo "timestamp=$(date +'%Y-%m-%dT%H:%M:%S')" >> $GITHUB_OUTPUT - - # Setup Java - - name: Set up JDK 17 - if: steps.run_result.outputs.run_result != 'success' - uses: actions/setup-java@v3 - with: - distribution: 'temurin' - java-version: '17' + fetch-depth: 0 - name: Download Docker image artifact - uses: actions/download-artifact@v3 + uses: dawidd6/action-download-artifact@v2 with: name: cicontainer + run_id: ${{ github.event.inputs.run_id }} - name: Load Docker image from tar file - run: docker load -i cicontainer.tar + run: | + gunzip cicontainer.tar.gz + docker load -i cicontainer.tar - name: Create folder if: steps.run_result.outputs.run_result != 'success' @@ -101,17 +63,7 @@ jobs: run: | mkdir -p cicontainerlocal/stacks/configuration/ - #Download the oldstacks from appsmithorg/oldstacks repo - - name: Download the oldstack - uses: actions/checkout@v3 - with: - ref: main - repository: appsmithorg/ci-oldstack - token: ${{ secrets.APPSMITH_CI_TEST_PAT }} - submodules: 'recursive' - path: cicontainerlocal/oldstack - - - name: Load docker image + - name: Run docker image if: steps.run_result.outputs.run_result != 'success' env: APPSMITH_LICENSE_KEY: ${{ secrets.APPSMITH_LICENSE_KEY }} @@ -130,21 +82,14 @@ jobs: -e APPSMITH_CLOUD_SERVICES_BASE_URL=http://host.docker.internal:5001 \ --add-host=host.docker.internal:host-gateway \ cicontainer - - - name: Use Node.js 16.14.0 - if: steps.run_result.outputs.run_result != 'success' - uses: actions/setup-node@v3 - with: - node-version: "16.14.0" - - # Install all the dependencies - - name: Install dependencies + + - name: Execute access to ci-debug.sh if: steps.run_result.outputs.run_result != 'success' + working-directory: "." run: | - cd app/client - yarn install + chmod +x ci-debug.sh # Start tmate session for tunnelling - name: Setup tmate session - if: always() + if: steps.run_result.outputs.run_result != 'success' uses: mxschmitt/action-tmate@v3 diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index caf0ba1cea..f99120a677 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -116,48 +116,15 @@ jobs: - name: cat run_result run: echo ${{ steps.run_result.outputs.run_result }} - # Setup Java - - name: Set up JDK 17 - if: steps.run_result.outputs.run_result != 'success' - uses: actions/setup-java@v3 - with: - distribution: 'temurin' - java-version: '17' - - - name: Download the client build artifact - if: steps.run_result.outputs.run_result != 'success' + - name: Download Docker image artifact uses: actions/download-artifact@v3 with: - name: client-build - path: app/client/build + name: cicontainer - - name: Download the server build artifact - if: steps.run_result.outputs.run_result != 'success' - uses: actions/download-artifact@v3 - with: - name: server-build - path: app/server/dist/ - - - name: Download the rts build artifact - if: steps.run_result.outputs.run_result != 'success' - uses: actions/download-artifact@v3 - with: - name: rts-dist - path: app/rts/dist - - - name: Un-tar the rts folder + - name: Load Docker image from tar file run: | - tar -xvf app/rts/dist/rts-dist.tar -C app/rts/ - echo "Cleaning up the tar files" - rm app/rts/dist/rts-dist.tar - - # We don't use Depot Docker builds because it's faster for local Docker images to be built locally. - # It's slower and more expensive to build these Docker images on Depot and download it back to the CI node. - - name: Build docker image - if: steps.run_result.outputs.run_result != 'success' - working-directory: "." - run: | - docker build -t cicontainer . + gunzip cicontainer.tar.gz + docker load -i cicontainer.tar - name: Create folder if: steps.run_result.outputs.run_result != 'success' @@ -177,7 +144,7 @@ jobs: submodules: 'recursive' path: cicontainerlocal/oldstack - - name: Load docker image + - name: Run docker image if: steps.run_result.outputs.run_result != 'success' env: APPSMITH_LICENSE_KEY: ${{ secrets.APPSMITH_LICENSE_KEY }} @@ -208,7 +175,9 @@ jobs: if: steps.run_result.outputs.run_result != 'success' run: | cd app/client - yarn install + yarn install \ + --cwd cypress \ + --modules-folder ../node_modules - name: Setting up the cypress tests if: steps.run_result.outputs.run_result != 'success' diff --git a/.github/workflows/integration-tests-command.yml b/.github/workflows/integration-tests-command.yml index 4810e1c4d8..76f19e573d 100644 --- a/.github/workflows/integration-tests-command.yml +++ b/.github/workflows/integration-tests-command.yml @@ -56,10 +56,20 @@ jobs: with: pr: ${{ github.event.client_payload.pull_request.number }} - ci-test: + build-docker-image: needs: [ client-build, server-build, rts-build ] # Only run if the build step is successful if: success() + name: build-docker-image + uses: ./.github/workflows/build-docker-image.yml + secrets: inherit + with: + pr: ${{ github.event.client_payload.pull_request.number }} + + ci-test: + needs: [ build-docker-image ] + # Only run if the build step is successful + if: success() name: ci-test uses: ./.github/workflows/ci-test.yml secrets: inherit diff --git a/.github/workflows/test-build-docker-image.yml b/.github/workflows/test-build-docker-image.yml index 80585da910..2b4536dd4c 100644 --- a/.github/workflows/test-build-docker-image.yml +++ b/.github/workflows/test-build-docker-image.yml @@ -46,10 +46,20 @@ jobs: with: pr: 0 - ci-test: + build-docker-image: needs: [ client-build, server-build, rts-build ] # Only run if the build step is successful if: success() + name: build-docker-image + uses: ./.github/workflows/build-docker-image.yml + secrets: inherit + with: + pr: 0 + + ci-test: + needs: [ build-docker-image ] + # Only run if the build step is successful + if: success() name: ci-test uses: ./.github/workflows/ci-test.yml secrets: inherit diff --git a/app/client/cypress/apply-patches.js b/app/client/cypress/apply-patches.js index 779a04367d..d4a94ebe60 100644 --- a/app/client/cypress/apply-patches.js +++ b/app/client/cypress/apply-patches.js @@ -5,7 +5,7 @@ const fs = require("fs/promises"); const path = require("path"); async function applyPatches() { - const patchesDir = path.join("cypress", "patches"); + const patchesDir = path.join("patches"); const patchesAbsDir = path.join(process.cwd(), patchesDir); const patches = await fs.readdir(patchesAbsDir); const installDir = getVersionDir(); diff --git a/app/client/cypress/package.json b/app/client/cypress/package.json new file mode 100644 index 0000000000..aaf8f30ec9 --- /dev/null +++ b/app/client/cypress/package.json @@ -0,0 +1,37 @@ +{ + "name": "appsmith-ci-test", + "version": "0.1.0", + "private": true, + "engines": { + "node": "^16.14.0", + "npm": "^8.5.5" + }, + "scripts": { + "install": "node apply-patches.js" + }, + "browserslist": [ + ">0.2%", + "not dead", + "not ie <= 11", + "not op_mini all" + ], + "devDependencies": { + "@typescript-eslint/eslint-plugin": "^5.25.0", + "@typescript-eslint/parser": "^5.25.0", + "@faker-js/faker": "^7.4.0", + "cypress-log-to-output": "^1.1.2", + "cy-verify-downloads": "^0.0.5", + "cypress": "9.7.0", + "cypress-file-upload": "^4.1.1", + "cypress-image-snapshot": "^4.0.1", + "cypress-multi-reporters": "^1.2.4", + "cypress-real-events": "^1.7.1", + "cypress-wait-until": "^1.7.2", + "cypress-xpath": "^1.4.0", + "eslint-plugin-cypress": "^2.11.2", + "dotenv": "^8.1.0", + "typescript": "4.5.5", + "diff": "^5.0.0", + "chalk": "^4.1.1" + } +} diff --git a/ci-debug.sh b/ci-debug.sh new file mode 100644 index 0000000000..956ad88d7c --- /dev/null +++ b/ci-debug.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +cd ~ +if [ ! -f "ngrok-v3-stable-linux-amd64.tgz" ]; then + wget https://bin.equinox.io/c/bNyj1mQVY4c/ngrok-v3-stable-linux-amd64.tgz + gunzip ngrok-v3-stable-linux-amd64.tgz + tar -xvf ngrok-v3-stable-linux-amd64.tar +else + echo "Starting ngrok" +fi + +if [ -z "$1" ]; then + read -p "Please enter ngrok token: " value +else + value="$1" +fi +./ngrok config add-authtoken $value +./ngrok http 80