ci: CI optimisation to reduce ci-test run time and modularise docker image building (#21219)

## Description
- Modularised the docker image building
- Optimised the `install dependancies` step to reduce the time from 8
mins to 30-40 secs now
- Removed unnecessary steps from the ci-dubugging.yml 
- Added ci-debug.sh file to ease the steps to run ngrok while running
ci-debugging
- Changes made to below files
1.  integration-tests-command.yml
2. test-build-docker-image.yml
3. ci-test.yml
4. build-docker-image.yml

## Type of change
- integration-tests-command.yml
- test-build-docker-image.yml
- ci-test.yml
- build-docker-image.yml

## How Has This Been Tested?
- Manual

## Checklist:
### QA activity:
- [ ] Test plan has been approved by relevant developers
- [ ] Test plan has been peer reviewed by QA
- [ ] Cypress test cases have been added and approved by either SDET or
manual QA
- [ ] Organized project review call with relevant stakeholders after
Round 1/2 of QA
- [ ] Added Test Plan Approved label after reveiwing all Cypress test

.github/workflows/build-docker-image.yml

@@ -83,11 +83,17 @@ jobs:
       
       # Saving the docker image to tar file
       - name: Save Docker image to tar file
-        run: docker save cicontainer -o cicontainer.tar
+        run: |
+          docker save cicontainer -o cicontainer.tar
+          gzip cicontainer.tar
       
       # Uploading the artifact to use it in other subsequent runners
       - name: Upload Docker image to artifacts
         uses: actions/upload-artifact@v3
         with:
           name: cicontainer
-          path: cicontainer.tar
+          path: cicontainer.tar.gz
+
+      - name: Save the status of the run
+        run: echo "run_result=success" >> $GITHUB_OUTPUT > ~/run_result
+        

.github/workflows/ci-debugging.yml

@@ -3,44 +3,16 @@ name: CI Debugging
 on:
   # This line enables manual triggering of this workflow.
   workflow_dispatch:
+    inputs:
+      run_id:
+        description: 'Run id to download artifacts:'
+        required: true
+        type: number      
 
 jobs:
-  server-build:
+  ci-debugger:
-    name: server-build
-    uses: ./.github/workflows/server-build.yml
-    secrets: inherit
-    with:
-      pr: ${{ github.event.client_payload.pull_request.number }}
-
-  client-build:
-    name: client-build
-    uses: ./.github/workflows/client-build.yml
-    secrets: inherit
-    with:
-      pr: ${{ github.event.client_payload.pull_request.number }}
-
-  rts-build:
-    name: rts-build
-    uses: ./.github/workflows/rts-build.yml
-    secrets: inherit
-    with:
-      pr: ${{ github.event.client_payload.pull_request.number }}
-
-  build-docker-image:
-    needs: [ client-build, server-build, rts-build ]
-    # Only run if the build step is successful
-    if: success()
-    name: build-docker-image
-    uses: ./.github/workflows/build-docker-image.yml
-    secrets: inherit
-    with:
-      pr: ${{ github.event.client_payload.pull_request.number }}
-
-  ci-test:
-    needs: [ build-docker-image ]
-    # Only run if the build step is successful
-    if: success()
     runs-on: ubuntu-latest
+    if: github.event_name == 'workflow_dispatch'
     defaults:
       run:
         shell: bash
@@ -67,31 +39,21 @@ jobs:
       - name: Set up Depot CLI
         uses: depot/setup-action@v1
 
-      # Check out merge commit
+      - name: Checkout the head commit of the branch
-      - name: Fork based /ok-to-test checkout
         uses: actions/checkout@v3
         with:
-          ref: "refs/pull/${{ github.event.client_payload.pull_request.number }}/merge"
+          fetch-depth: 0
-
-      # Timestamp will be used to create cache key
-      - id: timestamp
-        run: echo "timestamp=$(date +'%Y-%m-%dT%H:%M:%S')" >> $GITHUB_OUTPUT
-
-      # Setup Java
-      - name: Set up JDK 17
-        if: steps.run_result.outputs.run_result != 'success'
-        uses: actions/setup-java@v3
-        with:
-          distribution: 'temurin'
-          java-version: '17'
 
       - name: Download Docker image artifact
-        uses: actions/download-artifact@v3
+        uses: dawidd6/action-download-artifact@v2
         with:
           name: cicontainer
+          run_id: ${{ github.event.inputs.run_id }}
 
       - name: Load Docker image from tar file
-        run: docker load -i cicontainer.tar
+        run: |
+          gunzip cicontainer.tar.gz
+          docker load -i cicontainer.tar
 
       - name: Create folder
         if: steps.run_result.outputs.run_result != 'success'
@@ -101,17 +63,7 @@ jobs:
         run: |
           mkdir -p cicontainerlocal/stacks/configuration/
 
-      #Download the oldstacks from appsmithorg/oldstacks repo
+      - name: Run docker image
-      - name: Download the oldstack
-        uses: actions/checkout@v3
-        with:
-          ref: main
-          repository: appsmithorg/ci-oldstack
-          token: ${{ secrets.APPSMITH_CI_TEST_PAT }}
-          submodules: 'recursive'
-          path: cicontainerlocal/oldstack
-
-      - name: Load docker image
         if: steps.run_result.outputs.run_result != 'success'
         env:
           APPSMITH_LICENSE_KEY: ${{ secrets.APPSMITH_LICENSE_KEY }}
@@ -131,20 +83,13 @@ jobs:
             --add-host=host.docker.internal:host-gateway \
             cicontainer
             
-      - name: Use Node.js 16.14.0
+      - name: Execute access to ci-debug.sh
-        if: steps.run_result.outputs.run_result != 'success'
-        uses: actions/setup-node@v3
-        with:
-          node-version: "16.14.0"
-
-      # Install all the dependencies
-      - name: Install dependencies
         if: steps.run_result.outputs.run_result != 'success'
+        working-directory: "."
         run: |
-          cd app/client
+          chmod +x ci-debug.sh
-          yarn install
 
       # Start tmate session for tunnelling
       - name: Setup tmate session
-        if: always()
+        if: steps.run_result.outputs.run_result != 'success'
         uses: mxschmitt/action-tmate@v3

.github/workflows/ci-test.yml

@@ -116,48 +116,15 @@ jobs:
       - name: cat run_result
         run: echo ${{ steps.run_result.outputs.run_result }}
 
-      # Setup Java
+      - name: Download Docker image artifact
-      - name: Set up JDK 17
-        if: steps.run_result.outputs.run_result != 'success'
-        uses: actions/setup-java@v3
-        with:
-          distribution: 'temurin'
-          java-version: '17'
-
-      - name: Download the client build artifact
-        if: steps.run_result.outputs.run_result != 'success'
         uses: actions/download-artifact@v3
         with:
-          name: client-build
+          name: cicontainer
-          path: app/client/build
 
-      - name: Download the server build artifact
+      - name: Load Docker image from tar file
-        if: steps.run_result.outputs.run_result != 'success'
-        uses: actions/download-artifact@v3
-        with:
-          name: server-build
-          path: app/server/dist/
-
-      - name: Download the rts build artifact
-        if: steps.run_result.outputs.run_result != 'success'
-        uses: actions/download-artifact@v3
-        with:
-          name: rts-dist
-          path: app/rts/dist
-
-      - name: Un-tar the rts folder
         run: |
-          tar -xvf app/rts/dist/rts-dist.tar -C app/rts/
+          gunzip cicontainer.tar.gz
-          echo "Cleaning up the tar files"
+          docker load -i cicontainer.tar
-          rm app/rts/dist/rts-dist.tar
-
-      # We don't use Depot Docker builds because it's faster for local Docker images to be built locally.
-      # It's slower and more expensive to build these Docker images on Depot and download it back to the CI node.
-      - name: Build docker image
-        if: steps.run_result.outputs.run_result != 'success'
-        working-directory: "."
-        run: |
-          docker build -t cicontainer .
 
       - name: Create folder
         if: steps.run_result.outputs.run_result != 'success'
@@ -177,7 +144,7 @@ jobs:
           submodules: 'recursive'
           path: cicontainerlocal/oldstack
 
-      - name: Load docker image
+      - name: Run docker image
         if: steps.run_result.outputs.run_result != 'success'
         env:
           APPSMITH_LICENSE_KEY: ${{ secrets.APPSMITH_LICENSE_KEY }}
@@ -208,7 +175,9 @@ jobs:
         if: steps.run_result.outputs.run_result != 'success'
         run: |
           cd app/client
-          yarn install
+          yarn install \
+          --cwd cypress \
+          --modules-folder ../node_modules
 
       - name: Setting up the cypress tests
         if: steps.run_result.outputs.run_result != 'success'

.github/workflows/integration-tests-command.yml

@@ -56,10 +56,20 @@ jobs:
     with:
       pr: ${{ github.event.client_payload.pull_request.number }}
 
-  ci-test:
+  build-docker-image:
     needs: [ client-build, server-build, rts-build ]
     # Only run if the build step is successful
     if: success()
+    name: build-docker-image
+    uses: ./.github/workflows/build-docker-image.yml
+    secrets: inherit
+    with:
+      pr: ${{ github.event.client_payload.pull_request.number }}
+
+  ci-test:
+    needs: [ build-docker-image ]
+    # Only run if the build step is successful
+    if: success()
     name: ci-test
     uses: ./.github/workflows/ci-test.yml
     secrets: inherit

.github/workflows/test-build-docker-image.yml

@@ -46,10 +46,20 @@ jobs:
     with:
       pr: 0
 
-  ci-test:
+  build-docker-image:
     needs: [ client-build, server-build, rts-build ]
     # Only run if the build step is successful
     if: success()
+    name: build-docker-image
+    uses: ./.github/workflows/build-docker-image.yml
+    secrets: inherit
+    with:
+      pr: 0
+
+  ci-test:
+    needs: [ build-docker-image ]
+    # Only run if the build step is successful
+    if: success()
     name: ci-test
     uses: ./.github/workflows/ci-test.yml
     secrets: inherit

app/client/cypress/apply-patches.js

@@ -5,7 +5,7 @@ const fs = require("fs/promises");
 const path = require("path");
 
 async function applyPatches() {
-  const patchesDir = path.join("cypress", "patches");
+  const patchesDir = path.join("patches");
   const patchesAbsDir = path.join(process.cwd(), patchesDir);
   const patches = await fs.readdir(patchesAbsDir);
   const installDir = getVersionDir();

app/client/cypress/package.json

@@ -0,0 +1,37 @@
+{
+    "name": "appsmith-ci-test",
+    "version": "0.1.0",
+    "private": true,
+    "engines": {
+      "node": "^16.14.0",
+      "npm": "^8.5.5"
+    },
+    "scripts": {
+      "install": "node apply-patches.js"
+    },
+    "browserslist": [
+      ">0.2%",
+      "not dead",
+      "not ie <= 11",
+      "not op_mini all"
+    ],
+    "devDependencies": {
+      "@typescript-eslint/eslint-plugin": "^5.25.0",
+      "@typescript-eslint/parser": "^5.25.0",
+      "@faker-js/faker": "^7.4.0",
+      "cypress-log-to-output": "^1.1.2",
+      "cy-verify-downloads": "^0.0.5",
+      "cypress": "9.7.0",
+      "cypress-file-upload": "^4.1.1",
+      "cypress-image-snapshot": "^4.0.1",
+      "cypress-multi-reporters": "^1.2.4",
+      "cypress-real-events": "^1.7.1",
+      "cypress-wait-until": "^1.7.2",
+      "cypress-xpath": "^1.4.0",
+      "eslint-plugin-cypress": "^2.11.2",
+      "dotenv": "^8.1.0",
+      "typescript": "4.5.5",
+      "diff": "^5.0.0",
+      "chalk": "^4.1.1"
+    }
+}

ci-debug.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+cd ~
+if [ ! -f "ngrok-v3-stable-linux-amd64.tgz" ]; then
+        wget https://bin.equinox.io/c/bNyj1mQVY4c/ngrok-v3-stable-linux-amd64.tgz
+        gunzip ngrok-v3-stable-linux-amd64.tgz
+        tar -xvf ngrok-v3-stable-linux-amd64.tar
+else
+        echo "Starting ngrok"
+fi
+
+if [ -z "$1" ]; then
+        read -p "Please enter ngrok token: " value
+else
+        value="$1"
+fi
+./ngrok config add-authtoken $value
+./ngrok http 80