ci: Refactor workflows to reuse code (#20161)

.github/workflows/github-release.yml

@@ -40,7 +40,7 @@ jobs:
             echo "is_beta=false" >> $GITHUB_OUTPUT
           fi
 
-  buildClient:
+  client-build:
     needs:
       - prelude
 
@@ -98,7 +98,7 @@ jobs:
           name: client-build
           path: app/client/build/
 
-  buildServer:
+  server-build:
     needs:
       - prelude
 
@@ -147,7 +147,7 @@ jobs:
           name: server-build
           path: app/server/dist/
 
-  buildRts:
+  rts-build:
     needs:
       - prelude
 
@@ -187,7 +187,7 @@ jobs:
           path: app/rts/rts-dist.tar
 
   package:
-    needs: [prelude, buildClient, buildServer, buildRts]
+    needs: [prelude, client-build, server-build, rts-build]
 
     runs-on: ubuntu-latest
     permissions:

.github/workflows/test-build-docker-image.yml

@@ -14,393 +14,30 @@ on:
       - "!app/client/cypress/manual_TestSuite/**"
 
 jobs:
-  buildClient:
-    # If the build has been triggered manually via workflow_dispatch or via a push to protected branches
-    # then we don't check for the PR approved state
-    if: |
-      github.event_name == 'workflow_dispatch' ||
-      github.event_name == 'push' ||
-      (github.event_name == 'pull_request_review' &&
-      github.event.review.state == 'approved' &&
-      github.event.pull_request.head.repo.full_name == github.repository)
-    runs-on: ubuntu-latest-4-cores
-    defaults:
-      run:
-        working-directory: app/client
-        shell: bash
 
-    steps:
+  server-build:
-      # Checkout the code
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-
-      # Checkout the code
-      - name: Checkout the merged commit from PR and base branch
-        if: github.event_name == 'pull_request_review'
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-          ref: refs/pull/${{ github.event.pull_request.number }}/merge
-
-      - name: Checkout the head commit of the branch
-        if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-
-      - name: Figure out the PR number
-        run: echo ${{ github.event.pull_request.number }}
-
-      # Timestamp will be used to create cache key
-      - id: timestamp
-        run: echo "timestamp=$(date +'%Y-%m-%dT%H:%M:%S')" >> $GITHUB_OUTPUT
-
-      # In case this is second attempt try restoring status of the prior attempt from cache
-      - name: Restore the previous run result
-        uses: actions/cache@v3
-        with:
-          path: |
-            ~/run_result
-          key: ${{ github.run_id }}-${{ github.job }}-${{ steps.timestamp.outputs.timestamp }}
-          restore-keys: |
-            ${{ github.run_id }}-${{ github.job }}-
-
-      # Fetch prior run result
-      - name: Get the previous run result
-        id: run_result
-        run: cat ~/run_result 2>/dev/null || echo 'default'
-
-      # In case of prior failure run the job
-      - if: steps.run_result.outputs.run_result != 'success'
-        run: echo "I'm alive!" && exit 0
-
-      # Set status = success
-      - name: Save the status of the run
-        run: echo "run_result=success" >> $GITHUB_OUTPUT > ~/run_result
-
-      - name: Use Node.js 16.14.0
-        if: steps.run_result.outputs.run_result != 'success'
-        uses: actions/setup-node@v3
-        with:
-          node-version: "16.14.0"
-
-      - name: Get yarn cache directory path
-        if: steps.run_result.outputs.run_result != 'success'
-        id: yarn-dep-cache-dir-path
-        run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT
-
-      # Retrieve npm dependencies from cache. After a successful run, these dependencies are cached again
-      - name: Cache npm dependencies
-        if: steps.run_result.outputs.run_result != 'success'
-        id: yarn-dep-cache
-        uses: actions/cache@v3
-        env:
-          cache-name: cache-yarn-dependencies
-        with:
-          path: |
-            ${{ steps.yarn-dep-cache-dir-path.outputs.dir }}
-          key: ${{ runner.os }}-yarn-dep-${{ hashFiles('**/yarn.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-yarn-dep-
-
-      # Install all the dependencies
-      - name: Install dependencies
-        if: steps.run_result.outputs.run_result != 'success'
-        run: yarn install --frozen-lockfile
-
-      - name: Set the build environment based on the branch
-        if: steps.run_result.outputs.run_result != 'success'
-        id: vars
-        run: |
-          echo "REACT_APP_ENVIRONMENT=DEVELOPMENT" >> $GITHUB_OUTPUT
-          if [[ "${{github.ref}}" == "refs/heads/master" ]]; then
-              echo "REACT_APP_ENVIRONMENT=PRODUCTION" >> $GITHUB_OUTPUT
-          fi
-          if [[ "${{github.ref}}" == "refs/heads/release" ]]; then
-              echo "REACT_APP_ENVIRONMENT=STAGING" >> $GITHUB_OUTPUT
-          fi
-          # Since this is an unreleased build, we set the version to incremented version number with
-          # a `-SNAPSHOT` suffix.
-          latest_released_version="$(git tag --list 'v*' --sort=-version:refname | head -1)"
-          echo "latest_released_version = $latest_released_version"
-          next_version="$(echo "$latest_released_version" | awk -F. -v OFS=. '{ $NF++; print }')"
-          echo "next_version = $next_version"
-          echo version=$next_version-SNAPSHOT >> $GITHUB_OUTPUT
-
-      # We burn React environment & the Segment analytics key into the build itself.
-      # This is to ensure that we don't need to configure it in each installation
-      - name: Create the bundle
-        if: steps.run_result.outputs.run_result != 'success'
-        run: |
-          if [[ $GITHUB_REF == "refs/heads/release" ]]; then
-            REACT_APP_SEGMENT_CE_KEY=${{ secrets.APPSMITH_SEGMENT_CE_KEY_RELEASE }}
-          else
-            REACT_APP_SEGMENT_CE_KEY=${{ secrets.APPSMITH_SEGMENT_CE_KEY }}
-          fi
-          REACT_APP_ENVIRONMENT=${{steps.vars.outputs.REACT_APP_ENVIRONMENT}} \
-            REACT_APP_FUSIONCHARTS_LICENSE_KEY=${{ secrets.APPSMITH_FUSIONCHARTS_LICENSE_KEY }} \
-            REACT_APP_SEGMENT_CE_KEY="$REACT_APP_SEGMENT_CE_KEY" \
-            SENTRY_AUTH_TOKEN=${{ secrets.SENTRY_AUTH_TOKEN }} \
-            REACT_APP_VERSION_ID=${{ steps.vars.outputs.version }} \
-            REACT_APP_VERSION_RELEASE_DATE=$(date -u '+%Y-%m-%dT%H:%M:%SZ') \
-            REACT_APP_VERSION_EDITION="Community" \
-            REACT_APP_INTERCOM_APP_ID=${{ secrets.APPSMITH_INTERCOM_ID }} \
-            yarn build
-          ls -l build
-
-      # Restore the previous built bundle if present. If not push the newly built into the cache
-      - name: Restore the previous bundle
-        uses: actions/cache@v3
-        with:
-          path: |
-            app/client/build/
-          key: ${{ github.run_id }}-${{ github.job }}-${{ steps.timestamp.outputs.timestamp }}
-          restore-keys: |
-            ${{ github.run_id }}-${{ github.job }}
-
-      # Upload the build artifact so that it can be used by the test & deploy job in the workflow
-      - name: Upload react build bundle
-        uses: actions/upload-artifact@v3
-        with:
-          name: client-build
-          path: app/client/build/
-
-      # Set status = success
-      - name: Save the status of the run
-        run: echo "run_result=success" >> $GITHUB_OUTPUT > ~/run_result
-
-  buildServer:
-    defaults:
-      run:
-        working-directory: app/server
-    runs-on: ubuntu-latest
-    # Only run this workflow for internally triggered events
-    if: |
-      github.event_name == 'workflow_dispatch' ||
-      github.event_name == 'push' ||
-      (github.event_name == 'pull_request_review' &&
-      github.event.review.state == 'approved' &&
-      github.event.pull_request.head.repo.full_name == github.repository)
-
-    # Service containers to run with this job. Required for running tests
-    services:
-      # Label used to access the service container
-      redis:
-        # Docker Hub image for Redis
-        image: redis
-        ports:
-          # Opens tcp port 6379 on the host and service container
-          - 6379:6379
-      mongo:
-        image: mongo
-        ports:
-          - 27017:27017
-
-    steps:
-      # Checkout the code
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-
-      # Timestamp will be used to create cache key
-      - id: timestamp
-        run: echo "timestamp=$(date +'%Y-%m-%dT%H:%M:%S')" >> $GITHUB_OUTPUT
-
-      # In case this is second attempt try restoring status of the prior attempt from cache
-      - name: Restore the previous run result
-        uses: actions/cache@v3
-        with:
-          path: |
-            ~/run_result
-          key: ${{ github.run_id }}-${{ github.job }}-${{ steps.timestamp.outputs.timestamp }}
-          restore-keys: |
-            ${{ github.run_id }}-${{ github.job }}-
-
-      # Fetch prior run result
-      - name: Get the previous run result
-        id: run_result
-        run: cat ~/run_result 2>/dev/null || echo 'default'
-
-      # In case of prior failure run the job
-      - if: steps.run_result.outputs.run_result != 'success'
-        run: echo "I'm alive!" && exit 0
-
-      # Setup Java
-      - name: Set up JDK 17
-        if: steps.run_result.outputs.run_result != 'success'
-        uses: actions/setup-java@v3
-        with:
-          distribution: 'temurin'
-          java-version: '17'
-
-      # Retrieve maven dependencies from cache. After a successful run, these dependencies are cached again
-      - name: Cache maven dependencies
-        if: steps.run_result.outputs.run_result != 'success'
-        uses: actions/cache@v3
-        env:
-          cache-name: cache-maven-dependencies
-        with:
-          # maven dependencies are stored in `~/.m2` on Linux/macOS
-          path: ~/.m2
-          key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
-          restore-keys: ${{ runner.os }}-m2
-
-      # Here, the GITHUB_REF is of type /refs/head/<branch_name>. We extract branch_name from this by removing the
-      # first 11 characters. This can be used to build images for several branches
-      # Since this is an unreleased build, we get the latest released version number, increment the minor number in it,
-      # append a `-SNAPSHOT` at it's end to prepare the snapshot version number. This is used as the project's version.
-      - name: Get the version to tag the Docker image
-        if: steps.run_result.outputs.run_result != 'success'
-        id: vars
-        run: |
-          # Since this is an unreleased build, we set the version to incremented version number with a
-          # `-SNAPSHOT` suffix.
-          latest_released_version="$(git tag --list 'v*' --sort=-version:refname | head -1)"
-          echo "latest_released_version = $latest_released_version"
-          next_version="$(echo "$latest_released_version" | awk -F. -v OFS=. '{ $NF++; print }')"
-          echo "next_version = $next_version"
-          echo version=$next_version-SNAPSHOT >> $GITHUB_OUTPUT
-          echo tag=$(echo ${GITHUB_REF:11}) >> $GITHUB_OUTPUT
-
-      - name: Test and Build package
-        if: steps.run_result.outputs.run_result != 'success'
-        env:
-          APPSMITH_MONGODB_URI: "mongodb://localhost:27017/mobtools"
-          APPSMITH_REDIS_URL: "redis://127.0.0.1:6379"
-          APPSMITH_ENCRYPTION_PASSWORD: "password"
-          APPSMITH_ENCRYPTION_SALT: "salt"
-          APPSMITH_IS_SELF_HOSTED: false
-          APPSMITH_GIT_ROOT: "./container-volumes/git-storage"
-          APPSMITH_AUDITLOG_ENABLED: true
-        working-directory: app/server
-        run: |
-          mvn --batch-mode versions:set \
-            -DnewVersion=${{ steps.vars.outputs.version }} \
-            -DgenerateBackupPoms=false \
-            -DprocessAllModules=true
-          ./build.sh -DskipTests
-          ls -l dist
-
-      # Restore the previous built bundle if present. If not push the newly built into the cache
-      - name: Restore the previous bundle
-        uses: actions/cache@v3
-        with:
-          path: |
-            app/server/dist/
-          key: ${{ github.run_id }}-${{ github.job }}-${{ steps.timestamp.outputs.timestamp }}
-          restore-keys: |
-            ${{ github.run_id }}-${{ github.job }}
-
-      # Upload the build artifact so that it can be used by the test & deploy job in the workflow
-      - name: Upload server build bundle
-        uses: actions/upload-artifact@v3
-        with:
     name: server-build
-          path: app/server/dist/
+    uses: ./.github/workflows/server-build.yml
-
+    secrets: inherit
-      - name: Save the status of the run
-        run: echo "run_result=success" >> $GITHUB_OUTPUT > ~/run_result
-
-  buildRts:
-    defaults:
-      run:
-        working-directory: app/rts
-    runs-on: ubuntu-latest
-    # Only run this workflow for internally triggered events
-    if: |
-      github.event_name == 'workflow_dispatch' ||
-      github.event_name == 'push' ||
-      (github.event_name == 'pull_request_review' &&
-      github.event.review.state == 'approved' &&
-      github.event.pull_request.head.repo.full_name == github.repository)
-
-    steps:
-      # Checkout the code
-      - uses: actions/checkout@v3
     with:
-          fetch-depth: 0
+      pr: 0
 
-      # Timestamp will be used to create cache key
+  client-build:
-      - id: timestamp
+    name: client-build
-        run: echo "timestamp=$(date +'%Y-%m-%dT%H:%M:%S')" >> $GITHUB_OUTPUT
+    uses: ./.github/workflows/client-build.yml
-
+    secrets: inherit
-      # In case this is second attempt try restoring status of the prior attempt from cache
-      - name: Restore the previous run result
-        uses: actions/cache@v3
     with:
-          path: |
+      pr: 0
-            ~/run_result
-          key: ${{ github.run_id }}-${{ github.job }}-${{ steps.timestamp.outputs.timestamp }}
-          restore-keys: |
-            ${{ github.run_id }}-${{ github.job }}-
 
-      # Fetch prior run result
+  rts-build:
-      - name: Get the previous run result
+    name: rts-build
-        id: run_result
+    uses: ./.github/workflows/rts-build.yml
-        run: cat ~/run_result 2>/dev/null || echo 'default'
+    secrets: inherit
-
-      # In case of prior failure run the job
-      - if: steps.run_result.outputs.run_result != 'success'
-        run: echo "I'm alive!" && exit 0
-
-      - name: Use Node.js 16.14.0
-        if: steps.run_result.outputs.run_result != 'success'
-        uses: actions/setup-node@v3
     with:
-          node-version: "16.14.0"
+      pr: 0
-
-      # Here, the GITHUB_REF is of type /refs/head/<branch_name>. We extract branch_name from this by removing the
-      # first 11 characters. This can be used to build images for several branches
-      # Since this is an unreleased build, we get the latest released version number, increment the minor number in it,
-      # append a `-SNAPSHOT` at it's end to prepare the snapshot version number. This is used as the project's version.
-      - name: Get the version to tag the Docker image
-        if: steps.run_result.outputs.run_result != 'success'
-        id: vars
-        run: |
-          # Since this is an unreleased build, we set the version to incremented version number with a
-          # `-SNAPSHOT` suffix.
-          latest_released_version="$(git tag --list 'v*' --sort=-version:refname | head -1)"
-          echo "latest_released_version = $latest_released_version"
-          next_version="$(echo "$latest_released_version" | awk -F. -v OFS=. '{ $NF++; print }')"
-          echo "next_version = $next_version"
-          echo version=$next_version-SNAPSHOT >> $GITHUB_OUTPUT
-          echo tag=$(echo ${GITHUB_REF:11}) >> $GITHUB_OUTPUT
-
-      - name: Build
-        if: steps.run_result.outputs.run_result != 'success'
-        run: |
-          echo 'export const VERSION = "${{ steps.vars.outputs.version }}"' > src/version.js
-          yarn build
-
-      # Restore the previous built bundle if present. If not push the newly built into the cache
-      - name: Restore the previous bundle
-        uses: actions/cache@v3
-        with:
-          path: |
-            app/rts/dist/
-          key: ${{ github.run_id }}-${{ github.job }}-${{ steps.timestamp.outputs.timestamp }}
-          restore-keys: |
-            ${{ github.run_id }}-${{ github.job }}
-
-      # Tar the bundles to speed up the upload & download process
-      - name: Tar the rts bundles
-        run: |
-          ls -al dist/node_modules/@shared/ast
-          tar -cvf rts-dist.tar dist
-      # Upload the build artifacts and dependencies so that it can be used by the test & deploy job in other workflows
-      - name: Upload rts build bundle
-        uses: actions/upload-artifact@v3
-        with:
-          name: rts-dist
-          path: app/rts/rts-dist.tar
-
-      # Set status = success
-      - name: Save the status of the run
-        run: echo "run_result=success" >> $GITHUB_OUTPUT > ~/run_result
       
   perf-test:
-    needs: [buildClient, buildServer, buildRts]
+    needs: [client-build, server-build, rts-build]
     # Only run if the build step is successful
     if: success()
     name: perf-test
@@ -410,7 +47,7 @@ jobs:
       pr: 0
 
   ci-test:
-    needs: [buildClient, buildServer, buildRts]
+    needs: [client-build, server-build, rts-build]
     # Only run if the build step is successful
     # If the build has been triggered manually via workflow_dispatch or via a push to protected branches
     # then we don't check for the PR approved state
@@ -524,13 +161,13 @@ jobs:
       - name: Download the react build artifact
         uses: actions/download-artifact@v3
         with:
-          name: client-build
+          name: build
           path: app/client/build
 
       - name: Download the server build artifact
         uses: actions/download-artifact@v3
         with:
-          name: server-build
+          name: build
           path: app/server/dist
 
       - name: Download the rts build artifact