PromucFlow_constructor/.github/workflows/server-build.yml

# This workflow is responsible for building, testing & packaging the Java server codebase
name: Appsmith Server Workflow

on:
  workflow_call:
    inputs:
      pr:
        description: "This is the PR number in case the workflow is being called in a pull request"
        required: false
        type: number
      skip-tests:
        description: "This is a boolean value in case the workflow is being called in build deploy-preview"
        required: false
        type: string
        default: "false"
      branch:
        description: "This is the branch to be used for the build."
        required: false
        type: string
      is-pg-build:
        description: "This is a boolean value in case the workflow is being called for a PG build"
        required: false
        type: string
        default: "false"

  workflow_dispatch:
    inputs:
      pr:
        description: "PR number for the workflow"
        required: false
        type: number
      skip-tests:
        description: "Skip tests flag"
        required: false
        type: string
        default: "false"
      branch:
        description: "Branch for the build"
        required: false
        type: string
      is-pg-build:
        description: "Flag for PG build"
        required: false
        type: string
        default: "false"

# Change the working directory for all the jobs in this workflow
defaults:
  run:
    working-directory: app/server

jobs:
  server-unit-tests:
    runs-on: ubuntu-latest-8-cores

    # Service containers to run with this job. Required for running tests
    services:
      # Label used to access the service container
      redis:
        # Docker Hub image for Redis
        image: redis
        ports:
          # Opens tcp port 6379 on the host and service container
          - 6379:6379

    steps:
      # The checkout steps MUST happen first because the default directory is set according to the code base.
      # GitHub Action expects all future commands to be executed in the code directory. Hence, we need to check out
      # the code before doing anything else.

      # Check out merge commit with the base branch in case this workflow is invoked via pull request
      - name: Check out merged commit from PR and base branch
        uses: actions/checkout@v4
        if: inputs.pr != 0
        with:
          fetch-tags: true
          ref: refs/pull/${{ inputs.pr }}/merge

      # Check out the specified branch in case this workflow is called by another workflow
      - name: Checkout the specified branch
        if: inputs.pr == 0 && inputs.branch != ''
        uses: actions/checkout@v4
        with:
          fetch-tags: true
          ref: ${{ inputs.branch }}

      # Checkout the code in the current branch in case the workflow is called because of a branch push event
      - name: Check out the head commit of the branch
        uses: actions/checkout@v4
        if: inputs.pr == 0 && inputs.branch == ''
        with:
          fetch-tags: true

      - name: Figure out the PR number
        run: echo ${{ inputs.pr }}
      
      - name: Default database URL
        run: echo "Is this a PG build? ${{ inputs.is-pg-build }}"

      - name: Print the Github event
        run: echo ${{ github.event_name }}

      # - name: Get changed files in the server folder
      #   id: changed-files-specific
      #   uses: tj-actions/changed-files@v44
      #   with:
      #     files: "app/server/**"
      #     write_output_files: true

      - name: Updating the server changed file variable
        id: changed-files-specific
        run: echo "any_changed=true" >> "$GITHUB_OUTPUT"

      # - name: Run step if any file(s) in the server folder change
      #   if: steps.changed-files-specific.outputs.any_changed == 'true'
      #   run: |
      #     echo "One or more files in the server folder has changed."
      #     echo "List all the files that have changed:"
      #     cat "${{ github.workspace }}/.github/outputs/all_changed_files.txt"

      # In case this is second attempt try restoring status of the prior attempt from cache
      - name: Restore the previous run result
        if: inputs.skip-tests != 'true' && (steps.changed-files-specific.outputs.any_changed == 'true' ||  github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule')
        id: cache-appsmith
        uses: actions/cache@v4
        with:
          path: |
            ~/run_result            
          key: ${{ github.run_id }}-${{ github.job }}-server-junit

      # Fetch prior run result
      - name: Get the previous run result
        if: inputs.skip-tests != 'true' && (steps.changed-files-specific.outputs.any_changed == 'true' ||  github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule')
        id: run_result
        run: |
          if [ -f ~/run_result ]; then
            echo "run_result=$(cat ~/run_result)" >> $GITHUB_OUTPUT
          else
            echo "run_result=default" >> $GITHUB_OUTPUT
          fi          

      - name: Download the failed test artifact in case of rerun
        if: steps.run_result.outputs.run_result == 'failedtest' && (steps.changed-files-specific.outputs.any_changed == 'true' ||  github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule')
        uses: actions/download-artifact@v4
        with:
          name: failed-server-tests
          path: ~/failed-server-tests

      - name: Extract the tests for rerun
        id: failed_tests
        if: steps.run_result.outputs.run_result == 'failedtest'
        run: |
          failed_tests=$(awk '$0 != "" && !seen[$0]++ {printf("%s%s",sep,$0); sep=","}' ~/failed-server-tests/failed-server-tests.txt)
          echo "$failed_tests"
          echo "tests=$failed_tests" >> $GITHUB_OUTPUT          

      # In case of prior failure run the job
      - if: steps.run_result.outputs.run_result != 'success'  && (steps.changed-files-specific.outputs.any_changed == 'true' ||  github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule')
        run: echo "I'm alive!" && exit 0

      # Setup Java
      - name: Set up JDK 17
        if: steps.run_result.outputs.run_result != 'success'  && (steps.changed-files-specific.outputs.any_changed == 'true' ||  github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule')
        uses: actions/setup-java@v4
        with:
          distribution: "temurin"
          java-version: "17"

      - name: Conditionally start PostgreSQL
        if: |
          inputs.is-pg-build == 'true' && inputs.skip-tests != 'true'          
        run: |
          docker run --name appsmith-pg -p 5432:5432 -d -e POSTGRES_PASSWORD=password postgres:alpine postgres -N 1500          

      # Retrieve maven dependencies from cache. After a successful run, these dependencies are cached again
      - name: Cache maven dependencies
        if: steps.run_result.outputs.run_result != 'success'  && (steps.changed-files-specific.outputs.any_changed == 'true' ||  github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule')
        uses: actions/cache@v4
        env:
          cache-name: cache-maven-dependencies
        with:
          # maven dependencies are stored in `~/.m2` on Linux/macOS
          path: ~/.m2
          key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
          restore-keys: ${{ runner.os }}-m2

      # Build the code
      - name: Build
        if: steps.run_result.outputs.run_result != 'success'  && (steps.changed-files-specific.outputs.any_changed == 'true' ||  github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule')
        run: |
          ./build.sh -DskipTests          

      # Test the code
      - name: Run only tests
        if: (inputs.skip-tests != 'true' || steps.run_result.outputs.run_result == 'failedtest')  && (steps.changed-files-specific.outputs.any_changed == 'true' ||  github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule')
        env:
          ACTIVE_PROFILE: test
          APPSMITH_CLOUD_SERVICES_BASE_URL: "https://release-cs.appsmith.com"
          APPSMITH_CLOUD_SERVICES_TEMPLATE_UPLOAD_AUTH: ${{ secrets.APPSMITH_CLOUD_SERVICES_TEMPLATE_UPLOAD_AUTH }}
          APPSMITH_REDIS_URL: "redis://127.0.0.1:6379"
          APPSMITH_ENCRYPTION_PASSWORD: "password"
          APPSMITH_ENCRYPTION_SALT: "salt"
          APPSMITH_ENVFILE_PATH: /tmp/dummy.env
          APPSMITH_VERBOSE_LOGGING_ENABLED: false
        run: |
            if [[ "${{ inputs.is-pg-build }}" == "true" ]]; then
                export APPSMITH_DB_URL="postgresql://postgres:password@localhost:5432/postgres"
            else
                export APPSMITH_DB_URL="mongodb://localhost:27017/mobtools"
            fi

            args=()
            if [[ "${{ steps.run_result.outputs.run_result }}" == "failedtest" ]]; then
                failed_tests="${{ steps.failed_tests.outputs.tests }}"
                args+=("-DfailIfNoTests=false" "-Dsurefire.failIfNoSpecifiedTests=false" "-Dtest=${failed_tests}")
            fi

            # Run tests and capture logs
            mvn test "${args[@]}" | tee mvn_test.log
            
            # Check for "BUILD FAILURE" in the mvn_test.log
            if grep -q "BUILD FAILURE" mvn_test.log; then
                test_result="failed"
            else
                test_result="passed"
            fi

            echo "test_result variable value: ${test_result}"

            # Prepare output file for failed tests and ensure a fresh file is created
            OUTPUT_FILE="failed-server-tests.txt"
            rm -f "$OUTPUT_FILE"
            touch "$OUTPUT_FILE"

            failed_modules=()
            skipped_modules=()

            # Process mvn_test.log for FAILURE and SKIPPED statuses
            while IFS= read -r line; do
                if [[ $line == *"FAILURE"* ]]; then
                    module_name=$(echo "$line" | awk '{print $2}')
                    failed_modules+=("$module_name")
                elif [[ $line == *"SKIPPED"* ]]; then
                    module_name=$(echo "$line" | awk '{print $2}')
                    skipped_modules+=("$module_name")
                fi
            done < mvn_test.log

            echo "Failed Modules: ${failed_modules[*]}"
            echo "Skipped Modules: ${skipped_modules[*]}"

            # Handle older approach for reading failed tests from XML files
            failed_tests_from_xml="$PWD/failed-tests-from-xml.txt"
            gawk -F\" '/<testcase / {cur_test = $4 "#" $2} /<(failure|error) / {print cur_test}' $(find . -type f -name 'TEST-*.xml') \
              | sort -u \
              | tee "$failed_tests_from_xml"

            # Filter out failed modules and add only relevant tests to the final failed list
            for module in "${failed_modules[@]}"; do
                grep -v "$module" "$failed_tests_from_xml" > temp_file && mv temp_file "$failed_tests_from_xml"
            done

            # Include all skipped module test files in the final list
            for module in "${skipped_modules[@]}"; do
                module_directories=$(find . -path "*/${module}*/src/test/java/*" -type f -name "*Test.java" -exec dirname {} \; | sort -u)
                for module_directory in $module_directories; do
                    test_classes=$(find "$module_directory" -type f -name "*Test.java" | sed 's|.*/src/test/java/||; s|\.java$||; s|/|.|g')
                    for class_name in $test_classes; do
                        if [[ ${#class_name} -le 240 ]] && ! grep -Fxq "$class_name#" "$OUTPUT_FILE"; then
                            echo "${class_name}#" >> "$OUTPUT_FILE"
                        fi
                    done
                done
            done

            # Combine the XML file test cases and skipped module test files into the final output file
            cat "$failed_tests_from_xml" >> "$OUTPUT_FILE"
            
            # Print the final output
            cat "$OUTPUT_FILE"

            if [[ -s $OUTPUT_FILE ]]; then
                content="$(
                    echo "## Failed server tests"
                    echo
                    sed 's/^/- /' "$OUTPUT_FILE"
                )"
                echo "$content" >> "$GITHUB_STEP_SUMMARY"
                
                # Post a comment to the PR
                curl --silent --show-error \
                    --header "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
                    --data "$(jq -n --arg body "$content" '$ARGS.named')" \
                    "https://api.github.com/repos/$GITHUB_REPOSITORY/issues/${{ inputs.pr }}/comments" \
                    > /dev/null
            fi

            # Fail the script if tests did not pass
            if [[ "$test_result" == "failed" ]]; then
                echo "Tests failed, exiting with status 1."
                exit 1
            fi            


      # Set status = failedtest
      - name: Set fail if there are test failures
        if: failure()
        run: |
          echo "run_result=failedtest" >> $GITHUB_OUTPUT
          echo "failedtest" > ~/run_result          

      # Force store previous run result to cache
      - name: Store the previous run result
        if: failure()
        uses: actions/cache/save@v4
        with:
          path: |
            ~/run_result            
          key: ${{ github.run_id }}-${{ github.job }}-server-junit

      - name: Upload the failed tests report
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: failed-server-tests
          path: app/server/failed-server-tests.txt
          if-no-files-found: ignore
          overwrite: true

      - name: Fetch server build from cache
        if: steps.changed-files-specific.outputs.any_changed == 'false' && success()  && github.event_name != 'push' && github.event_name != 'workflow_dispatch'  && github.event_name != 'schedule'
        env:
          cachetoken: ${{ secrets.CACHETOKEN }}
          reponame: ${{ github.event.repository.name }}
          gituser: ${{ secrets.CACHE_GIT_USER }}
          gituseremail: ${{ secrets.CACHE_GIT_EMAIL }}
        run: |
          mkdir cacherepo
          cd ./cacherepo
          git lfs install
          git config --global user.email "$gituseremail"
          git config --global user.name "$gituser"
          git clone https://$cachetoken@github.com/appsmithorg/cibuildcache.git
          git lfs install
          git lfs migrate import --everything --yes
          if [ "$reponame" = "appsmith" ]; then export repodir="CE"; fi
          if [ "$reponame" = "appsmith-ee" ]; then export repodir="EE"; fi
          cd cibuildcache/$repodir/release/server
          git lfs pull ./server.jar
          mv ./server.jar ../../../../../server.jar
          cd ../../../../../
          tar -xzvf ./server.jar          

      # Restore the previous built bundle if present. If not push the newly built into the cache
      - name: Restore the previous bundle
        if: steps.changed-files-specific.outputs.any_changed == 'true' ||  github.event_name == 'push' || github.event_name == 'workflow_dispatch'
        uses: actions/cache@v4
        with:
          path: |
            app/server/dist/            
          key: ${{ github.run_id }}-${{ github.job }}-server

      # Upload the build artifact so that it can be used by the test & deploy job in the workflow
      - name: Upload server build bundle
        uses: actions/upload-artifact@v4
        with:
          name: server-build
          path: app/server/dist/
          overwrite: true

      - name: Put release build in cache
        if: success() && github.ref == 'refs/heads/release' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule')
        env:
          cachetoken: ${{ secrets.CACHETOKEN }}
          reponame: ${{ github.event.repository.name }}
          gituser: ${{ secrets.CACHE_GIT_USER }}
          gituseremail: ${{ secrets.CACHE_GIT_EMAIL }}
        run: |
          pwd
          tar -czvf server.jar dist/
          mkdir cacherepo
          cd ./cacherepo
          git config --global user.email "$gituseremail"
          git config --global user.name "$gituser"
          git clone https://$cachetoken@github.com/appsmithorg/cibuildcache.git
          git lfs install
          cd cibuildcache/
          if [ "$reponame" = "appsmith" ]; then export repodir="CE"; fi
          if [ "$reponame" = "appsmith-ee" ]; then export repodir="EE"; fi
          cd $repodir/release/server
          cp ../../../../../server.jar ./
          git lfs track "server.jar"
          git add  server.jar
          git commit --allow-empty -m "Update Latest Server.jar"
          git push          

      - name: Save the status of the run
        run: echo "run_result=success" >> $GITHUB_OUTPUT > ~/run_result