From 828c6aeefedf25a2715b132c60023151785fea2e Mon Sep 17 00:00:00 2001 From: Sagar Khalasi Date: Mon, 5 Aug 2024 21:04:45 +0530 Subject: [PATCH] chore: Adding new command for running test case (#35419) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Description It is for running a spec file with given run_count with PR comment. Fixes #`34956` ## Automation /ok-to-test tags="@tag.Sanity" ### :mag: Cypress test results > [!TIP] > 🟢 🟢 🟢 All cypress tests have passed! 🎉 🎉 🎉 > Workflow run: > Commit: 69556d529719ff00d6d0b6961edb83a2a6112541 > Cypress dashboard. > Tags: `@tag.Sanity` > Spec: >
Mon, 05 Aug 2024 14:49:54 UTC ## Communication Should the DevRel and Marketing teams inform users about this change? - [ ] Yes - [x] No ## Summary by CodeRabbit - **New Features** - Introduced a CI/CD workflow to manage client and server builds with Cypress testing. - Added a job to enforce TypeScript file checks, providing feedback for corrections on pull requests. - Expanded testing capabilities with a new job for managing testing limits. - **Improvements** - Enhanced logging for better visibility into test execution attempts. --- .../workflows/build-client-server-count.yml | 380 ++++++++++++++++++ .github/workflows/ok-to-test.yml | 1 + .../cypress/scripts/cypress-split-static.ts | 1 + 3 files changed, 382 insertions(+) create mode 100644 .github/workflows/build-client-server-count.yml diff --git a/.github/workflows/build-client-server-count.yml b/.github/workflows/build-client-server-count.yml new file mode 100644 index 0000000000..a05ca04cc8 --- /dev/null +++ b/.github/workflows/build-client-server-count.yml @@ -0,0 +1,380 @@ +name: Build Client, Server & Run only Cypress + +on: + repository_dispatch: + types: [ci-test-limit-count] + +jobs: + file-check: + runs-on: ubuntu-latest + # Set job outputs to values from filter step + outputs: + non_ts_files: ${{ steps.check_files.outputs.non_ts_files }} + non_ts_files_count: ${{ steps.check_files.outputs.non_ts_files_count }} + pr: ${{steps.args.outputs.pr}} + runId: ${{steps.args.outputs.runId}} + matrix_count: ${{steps.matrix.outputs.matrix_count}} + steps: + - name: Checkout the head commit of the branch + uses: actions/checkout@v4 + + - name: Set matrix jobs + id: matrix + run: | + echo "matrix_count=[0, 1, 2]" >> $GITHUB_OUTPUT + + - name: Set args + id: args + run: | + echo "pr=${{ github.event.client_payload.pull_request.number }}" >> $GITHUB_OUTPUT + checkArg=`echo '${{toJSON(github.event.client_payload.slash_command.args.named)}}' | jq 'has("runId")'` + if [[ $checkArg == 'true' ]]; then + echo "runId=${{ github.event.client_payload.slash_command.args.named.runId }}" >> $GITHUB_OUTPUT + else + echo "runId=0" >> $GITHUB_OUTPUT + fi + + - name: Get the diff from base branch + continue-on-error: true + id: files + run: | + git fetch origin release + git diff --name-only --diff-filter=A remotes/origin/release...${{ github.ref_name }} -- 'app/client/cypress/e2e' > diff + echo "files_added=$(cat diff)" >> $GITHUB_OUTPUT + cat diff + + - name: Check the newly added files are written in ts + id: check_files + run: | + files=(${{steps.files.outputs.files_added}}) + non_ts_files=() + for file in "${files[@]}"; do + if [[ $file != *.ts ]]; then + non_ts_files+=("
  • $file") + fi + done + echo "non_ts_files=${non_ts_files[@]}" >> $GITHUB_OUTPUT + echo "non_ts_files_count=${#non_ts_files[@]}" >> $GITHUB_OUTPUT + + - name: Print the files + if: steps.check_files.outputs.non_ts_files_count != 0 && steps.args.outputs.pr == '0' + run: | + echo "${{ steps.check_files.outputs.non_ts_files }}" + + - name: Comment the filenames if PR is there + if: steps.check_files.outputs.non_ts_files_count != 0 && steps.args.outputs.pr != '0' + uses: peter-evans/create-or-update-comment@v3 + with: + issue-number: ${{ fromJson(steps.args.outputs.pr) }} + body: | + Below new test files are written in js 🔴 + Expected format ts. Please fix and retrigger ci-test-limit: +
      ${{ steps.check_files.outputs.non_ts_files }}
    + - if: steps.check_files.outputs.non_ts_files_count != 0 + run: exit 1 + + - name: Add a comment on the PR with link to workflow run + if: success() && steps.args.outputs.pr != '0' + uses: peter-evans/create-or-update-comment@v3 + with: + issue-number: ${{ fromJson(steps.args.outputs.pr) }} + body: | + Tests running at: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}>. + [Cypress dashboard](https://internal.appsmith.com/app/cypress-dashboard/rundetails-65890b3c81d7400d08fa9ee5?branch=master&workflowId=${{ github.run_id }}&attempt=${{ github.run_attempt }}&selectiontype=test&testsstatus=failed&specsstatus=fail) + PR: #${{ fromJson(steps.args.outputs.pr) }}. + + server-build: + name: server-build + needs: [file-check] + if: success() && needs.file-check.outputs.runId == '0' + uses: ./.github/workflows/server-build.yml + secrets: inherit + with: + pr: ${{fromJson(needs.file-check.outputs.pr)}} + skip-tests: "true" + + client-build: + name: client-build + needs: [file-check] + if: success() && needs.file-check.outputs.runId == '0' + uses: ./.github/workflows/client-build.yml + secrets: inherit + with: + pr: ${{fromJson(needs.file-check.outputs.pr)}} + skip-tests: "true" + + rts-build: + name: rts-build + needs: [file-check] + if: success() && needs.file-check.outputs.runId == '0' + uses: ./.github/workflows/rts-build.yml + secrets: inherit + with: + pr: ${{fromJson(needs.file-check.outputs.pr)}} + + build-docker-image: + needs: [file-check, client-build, server-build, rts-build] + # Only run if the build step is successful + if: success() && needs.file-check.outputs.runId == '0' + name: build-docker-image + uses: ./.github/workflows/build-docker-image.yml + secrets: inherit + with: + pr: ${{fromJson(needs.file-check.outputs.pr)}} + + ci-test-limited: + needs: [file-check, build-docker-image] + # Only run if the build step is successful + if: success() && needs.file-check.outputs.runId == '0' + name: ci-test-limited + uses: ./.github/workflows/ci-test-limited-with-count.yml + secrets: inherit + with: + pr: ${{fromJson(needs.file-check.outputs.pr)}} + + ci-test-limited-existing-docker-image: + needs: [file-check] + # Only run if the previous run-id is provided + if: success() && needs.file-check.outputs.runId != '0' + name: ci-test-limited-existing-image + uses: ./.github/workflows/ci-test-limited-with-count.yml + secrets: inherit + with: + pr: ${{fromJson(needs.file-check.outputs.pr)}} + previous-workflow-run-id: ${{ fromJson(needs.file-check.outputs.runId) }} + + ci-test-limited-result: + needs: [file-check, ci-test-limited] + # Only run if the file-check.runId == 0 + if: always() && needs.file-check.outputs.runId == '0' + runs-on: ubuntu-latest + defaults: + run: + shell: bash + steps: + - name: Setup node + if: needs.ci-test-limited.result != 'success' + uses: actions/setup-node@v4 + with: + node-version-file: app/client/package.json + + - name: install pg + if: needs.ci-test-limited.result != 'success' + run: npm install pg + + - name: Fetch the failed specs + if: needs.ci-test-limited.result != 'success' + id: failed_specs + env: + DB_HOST: ${{ secrets.CYPRESS_DB_HOST }} + DB_NAME: ${{ secrets.CYPRESS_DB_NAME }} + DB_USER: ${{ secrets.CYPRESS_DB_USER }} + DB_PWD: ${{ secrets.CYPRESS_DB_PWD }} + RUN_ID: ${{ github.run_id }} + ATTEMPT_NUMBER: ${{ github.run_attempt }} + uses: actions/github-script@v7 + with: + script: | + const { Pool } = require("pg"); + const { DB_HOST, DB_NAME, DB_USER, DB_PWD, RUN_ID, ATTEMPT_NUMBER } = process.env + + const client = await new Pool({ + user: DB_USER, + host: DB_HOST, + database: DB_NAME, + password: DB_PWD, + port: 5432, + connectionTimeoutMillis: 60000, + }).connect(); + + const result = await client.query( + `SELECT DISTINCT name FROM public."specs" + WHERE "matrixId" IN + (SELECT id FROM public."matrix" + WHERE "attemptId" = ( + SELECT id FROM public."attempt" WHERE "workflowId" = $1 and "attempt" = $2 + ) + ) AND status = 'fail'`, + [RUN_ID, ATTEMPT_NUMBER], + ); + client.release(); + return result.rows.map((spec) => spec.name); + + # In case for any ci job failure, create combined failed spec + - name: combine all specs for CI + id: combine_ci + if: needs.ci-test-limited.result != 'success' + run: | + failed_specs=$(echo ${{steps.failed_specs.outputs.result}} | sed 's/\[\|\]//g' | tr -d ' ' | tr ',' '\n') + while read -r line; do + echo "$line" >> ~/combined_failed_spec_ci + done <<< "$failed_specs" + + # Upload combined failed CI spec list to a file + # This is done for debugging. + - name: upload combined failed spec + if: needs.ci-test-limited.result != 'success' + uses: actions/upload-artifact@v4 + with: + name: combined_failed_spec_ci + path: ~/combined_failed_spec_ci + overwrite: true + + - name: Get Latest flaky Tests + shell: bash + run: | + curl --request POST --url https://yatin-s-workspace-jk8ru5.us-east-1.xata.sh/db/CypressKnownFailures:main/tables/CypressKnownFailuires/query --header 'Authorization: Bearer ${{ secrets.XATA_TOKEN }}' --header 'Content-Type: application/json'|jq -r |grep Spec|cut -d ':' -f 2 2> /dev/null|sed 's/"//g'|sed 's/,//g' > ~/knownfailures + + # Verify CI test failures against known failures + - name: Verify CI test failures against known failures + if: needs.ci-test-limited.result != 'success' + shell: bash + run: | + new_failed_spec_env="
      $(comm -1 -3 <(sort ~/knownfailures) <(sort -u ~/combined_failed_spec_ci) | sed 's/|cypress|cypress/\n/g' | sed 's/^/
    1. /')
    " + echo "$new_failed_spec_env" + echo "new_failed_spec_env<> $GITHUB_ENV + echo "$new_failed_spec_env" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + + - name: Add a comment on the PR with new CI failures + if: needs.ci-test-limited.result != 'success' && needs.file-check.outputs.pr != '0' + uses: peter-evans/create-or-update-comment@v3 + with: + issue-number: ${{fromJson(needs.file-check.outputs.pr)}} + body: | + Workflow run: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}>. + Cypress dashboard: Click here! + The following are new failures, please fix them before merging the PR: ${{env.new_failed_spec_env}} + To know the list of identified flaky tests - Refer here + + - name: Add a comment on the PR when ci-test-limited is success + if: needs.ci-test-limited.result == 'success' && needs.file-check.outputs.pr != '0' + uses: peter-evans/create-or-update-comment@v3 + with: + issue-number: ${{fromJson(needs.file-check.outputs.pr)}} + body: | + Workflow run: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}>. + Cypress dashboard url: Click here! + All cypress tests have passed 🎉🎉🎉 + + - name: Check ci-test-limited set status + if: needs.ci-test-limited.result != 'success' + run: exit 1 + + ci-test-limited-result-existing: + needs: [file-check, ci-test-limited-existing-docker-image] + # Only run if the file-check.runId !=0 + if: always() && needs.file-check.outputs.runId != '0' + runs-on: ubuntu-latest + defaults: + run: + shell: bash + steps: + - name: Setup node + if: needs.ci-test-limited-existing-docker-image.result != 'success' + uses: actions/setup-node@v4 + with: + node-version-file: app/client/package.json + + - name: install pg + if: needs.ci-test-limited-existing-docker-image.result != 'success' + run: npm install pg + + - name: Fetch the failed specs + if: needs.ci-test-limited-existing-docker-image.result != 'success' + id: failed_specs + env: + DB_HOST: ${{ secrets.CYPRESS_DB_HOST }} + DB_NAME: ${{ secrets.CYPRESS_DB_NAME }} + DB_USER: ${{ secrets.CYPRESS_DB_USER }} + DB_PWD: ${{ secrets.CYPRESS_DB_PWD }} + RUN_ID: ${{ github.run_id }} + ATTEMPT_NUMBER: ${{ github.run_attempt }} + uses: actions/github-script@v7 + with: + script: | + const { Pool } = require("pg"); + const { DB_HOST, DB_NAME, DB_USER, DB_PWD, RUN_ID, ATTEMPT_NUMBER } = process.env + + const client = await new Pool({ + user: DB_USER, + host: DB_HOST, + database: DB_NAME, + password: DB_PWD, + port: 5432, + connectionTimeoutMillis: 60000, + }).connect(); + + const result = await client.query( + `SELECT DISTINCT name FROM public."specs" + WHERE "matrixId" IN + (SELECT id FROM public."matrix" + WHERE "attemptId" = ( + SELECT id FROM public."attempt" WHERE "workflowId" = $1 and "attempt" = $2 + ) + ) AND status = 'fail'`, + [RUN_ID, ATTEMPT_NUMBER], + ); + client.release(); + return result.rows.map((spec) => spec.name); + + # In case for any ci job failure, create combined failed spec + - name: combine all specs for CI + id: combine_ci + if: needs.ci-test-limited-existing-docker-image.result != 'success' + run: | + failed_specs=$(echo ${{steps.failed_specs.outputs.result}} | sed 's/\[\|\]//g' | tr -d ' ' | tr ',' '\n') + while read -r line; do + echo "$line" >> ~/combined_failed_spec_ci + done <<< "$failed_specs" + + # Upload combined failed CI spec list to a file + # This is done for debugging. + - name: upload combined failed spec + if: needs.ci-test-limited-existing-docker-image.result != 'success' + uses: actions/upload-artifact@v4 + with: + name: combined_failed_spec_ci + path: ~/combined_failed_spec_ci + overwrite: true + + - name: Get Latest flaky Tests + shell: bash + run: | + curl --request POST --url https://yatin-s-workspace-jk8ru5.us-east-1.xata.sh/db/CypressKnownFailures:main/tables/CypressKnownFailuires/query --header 'Authorization: Bearer ${{ secrets.XATA_TOKEN }}' --header 'Content-Type: application/json'|jq -r |grep Spec|cut -d ':' -f 2 2> /dev/null|sed 's/"//g'|sed 's/,//g' > ~/knownfailures + + # Verify CI test failures against known failures + - name: Verify CI test failures against known failures + if: needs.ci-test-limited-existing-docker-image.result != 'success' + shell: bash + run: | + new_failed_spec_env="
      $(comm -1 -3 <(sort ~/knownfailures) <(sort -u ~/combined_failed_spec_ci) | sed 's/|cypress|cypress/\n/g' | sed 's/^/
    1. /')
    " + echo "$new_failed_spec_env" + echo "new_failed_spec_env<> $GITHUB_ENV + echo "$new_failed_spec_env" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + + - name: Add a comment on the PR with new CI failures + if: needs.ci-test-limited-existing-docker-image.result != 'success' && needs.file-check.outputs.pr != '0' + uses: peter-evans/create-or-update-comment@v3 + with: + issue-number: ${{fromJson(needs.file-check.outputs.pr)}} + body: | + Workflow run: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}>. + Cypress dashboard: Click here! + The following are new failures, please fix them before merging the PR: ${{env.new_failed_spec_env}} + To know the list of identified flaky tests - Refer here + + - name: Add a comment on the PR when ci-test-limited-existing-docker-image is success + if: needs.ci-test-limited-existing-docker-image.result == 'success' && needs.file-check.outputs.pr != '0' + uses: peter-evans/create-or-update-comment@v3 + with: + issue-number: ${{fromJson(needs.file-check.outputs.pr)}} + body: | + Workflow run: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}>. + Cypress dashboard url: Click here! + All cypress tests have passed 🎉🎉🎉 + + - name: Check ci-test-limited-existing-docker-image set status + if: needs.ci-test-limited-existing-docker-image.result != 'success' + run: exit 1 \ No newline at end of file diff --git a/.github/workflows/ok-to-test.yml b/.github/workflows/ok-to-test.yml index 55d45a6c06..7f51a62e8e 100644 --- a/.github/workflows/ok-to-test.yml +++ b/.github/workflows/ok-to-test.yml @@ -31,4 +31,5 @@ jobs: ok-to-test ci-test-limit build-deploy-preview + ci-test-limit-count permission: write diff --git a/app/client/cypress/scripts/cypress-split-static.ts b/app/client/cypress/scripts/cypress-split-static.ts index 5e75b51901..ccd8cb8e48 100644 --- a/app/client/cypress/scripts/cypress-split-static.ts +++ b/app/client/cypress/scripts/cypress-split-static.ts @@ -220,6 +220,7 @@ export class staticSplit { const specs = (await this.getSpecsToRun(specPattern, ignorePattern, attempt)) ?? []; console.log("SPECS TO RUN ----------> :", specs); + console.log("attempt ID ----------> :", attempt); if (specs.length > 0 && !specs.includes(defaultSpec)) { config.specPattern = specs.length == 1 ? specs[0] : specs; } else {