diff --git a/.github/workflows/integration-tests-command.yml b/.github/workflows/integration-tests-command.yml index 6f237fd821..1d2e867776 100644 --- a/.github/workflows/integration-tests-command.yml +++ b/.github/workflows/integration-tests-command.yml @@ -12,396 +12,12 @@ jobs: tags: ${{ steps.checkAll.outputs.tags }} matrix: ${{ steps.checkAll.outputs.matrix }} steps: - - name: Check if tags arg is present in the command - id: checkTags - run: | - checkArg=`echo '${{toJSON(github.event.client_payload.slash_command.args.named)}}' | jq 'has("tags")'` - echo "tags=$checkArg" >> $GITHUB_OUTPUT - - - name: Add a comment on the PR with tags documentation link - if: steps.checkTags.outputs.tags != 'true' + - name: Add a comment on the PR mentioning that this method of running ok-to-test is now defunct uses: peter-evans/create-or-update-comment@v3 with: issue-number: ${{ github.event.client_payload.pull_request.number }} body: | - The provided command lacks any tags. Please execute '/ok-to-test' again, specifying the tags you want to include or use `/ok-to-test tags="@tag.All"` to run all specs. - Explore the tags documentation [here](https://www.notion.so/appsmith/Ok-to-test-With-Tags-7c0fc64d4efb4afebf53348cd6252918) + Whoops! Looks like you're using an outdated method of running the Cypress suite. + Please check this doc to learn how to correct this! - - name: Stop the workflow run if tags are not present - if: steps.checkTags.outputs.tags != 'true' - run: exit 1 - - - name: Check if @tag.All is present in tags - if: steps.checkTags.outputs.tags == 'true' - id: checkAll - run: | - tags="${{ github.event.client_payload.slash_command.args.named.tags }}" - if [[ ($tags == *"ALL"* || $tags == *"All"* || $tags == *"all"*) && $tags != *"@tag.All"* ]]; then - echo "invalid_tags_all=$tags" >> $GITHUB_OUTPUT - elif [[ $tags == *"@tag.All"* ]]; then - echo "tags=" >> $GITHUB_OUTPUT - echo "matrix=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59]" >> $GITHUB_OUTPUT - else - echo "tags=$tags" >> $GITHUB_OUTPUT - echo "matrix=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19]" >> $GITHUB_OUTPUT - fi - - - name: Add comment to use correct @tag.All format - if: steps.checkAll.outputs.invalid_tags_all != '' - uses: peter-evans/create-or-update-comment@v3 - with: - issue-number: ${{ github.event.client_payload.pull_request.number }} - body: | - Please use `/ok-to-test tags="@tag.All"` to run all specs. - Explore the tags documentation [here](https://www.notion.so/appsmith/Ok-to-test-With-Tags-7c0fc64d4efb4afebf53348cd6252918) - - - name: Stop the workflow run if given @tag.All format is wrong - if: steps.checkAll.outputs.invalid_tags_all != '' - run: exit 1 - - # tags is set to empty string if @tag.All is given in step - Check if @tag.All is present in tags - - name: Add suggestion in the PR on using @tag.All - if: steps.checkAll.outputs.tags == '' - uses: peter-evans/create-or-update-comment@v3 - with: - issue-number: ${{ github.event.client_payload.pull_request.number }} - body: | - Whoa, @tag.All spotted in your test suite! 🚀 - While @tag.All is cool, like a catch-all net, why not try specific tags? 🏷️ - Narrow down your suite with specific tags for quicker and more accurate tests! 🚀 Less waiting, more zipping through tests like a ninja! - Explore the tags documentation [here](https://www.notion.so/appsmith/Ok-to-test-With-Tags-7c0fc64d4efb4afebf53348cd6252918) - - # This step creates a comment on the PR with a link to this workflow run. - - name: Add a comment on the PR with link to workflow run - if: steps.checkTags.outputs.tags == 'true' - uses: peter-evans/create-or-update-comment@v3 - with: - issue-number: ${{ github.event.client_payload.pull_request.number }} - body: | - Tests running at: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}>. - Workflow: `${{ github.workflow }}`. - Tags: `${{ steps.checkAll.outputs.tags }}`. - - server-build: - needs: [tag-check] - if: success() - name: server-build - uses: ./.github/workflows/server-build.yml - secrets: inherit - with: - pr: ${{ github.event.client_payload.pull_request.number }} - skip-tests: true - - client-build: - needs: [tag-check] - if: success() - name: client-build - uses: ./.github/workflows/client-build.yml - secrets: inherit - with: - pr: ${{ github.event.client_payload.pull_request.number }} - check-test-files: "true" - - rts-build: - needs: [tag-check] - if: success() - name: rts-build - uses: ./.github/workflows/rts-build.yml - secrets: inherit - with: - pr: ${{ github.event.client_payload.pull_request.number }} - - test-appsmithctl: - needs: [tag-check] - if: success() - name: appsmithctl - uses: ./.github/workflows/appsmithctl.yml - secrets: inherit - with: - pr: ${{ github.event.client_payload.pull_request.number }} - - build-docker-image: - needs: [tag-check, client-build, server-build, rts-build] - # Only run if the build step is successful - if: success() - name: build-docker-image - uses: ./.github/workflows/build-docker-image.yml - secrets: inherit - with: - pr: ${{ github.event.client_payload.pull_request.number }} - - ci-test: - needs: [tag-check, build-docker-image] - # Only run if the build step is successful - if: success() - name: ci-test - uses: ./.github/workflows/ci-test-custom-script.yml - secrets: inherit - with: - pr: ${{ github.event.client_payload.pull_request.number }} - tags: ${{ needs.tag-check.outputs.tags }} - matrix: ${{ needs.tag-check.outputs.matrix }} - - - ci-test-result: - needs: [ci-test] - # Only run if the ci-test with matrices step is successful - if: always() - runs-on: ubuntu-latest - defaults: - run: - shell: bash - steps: - - name: Dump the client payload context - env: - PAYLOAD_CONTEXT: ${{ toJson(github.event.client_payload) }} - run: echo "$PAYLOAD_CONTEXT" - - - name: Setup node - if: needs.ci-test.result != 'success' - uses: actions/setup-node@v3 - with: - node-version: 18 - - - name: install pg - if: needs.ci-test.result != 'success' - run: npm install pg - - - name: Fetch the failed specs - if: needs.ci-test.result != 'success' - id: failed_specs - env: - DB_HOST: ${{ secrets.CYPRESS_DB_HOST }} - DB_NAME: ${{ secrets.CYPRESS_DB_NAME }} - DB_USER: ${{ secrets.CYPRESS_DB_USER }} - DB_PWD: ${{ secrets.CYPRESS_DB_PWD }} - RUN_ID: ${{ github.run_id }} - ATTEMPT_NUMBER: ${{ github.run_attempt }} - uses: actions/github-script@v6 - with: - script: | - const { Pool } = require("pg"); - const { DB_HOST, DB_NAME, DB_USER, DB_PWD, RUN_ID, ATTEMPT_NUMBER } = process.env - - const client = await new Pool({ - user: DB_USER, - host: DB_HOST, - database: DB_NAME, - password: DB_PWD, - port: 5432, - connectionTimeoutMillis: 60000, - }).connect(); - - const result = await client.query( - `SELECT DISTINCT name FROM public."specs" - WHERE "matrixId" IN - (SELECT id FROM public."matrix" - WHERE "attemptId" = ( - SELECT id FROM public."attempt" WHERE "workflowId" = $1 and "attempt" = $2 - ) - ) AND status = 'fail'`, - [RUN_ID, ATTEMPT_NUMBER], - ); - client.release(); - return result.rows.map((spec) => spec.name); - - # In case for any ci job failure, create combined failed spec - - name: combine all specs for CI - id: combine_ci - if: needs.ci-test.result != 'success' - run: | - failed_specs=$(echo ${{steps.failed_specs.outputs.result}} | sed 's/\[\|\]//g' | tr -d ' ' | tr ',' '\n') - while read -r line; do - echo "$line" >> ~/combined_failed_spec_ci - done <<< "$failed_specs" - if [[ -z $(grep '[^[:space:]]' ~/combined_failed_spec_ci) ]] ; then - echo "specs_failed=0" >> $GITHUB_OUTPUT - else - echo "specs_failed=1" >> $GITHUB_OUTPUT - fi - - # Upload combined failed CI spec list to a file - # This is done for debugging. - - name: upload combined failed spec - if: needs.ci-test.result != 'success' - uses: actions/upload-artifact@v3 - with: - name: combined_failed_spec_ci - path: ~/combined_failed_spec_ci - - - name: Get Latest flaky Tests - shell: bash - run: | - curl --request POST --url https://yatin-s-workspace-jk8ru5.us-east-1.xata.sh/db/CypressKnownFailures:main/tables/CypressKnownFailuires/query --header 'Authorization: Bearer ${{ secrets.XATA_TOKEN }}' --header 'Content-Type: application/json'|jq -r |grep Spec|cut -d ':' -f 2 2> /dev/null|sed 's/"//g'|sed 's/,//g' > ~/knownfailures - - # Verify CI test failures against known failures - - name: Verify CI test failures against known failures - if: needs.ci-test.result != 'success' - shell: bash - run: | - new_failed_spec_env="
    $(comm -1 -3 <(sort ~/knownfailures) <(sort -u ~/combined_failed_spec_ci) | sed 's/|cypress|cypress/\n/g' | sed 's/^/
  1. /')
" - echo "$new_failed_spec_env" - echo "new_failed_spec_env<> $GITHUB_ENV - echo "$new_failed_spec_env" >> $GITHUB_ENV - echo "EOF" >> $GITHUB_ENV - - - name: Add a comment on the PR with new CI failures - if: needs.ci-test.result != 'success' && steps.combine_ci.outputs.specs_failed == '1' - uses: peter-evans/create-or-update-comment@v3 - with: - issue-number: ${{ github.event.client_payload.pull_request.number }} - body: | - Workflow run: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}>. - Commit: `${{ github.event.client_payload.slash_command.args.named.sha }}`. - Cypress dashboard: Click here! - The following are new failures, please fix them before merging the PR: ${{env.new_failed_spec_env}} - To know the list of identified flaky tests - Refer here - - - name: Add a comment on the PR when ci-test is failed but no specs found - if: needs.ci-test.result != 'success' && steps.combine_ci.outputs.specs_failed == '0' - uses: peter-evans/create-or-update-comment@v3 - with: - issue-number: ${{ github.event.client_payload.pull_request.number }} - body: | - Workflow run: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}>. - Commit: `${{ github.event.client_payload.slash_command.args.named.sha }}`. - Cypress dashboard url: Click here! - It seems like **no tests ran** 😔. We are not able to recognize it, please check workflow here. - - - name: Add a comment on the PR when ci-test is success - if: needs.ci-test.result == 'success' && steps.combine_ci.outputs.specs_failed == '0' - uses: peter-evans/create-or-update-comment@v3 - with: - issue-number: ${{ github.event.client_payload.pull_request.number }} - body: | - Workflow run: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}>. - Commit: `${{ github.event.client_payload.slash_command.args.named.sha }}`. - Cypress dashboard url: Click here! - All cypress tests have passed 🎉🎉🎉 - - # Update check run called "ci-test-result" - - name: Mark ci-test-result job as complete - uses: actions/github-script@v6 - id: update-check-run - if: ${{ always() }} - env: - run_id: ${{ github.run_id }} - repository: ${{ github.repository }} - number: ${{ github.event.client_payload.pull_request.number }} - job: ${{ github.job }} - # Conveniently, job.status maps to https://developer.github.com/v3/checks/runs/#update-a-check-run - conclusion: ${{ job.status }} - matrix_result: ${{ toJson(needs.ci-test) }} - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const { data: pull } = await github.rest.pulls.get({ - ...context.repo, - pull_number: process.env.number - }); - const ref = pull.head.sha; - - const { data: checks } = await github.rest.checks.listForRef({ - ...context.repo, - ref - }); - - try { - console.log("Going to create the check run: ci-test-result"); - const head_sha = pull.head.sha; - const { data: completed_at } = await github.rest.checks.create({ - owner: context.repo.owner, - repo: context.repo.repo, - head_sha: head_sha, - name: process.env.job, - status: 'completed', - conclusion: JSON.parse(process.env.matrix_result).result, - output: { - title: "Integration tests result for ok to test", - summary: "https://github.com/" + process.env.repository + "/actions/runs/" + process.env.run_id - } - }); - console.log({ completed_at }); - return completed_at; - } catch(e) { - console.error("Error while creating the check run: ci-test-result"); - console.error({ error: e.message }); - } - - - name: Dump the client payload context - env: - PAYLOAD_CONTEXT: ${{ toJson(github.event.client_payload) }} - run: echo "$PAYLOAD_CONTEXT" - - - name: Check ci-test set status - if: needs.ci-test.result != 'success' - run: exit 1 - - package: - needs: [ci-test] - runs-on: ubuntu-latest - defaults: - run: - working-directory: app/client - # Run this job only if all the previous steps are a success and the reference if the release or master branch - if: success() && (github.ref == 'refs/heads/release' || github.ref == 'refs/heads/master') - - steps: - # Update check run called "package" - - name: Mark package job as complete - uses: actions/github-script@v6 - id: update-check-run - if: ${{ always() }} - env: - run_id: ${{ github.run_id }} - repository: ${{ github.repository }} - number: ${{ github.event.client_payload.pull_request.number }} - job: ${{ github.job }} - # Conveniently, job.status maps to https://developer.github.com/v3/checks/runs/#update-a-check-run - conclusion: ${{ job.status }} - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const { data: pull } = await github.rest.pulls.get({ - ...context.repo, - pull_number: process.env.number - }); - const ref = pull.head.sha; - - const { data: checks } = await github.rest.checks.listForRef({ - ...context.repo, - ref - }); - - const check = checks.check_runs.filter(c => c.name === process.env.job); - - if(check.length == 0) { - const head_sha = pull.head.sha; - const { data: completed_at } = await github.rest.checks.create({ - owner: context.repo.owner, - repo: context.repo.repo, - head_sha: head_sha, - name: process.env.job, - status: 'completed', - conclusion: process.env.conclusion, - output: { - title: "Package result for ok to test", - summary: "https://github.com/" + process.env.repository + "/actions/runs/" + process.env.run_id - } - }); - - return completed_at; - } else { - const { data: result } = await github.rest.checks.update({ - ...context.repo, - check_run_id: check[0].id, - status: 'completed', - conclusion: process.env.conclusion, - output: { - title: "Package result for ok to test", - summary: "https://github.com/" + process.env.repository + "/actions/runs/" + process.env.run_id - } - }); - - return result; - } + diff --git a/.github/workflows/pr-cypress.yml b/.github/workflows/pr-cypress.yml index 34215ec8b7..92ff590915 100644 --- a/.github/workflows/pr-cypress.yml +++ b/.github/workflows/pr-cypress.yml @@ -164,7 +164,7 @@ jobs: if: needs.ci-test.result != 'success' shell: bash run: | - new_failed_spec_env="
    \n$(comm -1 -3 <(sort ~/knownfailures) <(sort -u ~/combined_failed_spec_ci) | sed 's/|cypress|cypress/\n/g' | sed 's/^/>
  1. /')
" + new_failed_spec_env="$(comm -1 -3 <(sort ~/knownfailures) <(sort -u ~/combined_failed_spec_ci) | sed 's/|cypress|cypress/\n/g' | sed 's/^/>
  • /')" echo "$new_failed_spec_env" echo "new_failed_spec_env<> $GITHUB_ENV echo "$new_failed_spec_env" >> $GITHUB_ENV @@ -180,7 +180,8 @@ jobs: > Workflow run: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}> > Commit: `${{ github.event.pull_request.head.sha }}` > Cypress dashboard: Click here! - > The following are new failures, please fix them before merging the PR: ${{env.new_failed_spec_env}} + > The following are new failures, please fix them before merging the PR:
      + ${{env.new_failed_spec_env}}
    > To know the list of identified flaky tests - Refer here