From 876cd428f94926f7701e1e2e8366a7ea1589b8af Mon Sep 17 00:00:00 2001 From: yatinappsmith <84702014+yatinappsmith@users.noreply.github.com> Date: Mon, 19 Dec 2022 16:23:39 +0530 Subject: [PATCH] ci: Comment failing tests to PR (#18450) ## Description Comment failing tests to PR Fixes # ## Type of change - New feature (non-breaking change which adds functionality) ## How Has This Been Tested? Code review. Tested in local fork ### Test Plan > Add Testsmith test cases links that relate to this PR ### Issues raised during DP testing > Link issues raised during DP testing for better visiblity and tracking (copy link from comments dropped on this PR) ## Checklist: ### Dev activity - [X] My code follows the style guidelines of this project - [X] I have performed a self-review of my own code - [X] I have commented my code, particularly in hard-to-understand areas - [X] I have made corresponding changes to the documentation - [X] My changes generate no new warnings - [X] I have added tests that prove my fix is effective or that my feature works - [X] New and existing unit tests pass locally with my changes - [X] PR is being merged under a feature flag ### QA activity: - [ ] Test plan has been approved by relevant developers - [ ] Test plan has been peer reviewed by QA - [ ] Cypress test cases have been added and approved by either SDET or manual QA - [ ] Organized project review call with relevant stakeholders after Round 1/2 of QA - [ ] Added Test Plan Approved label after reveiwing all Cypress test --- .../workflows/CypressAddKnownfailedtests.yml | 52 +++++++++++++++++++ .../workflows/integration-tests-command.yml | 43 +++++++++++++++ app/client/cypress/xataadd.sh | 3 ++ app/client/cypress/xatadel.sh | 9 ++++ 4 files changed, 107 insertions(+) create mode 100644 .github/workflows/CypressAddKnownfailedtests.yml create mode 100644 app/client/cypress/xataadd.sh create mode 100644 app/client/cypress/xatadel.sh diff --git a/.github/workflows/CypressAddKnownfailedtests.yml b/.github/workflows/CypressAddKnownfailedtests.yml new file mode 100644 index 0000000000..dae6c4a4b1 --- /dev/null +++ b/.github/workflows/CypressAddKnownfailedtests.yml @@ -0,0 +1,52 @@ +name: Cypress Add Known failed tests + +on: + issues: + types: [opened, closed] + +jobs: + IssueOpened: + if: github.event.action == 'opened' && contains( github.event.issue.labels.*.name, 'cypress_failed_test') + runs-on: ubuntu-latest + permissions: + issues: write + steps: + - name: Checkout the merged commit from PR and base branch + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: View issue information + env: + GITHUB_CONTEXT: ${{ toJson(github) }} + XATATOKEN: ${{ secrets.XATA_TOKEN }} + run: | + echo "Issue title: ${{ github.event.issue.title }}" + echo "Issue body: ${{ github.event.issue.body }}" + #echo "$GITHUB_CONTEXT" + chmod a+x app/client/cypress/xataadd.sh + echo "${{ github.event.issue.title }}"|awk '{print $2}' + echo "${{ github.event.issue.title }}"|awk '{print $2}'|xargs app/client/cypress/xataadd.sh + + IssueClosed: + if: github.event.action == 'closed' && contains( github.event.issue.labels.*.name, 'cypress_failed_test') + runs-on: ubuntu-latest + permissions: + issues: write + steps: + - name: Checkout the merged commit from PR and base branch + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: View issue information + env: + GITHUB_CONTEXT: ${{ toJson(github) }} + XATATOKEN: ${{ secrets.XATA_TOKEN }} + run: | + echo "Issue title: ${{ github.event.issue.title }}" + echo "Issue body: ${{ github.event.issue.body }}" + #echo "$GITHUB_CONTEXT" + chmod a+x app/client/cypress/xatadel.sh + echo "${{ github.event.issue.title }}"|awk '{print $2}'|xargs app/client/cypress/xatadel.sh + + + diff --git a/.github/workflows/integration-tests-command.yml b/.github/workflows/integration-tests-command.yml index d5b8971ffc..f82d8654f2 100644 --- a/.github/workflows/integration-tests-command.yml +++ b/.github/workflows/integration-tests-command.yml @@ -917,6 +917,49 @@ jobs: with: name: combined_failed_spec_fat path: ~/combined_failed_spec_fat + + - name: Get Latest flaky Tests + shell: bash + run: | + curl --request POST --url https://yatin-s-workspace-jk8ru5.us-east-1.xata.sh/db/CypressKnownFailures:main/tables/CypressKnownFailuires/query --header 'Authorization: Bearer ${{ secrets.XATA_TOKEN }}' --header 'Content-Type: application/json'|jq -r |grep Spec|cut -d ':' -f 2 2> /dev/null|sed 's/"//g'|sed 's/,//g' > ~/knownfailures + + # Verify non-fat test failures against known failures + - name: Verify non-fat test failures against known failures + if: needs.ui-test.result != 'success' + shell: bash + run: | + new_failed_spec_env=$(comm -1 -3 <(sort ~/knownfailures) <(sort ~/combined_failed_spec) | sed 's/|cypress|cypress/\n/g') + echo $new_failed_spec_env + echo "new_failed_spec_env<> $GITHUB_ENV + echo "$new_failed_spec_env" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + + - name: Add a comment on the PR with new non fat failures + if: needs.ui-test.result != 'success' + uses: peter-evans/create-or-update-comment@v1 + with: + issue-number: ${{ github.event.issue.number }} + body: | + The following are new failures, please fix them before merging the PR ${{ env.new_failed_spec_env }} + + # Verify fat test failures against known failures + - name: Verify fat test failures against known failures + if: needs.fat-container-test.result != 'success' + shell: bash + run: | + new_failed_spec_env=$(comm -1 -3 <(sort ./release/app/client/cypress/knownfailures) <(sort ~/combined_failed_spec_fat) | sed 's/|cypress|cypress/\n/g') + echo $new_failed_spec_env + echo "new_failed_spec_env<> $GITHUB_ENV + echo "$new_failed_spec_env" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + + - name: Add a comment on the PR with new fat failures + if: needs.fat-container-test.result != 'success' + uses: peter-evans/create-or-update-comment@v1 + with: + issue-number: ${{ github.event.issue.number }} + body: | + The following are new failures, please fix them before merging the PR ${{ env.new_failed_spec_env }} # Update check run called "ui-test-result" - name: Mark ui-test-result job as complete diff --git a/app/client/cypress/xataadd.sh b/app/client/cypress/xataadd.sh new file mode 100644 index 0000000000..d45d40bc34 --- /dev/null +++ b/app/client/cypress/xataadd.sh @@ -0,0 +1,3 @@ +#!/bin/bash -x + +curl --request POST --url 'https://yatin-s-workspace-jk8ru5.us-east-1.xata.sh/db/CypressKnownFailures:main/tables/CypressKnownFailuires/data?columns=id' --header "Authorization: Bearer $XATATOKEN" --header 'Content-Type: application/json' --data "{\"Spec\":\"$1\"}" diff --git a/app/client/cypress/xatadel.sh b/app/client/cypress/xatadel.sh new file mode 100644 index 0000000000..aba9709588 --- /dev/null +++ b/app/client/cypress/xatadel.sh @@ -0,0 +1,9 @@ +#!/bin/bash -x + +idlist=$(curl --request POST --url https://yatin-s-workspace-jk8ru5.us-east-1.xata.sh/db/CypressKnownFailures:main/tables/CypressKnownFailuires/query --header "Authorization: Bearer $XATATOKEN" --header 'Content-Type: application/json'|jq -r ".[] | .[] | select(.Spec==\"$1\") | .id") + +while IFS= read -r line; do + echo "... $line ..." + curl --request DELETE --url "https://yatin-s-workspace-jk8ru5.us-east-1.xata.sh/db/CypressKnownFailures:main/tables/CypressKnownFailuires/data/$line?columns=id" --header "Authorization: Bearer $XATATOKEN" + +done <<< "$idlist"