ci: Add command to run ci-test-limited (#25180)

## Description
- Add command to run ci-test-limited
#### PR fixes following issue(s)
Fixes # (issue number)
> if no issue exists, please create an issue and ask the maintainers
about this first
>
>
#### Media
> A video or a GIF is preferred. when using Loom, don’t embed because it
looks like it’s a GIF. instead, just link to the video
>
>
#### Type of change
> Please delete options that are not relevant.
- Bug fix (non-breaking change which fixes an issue)
- New feature (non-breaking change which adds functionality)
- Breaking change (fix or feature that would cause existing
functionality to not work as expected)
- Chore (housekeeping or task changes that don't impact user perception)
- This change requires a documentation update
>
>
>
## Testing
>
#### How Has This Been Tested?
> Please describe the tests that you ran to verify your changes. Also
list any relevant details for your test configuration.
> Delete anything that is not relevant
- [ ] Manual
- [ ] Jest
- [ ] Cypress
>
>
#### Test Plan
> Add Testsmith test cases links that relate to this PR
>
>
#### Issues raised during DP testing
> Link issues raised during DP testing for better visiblity and tracking
(copy link from comments dropped on this PR)
>
>
>
## Checklist:
#### Dev activity
- [ ] My code follows the style guidelines of this project
- [ ] I have performed a self-review of my own code
- [ ] I have commented my code, particularly in hard-to-understand areas
- [ ] I have made corresponding changes to the documentation
- [ ] My changes generate no new warnings
- [ ] I have added tests that prove my fix is effective or that my
feature works
- [ ] New and existing unit tests pass locally with my changes
- [ ] PR is being merged under a feature flag


#### QA activity:
- [ ] [Speedbreak
features](https://github.com/appsmithorg/TestSmith/wiki/Guidelines-for-test-plans#speedbreakers-)
have been covered
- [ ] Test plan covers all impacted features and [areas of
interest](https://github.com/appsmithorg/TestSmith/wiki/Guidelines-for-test-plans#areas-of-interest-)
- [ ] Test plan has been peer reviewed by project stakeholders and other
QA members
- [ ] Manually tested functionality on DP
- [ ] We had an implementation alignment call with stakeholders post QA
Round 2
- [ ] Cypress test cases have been added and approved by SDET/manual QA
- [ ] Added `Test Plan Approved` label after Cypress tests were reviewed
- [ ] Added `Test Plan Approved` label after JUnit tests were reviewed
This commit is contained in:
Saroj 2023-07-07 12:26:55 +05:30 committed by GitHub
parent 9304a568d5
commit 50240fc74b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 329 additions and 26 deletions

View File

@ -1,6 +1,8 @@
name: Build Client, Server & Run only Cypress
on:
repository_dispatch:
types: [ ci-test-limit-command ]
# This workflow can be triggered manually from the GitHub Actions page
workflow_dispatch:
inputs:
@ -17,11 +19,42 @@ jobs:
outputs:
non_ts_files: ${{ steps.check_files.outputs.non_ts_files }}
non_ts_files_count: ${{ steps.check_files.outputs.non_ts_files_count }}
pr: ${{steps.args.outputs.pr}}
runId: ${{steps.args.outputs.runId}}
steps:
- name: Checkout the head commit of the branch
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Get the PR number if workflow is triggered manually
id: fetch_pr
if: github.event_name == 'workflow_dispatch'
run: |
res=`curl -s -H "Authorization: Bearer ${{ secrets.APPSMITH_CI_TEST_PAT}}" https://api.github.com/repos/${{ github.repository }}/pulls?head=appsmithorg:${{ github.ref_name }}`
response_length=`echo "$res" | jq -r 'length'`
if [[ $response_length -ne 0 ]]; then
pr_number=`echo $res | jq -r '.[0] | .number'`
echo "pr=$pr_number" >> $GITHUB_OUTPUT
else
echo "pr=0" >> $GITHUB_OUTPUT
fi
- name: Set args
id: args
run: |
if [[ ${{github.event_name}} == 'repository_dispatch' ]]; then
echo "pr=${{ github.event.client_payload.pull_request.number }}" >> $GITHUB_OUTPUT
checkArg=`echo '${{toJSON(github.event.client_payload.slash_command.args.named)}}' | jq 'has("runId")'`
if [[ $checkArg == 'true' ]]; then
echo "runId=${{ github.event.client_payload.slash_command.args.named.runId }}" >> $GITHUB_OUTPUT
else
echo "runId=0" >> $GITHUB_OUTPUT
fi
else
echo "runId=${{ inputs.previous_run_id }}" >> $GITHUB_OUTPUT
echo "pr=${{ steps.fetch_pr.outputs.pr }}" >> $GITHUB_OUTPUT
fi
- name: Get the diff from base branch
continue-on-error: true
@ -39,74 +72,293 @@ jobs:
non_ts_files=()
for file in "${files[@]}"; do
if [[ $file != *.ts ]]; then
non_ts_files+=("$file")
non_ts_files+=("<li> $file")
fi
done
echo "non_ts_files=${non_ts_files[@]}" >> $GITHUB_OUTPUT
echo "non_ts_files_count=${#non_ts_files[@]}" >> $GITHUB_OUTPUT
- name: Print the files
if: steps.check_files.outputs.non_ts_files_count != 0
if: steps.check_files.outputs.non_ts_files_count != 0 && steps.args.outputs.pr == '0'
run: |
echo "${{ steps.check_files.outputs.non_ts_files }}"
exit 1
- name: Comment the filenames if PR is there
if: steps.check_files.outputs.non_ts_files_count != 0 && steps.args.outputs.pr != '0'
uses: peter-evans/create-or-update-comment@v1
with:
issue-number: ${{ fromJson(steps.args.outputs.pr) }}
body: |
<b>Below new test files are written in js 🔴 </b>
<b>Expected format ts. Please fix and retrigger ci-test-limit:</b>
<ol>${{ steps.check_files.outputs.non_ts_files }}</ol>
- if: steps.check_files.outputs.non_ts_files_count != 0
run: exit 1
- name: Add a comment on the PR with link to workflow run
if: success() && steps.args.outputs.pr != '0'
uses: peter-evans/create-or-update-comment@v2
with:
issue-number: ${{ fromJson(steps.args.outputs.pr) }}
body: |
Tests running at: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}>.
Workflow: `ci-test-limited`.
PR: ${{ fromJson(steps.args.outputs.pr) }}.
server-build:
name: server-build
needs: [file-check]
if: success() && inputs.previous_run_id == '0'
if: success() && needs.file-check.outputs.runId == '0'
uses: ./.github/workflows/server-build.yml
secrets: inherit
with:
pr: 0
pr: ${{fromJson(needs.file-check.outputs.pr)}}
skip-tests: "true"
client-build:
name: client-build
needs: [file-check]
if: success() && inputs.previous_run_id == '0'
if: success() && needs.file-check.outputs.runId == '0'
uses: ./.github/workflows/client-build.yml
secrets: inherit
with:
pr: 0
pr: ${{fromJson(needs.file-check.outputs.pr)}}
skip-tests: "true"
rts-build:
name: rts-build
needs: [file-check]
if: success() && inputs.previous_run_id == '0'
if: success() && needs.file-check.outputs.runId == '0'
uses: ./.github/workflows/rts-build.yml
secrets: inherit
with:
pr: 0
pr: ${{fromJson(needs.file-check.outputs.pr)}}
build-docker-image:
needs: [ client-build, server-build, rts-build ]
needs: [ file-check, client-build, server-build, rts-build ]
# Only run if the build step is successful
if: success() && inputs.previous_run_id == '0'
if: success() && needs.file-check.outputs.runId == '0'
name: build-docker-image
uses: ./.github/workflows/build-docker-image.yml
secrets: inherit
with:
pr: 0
pr: ${{fromJson(needs.file-check.outputs.pr)}}
ci-test-limited:
needs: [ build-docker-image ]
needs: [ file-check, build-docker-image ]
# Only run if the build step is successful
if: success() && inputs.previous_run_id == '0'
if: success() && needs.file-check.outputs.runId == '0'
name: ci-test-limited
uses: ./.github/workflows/ci-test-limited.yml
secrets: inherit
with:
pr: 0
pr: ${{fromJson(needs.file-check.outputs.pr)}}
ci-test-limited-existing-docker-image:
needs: [file-check]
# Only run if the previous run-id is provided
if: success() && inputs.previous_run_id != '0'
if: success() && needs.file-check.outputs.runId != '0'
name: ci-test-limited-existing-image
uses: ./.github/workflows/ci-test-limited.yml
secrets: inherit
with:
pr: 0
previous-workflow-run-id: ${{ fromJson(inputs.previous_run_id) }}
pr: ${{fromJson(needs.file-check.outputs.pr)}}
previous-workflow-run-id: ${{ fromJson(needs.file-check.outputs.runId) }}
ci-test-result:
needs: [ file-check, ci-test-limited ]
# Only run if the ci-test-limited with matrices step is successful
if: always()
runs-on: ubuntu-latest
defaults:
run:
shell: bash
steps:
# Deleting the existing dir's if any
- name: Delete existing directories
if: needs.ci-test-limited.result != 'success'
run: |
rm -f ~/failed_spec_ci
rm -f ~/combined_failed_spec_ci
# Force store previous cypress dashboard url from cache
- name: Store the previous cypress dashboard url
if: success()
uses: actions/cache@v3
with:
path: |
~/cypress_url
key: ${{ github.run_id }}-dashboard-url-${{ github.run_attempt }}
restore-keys: |
${{ github.run_id }}-dashboard-url
- name: Print cypress dashboard url
id: dashboard_url
run: |
cypress_url=$(cat ~/cypress_url)
echo "dashboard_url=$cypress_url" >> $GITHUB_OUTPUT
# Download failed_spec list for all jobs
- uses: actions/download-artifact@v3
if: needs.ci-test-limited.result != 'success'
id: download_ci
with:
name: failed-spec-ci
path: ~/failed_spec_ci
# In case for any ci job failure, create combined failed spec
- name: "combine all specs for CI"
if: needs.ci-test-limited.result != 'success'
run: |
echo "Debugging: failed specs in ~/failed_spec_ci/failed_spec_ci*"
cat ~/failed_spec_ci/failed_spec_ci*
cat ~/failed_spec_ci/failed_spec_ci* | sort -u >> ~/combined_failed_spec_ci
# Upload combined failed CI spec list to a file
# This is done for debugging.
- name: upload combined failed spec
if: needs.ci-test-limited.result != 'success'
uses: actions/upload-artifact@v3
with:
name: combined_failed_spec_ci
path: ~/combined_failed_spec_ci
- name: Get Latest flaky Tests
shell: bash
run: |
curl --request POST --url https://yatin-s-workspace-jk8ru5.us-east-1.xata.sh/db/CypressKnownFailures:main/tables/CypressKnownFailuires/query --header 'Authorization: Bearer ${{ secrets.XATA_TOKEN }}' --header 'Content-Type: application/json'|jq -r |grep Spec|cut -d ':' -f 2 2> /dev/null|sed 's/"//g'|sed 's/,//g' > ~/knownfailures
# Verify CI test failures against known failures
- name: Verify CI test failures against known failures
if: needs.ci-test-limited.result != 'success'
shell: bash
run: |
new_failed_spec_env="<ol>$(comm -1 -3 <(sort ~/knownfailures) <(sort -u ~/combined_failed_spec_ci) | sed 's/|cypress|cypress/\n/g' | sed 's/^/<li>/')</ol>"
echo "$new_failed_spec_env"
echo "new_failed_spec_env<<EOF" >> $GITHUB_ENV
echo "$new_failed_spec_env" >> $GITHUB_ENV
echo "EOF" >> $GITHUB_ENV
- name: Add a comment on the PR with new CI failures
if: needs.ci-test-limited.result != 'success'
uses: peter-evans/create-or-update-comment@v1
with:
issue-number: ${{fromJson(needs.file-check.outputs.pr)}}
body: |
Workflow run: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}>.
Cypress dashboard: <a href="${{ steps.dashboard_url.outputs.dashboard_url }}" target="_blank"> Click here!</a>
The following are new failures, please fix them before merging the PR: ${{env.new_failed_spec_env}}
To know the list of identified flaky tests - <a href="https://app.appsmith.com/applications/613868bedd7786286ddd4a6a/pages/63ec710e8e503f763651791a" target="_blank">Refer here</a>
- name: Add a comment on the PR when ci-test-limited is success
if: needs.ci-test-limited.result == 'success'
uses: peter-evans/create-or-update-comment@v1
with:
issue-number: ${{fromJson(needs.file-check.outputs.pr)}}
body: |
Workflow run: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}>.
Cypress dashboard url: <a href="${{ steps.dashboard_url.outputs.dashboard_url }}" target="_blank">Click here!</a>
All cypress tests have passed 🎉🎉🎉
- name: Check ci-test-limited set status
if: needs.ci-test-limited.result != 'success'
run: exit 1
ci-test-result-existing:
needs: [ file-check, ci-test-limited-existing-docker-image ]
# Only run if the ci-test-limited with matrices step is successful
if: needs.file-check.outputs.runId != '0'
runs-on: ubuntu-latest
defaults:
run:
shell: bash
steps:
# Deleting the existing dir's if any
- name: Delete existing directories
if: needs.ci-test-limited-existing-docker-image.result != 'success'
run: |
rm -f ~/failed_spec_ci
rm -f ~/combined_failed_spec_ci
# Force store previous cypress dashboard url from cache
- name: Store the previous cypress dashboard url
if: success()
uses: actions/cache@v3
with:
path: |
~/cypress_url
key: ${{ github.run_id }}-dashboard-url-${{ github.run_attempt }}
restore-keys: |
${{ github.run_id }}-dashboard-url
- name: Print cypress dashboard url
id: dashboard_url
run: |
cypress_url=$(cat ~/cypress_url)
echo "dashboard_url=$cypress_url" >> $GITHUB_OUTPUT
# Download failed_spec list for all jobs
- uses: actions/download-artifact@v3
if: needs.ci-test-limited-existing-docker-image.result != 'success'
id: download_ci
with:
name: failed-spec-ci
path: ~/failed_spec_ci
# In case for any ci job failure, create combined failed spec
- name: "combine all specs for CI"
if: needs.ci-test-limited-existing-docker-image.result != 'success'
run: |
echo "Debugging: failed specs in ~/failed_spec_ci/failed_spec_ci*"
cat ~/failed_spec_ci/failed_spec_ci*
cat ~/failed_spec_ci/failed_spec_ci* | sort -u >> ~/combined_failed_spec_ci
# Upload combined failed CI spec list to a file
# This is done for debugging.
- name: upload combined failed spec
if: needs.ci-test-limited-existing-docker-image.result != 'success'
uses: actions/upload-artifact@v3
with:
name: combined_failed_spec_ci
path: ~/combined_failed_spec_ci
- name: Get Latest flaky Tests
shell: bash
run: |
curl --request POST --url https://yatin-s-workspace-jk8ru5.us-east-1.xata.sh/db/CypressKnownFailures:main/tables/CypressKnownFailuires/query --header 'Authorization: Bearer ${{ secrets.XATA_TOKEN }}' --header 'Content-Type: application/json'|jq -r |grep Spec|cut -d ':' -f 2 2> /dev/null|sed 's/"//g'|sed 's/,//g' > ~/knownfailures
# Verify CI test failures against known failures
- name: Verify CI test failures against known failures
if: needs.ci-test-limited-existing-docker-image.result != 'success'
shell: bash
run: |
new_failed_spec_env="<ol>$(comm -1 -3 <(sort ~/knownfailures) <(sort -u ~/combined_failed_spec_ci) | sed 's/|cypress|cypress/\n/g' | sed 's/^/<li>/')</ol>"
echo "$new_failed_spec_env"
echo "new_failed_spec_env<<EOF" >> $GITHUB_ENV
echo "$new_failed_spec_env" >> $GITHUB_ENV
echo "EOF" >> $GITHUB_ENV
- name: Add a comment on the PR with new CI failures
if: needs.ci-test-limited-existing-docker-image.result != 'success' && needs.file-check.outputs.pr != '0'
uses: peter-evans/create-or-update-comment@v1
with:
issue-number: ${{fromJson(needs.file-check.outputs.pr)}}
body: |
Workflow run: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}>.
Cypress dashboard: <a href="${{ steps.dashboard_url.outputs.dashboard_url }}" target="_blank"> Click here!</a>
The following are new failures, please fix them before merging the PR: ${{env.new_failed_spec_env}}
To know the list of identified flaky tests - <a href="https://app.appsmith.com/applications/613868bedd7786286ddd4a6a/pages/63ec710e8e503f763651791a" target="_blank">Refer here</a>
- name: Add a comment on the PR when ci-test-limited-existing-docker-image is success
if: needs.ci-test-limited-existing-docker-image.result == 'success' && needs.file-check.outputs.pr != '0'
uses: peter-evans/create-or-update-comment@v1
with:
issue-number: ${{fromJson(needs.file-check.outputs.pr)}}
body: |
Workflow run: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}>.
Cypress dashboard url: <a href="${{ steps.dashboard_url.outputs.dashboard_url }}" target="_blank">Click here!</a>
All cypress tests have passed 🎉🎉🎉
- name: Check ci-test-limited-existing-docker-image set status
if: needs.ci-test-limited-existing-docker-image.result != 'success'
run: exit 1

View File

@ -0,0 +1,22 @@
# If someone with write access comments "/ci-test-limit" on a pull request, emit a repository_dispatch event
name: CI Test Limited Command
on:
issue_comment:
types: [created]
jobs:
ci-test-limit:
runs-on: ubuntu-latest
# Only run for PRs, not issue comments
if: |
github.event.issue.pull_request
steps:
- name: Slash Command Dispatch
uses: peter-evans/slash-command-dispatch@v3
with:
issue-type: pull-request
token: ${{ secrets.APPSMITH_CI_TEST_PAT }}
commands: |
ci-test-limit

View File

@ -87,20 +87,28 @@ jobs:
run_result_env=$(cat ~/run_result)
echo "::set-output name=run_result::$run_result_env"
# Download failed_spec list for all jobs in case of rerun
- uses: actions/download-artifact@v3
# In case this is second attempt try restoring failed tests
- name: Restore the previous failed combine result
if: steps.run_result.outputs.run_result == 'failedtest'
uses: actions/download-artifact@v3
with:
name: failed-spec-ci
path: ~/failed_spec_ci
name: combined_failed_spec_ci
path: ~/combined_failed_spec_ci
# In case of rerun combine all the failed specs and set it in env
- name: combine all specs for CI in case of rerun
# failed_spec_env will contain list of all failed specs
# We are using environment variable instead of regular to support multiline
- name: Get failed_spec
id: failed_spec
if: steps.run_result.outputs.run_result == 'failedtest'
working-directory: app/client
run: |
failed_spec_env=$(cat ~/failed_spec_ci/failed_spec_ci* | sort -u)
echo "failed_spec_env<<EOF" >> $GITHUB_ENV
echo "$failed_spec_env" >> $GITHUB_ENV
while IFS= read -r line
do
spec_name=$(echo $line | awk -F'/' '{print $NF}')
failed_spec_env=$(find . -name $spec_name | sed 's|./||')
echo "$failed_spec_env" >> $GITHUB_ENV
done < ~/combined_failed_spec_ci/combined_failed_spec_ci
echo "EOF" >> $GITHUB_ENV
# Get specs to run
@ -270,6 +278,7 @@ jobs:
echo Remote $COMMIT_INFO_REMOTE
- name: Run the cypress test
id: cypress_test
if: steps.run_result.outputs.run_result != 'success' && steps.run_result.outputs.run_result != 'failedtest'
uses: cypress-io/github-action@v5
env:
@ -331,6 +340,7 @@ jobs:
# In case of second attempt only run failed specs
- name: Run the cypress test with failed tests
id: cypress_test_failedtest
if: steps.run_result.outputs.run_result == 'failedtest'
uses: cypress-io/github-action@v5
env:
@ -460,6 +470,25 @@ jobs:
name: server-logs-${{ matrix.job }}
path: app/server/server-logs.log
- name: get cypress url dashboard url
id: dashboard_url
if: always()
run: |
if [[ "${{steps.run_result.outputs.run_result }}" != "success" && "${{steps.run_result.outputs.run_result }}" != "failedtest" ]]; then
echo ${{ steps.cypress_test.outputs.resultsUrl }} >> ~/cypress_url
elif [[ "${{steps.run_result.outputs.run_result }}" == "failedtest" ]]; then
echo ${{ steps.cypress_test_failedtest.outputs.resultsUrl }} >> ~/cypress_url
fi
# Force store previous run result to cache
- name: Store the previous run result
if: success()
uses: actions/cache/save@v3
with:
path: |
~/cypress_url
key: ${{ github.run_id }}-dashboard-url-${{ github.run_attempt }}
# Set status = success
- name: Save the status of the run
run: |