4382484d29
This PR doesn't have any real affect. It makes the `spec` parameter to
the Cypress action, open to being set. In a following PR, we'll make
this configurable via an extended `/test` command syntax, from
`pr-automation.yml`, coming up.
The `spec` parameter to the Cypress action will be the empty string
here, since we aren't setting it to anything anywhere. The action is
coded to handle empty string here. As evident in this function:
1967c2d49b/index.js (L340).
This is what `ci-test-limited.yml` uses, but is not very convenient
since we have to make another commit to revert the changes in the file
after we're done. Instead, we want to extend the `/test` command's
syntax to be able to inline the spec list in the PR description itself.
🤞
**/test rating**
<!-- This is an auto-generated comment: Cypress test results -->
> [!TIP]
> 🟢 🟢 🟢 All cypress tests have passed! 🎉 🎉 🎉
> Workflow run:
<https://github.com/appsmithorg/appsmith/actions/runs/9797382840>
> Commit: 701bd3b70a25f3554aac143fdbd4cf1b44d258b9
> <a
href="https://internal.appsmith.com/app/cypress-dashboard/rundetails-65890b3c81d7400d08fa9ee5?branch=master&workflowId=9797382840&attempt=1"
target="_blank">Cypress dashboard</a>.
> Tags: `@tag.Rating`
<!-- end of auto-generated comment: Cypress test results -->
<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit
- **Chores**
- Updated CI workflow to allow specifying spec files for Cypress tests.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->
239 lines
9.2 KiB
YAML
239 lines
9.2 KiB
YAML
name: Cypress test suite
|
|
|
|
on:
|
|
workflow_call:
|
|
inputs:
|
|
tags:
|
|
required: true
|
|
type: string
|
|
spec:
|
|
required: false
|
|
type: string
|
|
matrix:
|
|
required: true
|
|
type: string
|
|
is-pg-build:
|
|
description: "This is a boolean value in case the workflow is being called for a PG build"
|
|
required: false
|
|
type: string
|
|
default: "false"
|
|
|
|
jobs:
|
|
server-build:
|
|
if: success()
|
|
name: server-build
|
|
uses: ./.github/workflows/server-build.yml
|
|
secrets: inherit
|
|
with:
|
|
pr: ${{ github.event.number }}
|
|
skip-tests: true
|
|
is-pg-build: ${{ inputs.is-pg-build }}
|
|
|
|
client-build:
|
|
if: success()
|
|
name: client-build
|
|
uses: ./.github/workflows/client-build.yml
|
|
secrets: inherit
|
|
with:
|
|
pr: ${{ github.event.number }}
|
|
|
|
rts-build:
|
|
if: success()
|
|
name: rts-build
|
|
uses: ./.github/workflows/rts-build.yml
|
|
secrets: inherit
|
|
with:
|
|
pr: ${{ github.event.number }}
|
|
|
|
test-appsmithctl:
|
|
if: success()
|
|
name: appsmithctl
|
|
uses: ./.github/workflows/appsmithctl.yml
|
|
secrets: inherit
|
|
with:
|
|
pr: ${{ github.event.number }}
|
|
|
|
build-docker-image:
|
|
needs: [client-build, server-build, rts-build]
|
|
# Only run if the build step is successful
|
|
if: success()
|
|
name: build-docker-image
|
|
uses: ./.github/workflows/build-docker-image.yml
|
|
secrets: inherit
|
|
with:
|
|
pr: ${{ github.event.number }}
|
|
is-pg-build: ${{ inputs.is-pg-build }}
|
|
|
|
ci-test:
|
|
needs: [build-docker-image]
|
|
# Only run if the build step is successful
|
|
if: success()
|
|
name: ci-test
|
|
uses: ./.github/workflows/ci-test-custom-script.yml
|
|
secrets: inherit
|
|
with:
|
|
pr: ${{ github.event.number }}
|
|
tags: ${{ inputs.tags }}
|
|
spec: ${{ inputs.spec }}
|
|
matrix: ${{ inputs.matrix }}
|
|
|
|
ci-test-result:
|
|
needs: [ci-test]
|
|
# Only run if the ci-test with matrices step is successful
|
|
if: always()
|
|
runs-on: ubuntu-latest
|
|
defaults:
|
|
run:
|
|
shell: bash
|
|
steps:
|
|
- name: Checkout repo
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Dump the client payload context
|
|
env:
|
|
PAYLOAD_CONTEXT: ${{ toJson(github.event) }}
|
|
run: echo "$PAYLOAD_CONTEXT"
|
|
|
|
- name: Setup node
|
|
if: needs.ci-test.result != 'success'
|
|
uses: actions/setup-node@v4
|
|
with:
|
|
node-version-file: app/client/package.json
|
|
|
|
- name: Install pg
|
|
if: needs.ci-test.result != 'success'
|
|
run: npm install pg
|
|
|
|
- name: Fetch the failed specs
|
|
if: needs.ci-test.result != 'success'
|
|
id: failed_specs
|
|
env:
|
|
DB_HOST: ${{ secrets.CYPRESS_DB_HOST }}
|
|
DB_NAME: ${{ secrets.CYPRESS_DB_NAME }}
|
|
DB_USER: ${{ secrets.CYPRESS_DB_USER }}
|
|
DB_PWD: ${{ secrets.CYPRESS_DB_PWD }}
|
|
RUN_ID: ${{ github.run_id }}
|
|
ATTEMPT_NUMBER: ${{ github.run_attempt }}
|
|
uses: actions/github-script@v7
|
|
with:
|
|
script: |
|
|
const { Pool } = require("pg");
|
|
const { DB_HOST, DB_NAME, DB_USER, DB_PWD, RUN_ID, ATTEMPT_NUMBER } = process.env
|
|
|
|
const client = await new Pool({
|
|
user: DB_USER,
|
|
host: DB_HOST,
|
|
database: DB_NAME,
|
|
password: DB_PWD,
|
|
port: 5432,
|
|
connectionTimeoutMillis: 60000,
|
|
}).connect();
|
|
|
|
const result = await client.query(
|
|
`SELECT DISTINCT name FROM public."specs"
|
|
WHERE "matrixId" IN
|
|
(SELECT id FROM public."matrix"
|
|
WHERE "attemptId" = (
|
|
SELECT id FROM public."attempt" WHERE "workflowId" = $1 and "attempt" = $2
|
|
)
|
|
) AND status = 'fail'`,
|
|
[RUN_ID, ATTEMPT_NUMBER],
|
|
);
|
|
client.release();
|
|
return result.rows.map((spec) => spec.name);
|
|
|
|
# In case for any ci job failure, create combined failed spec
|
|
- name: Combine all specs for CI
|
|
id: combine_ci
|
|
if: needs.ci-test.result != 'success'
|
|
run: |
|
|
failed_specs=$(echo ${{steps.failed_specs.outputs.result}} | sed 's/\[\|\]//g' | tr -d ' ' | tr ',' '\n')
|
|
while read -r line; do
|
|
echo "$line" >> ~/combined_failed_spec_ci
|
|
done <<< "$failed_specs"
|
|
if [[ -z $(grep '[^[:space:]]' ~/combined_failed_spec_ci) ]] ; then
|
|
echo "specs_failed=0" >> $GITHUB_OUTPUT
|
|
else
|
|
echo "specs_failed=1" >> $GITHUB_OUTPUT
|
|
fi
|
|
|
|
# Upload combined failed CI spec list to a file
|
|
# This is done for debugging
|
|
- name: Upload combined failed spec
|
|
if: needs.ci-test.result != 'success'
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: combined_failed_spec_ci
|
|
path: ~/combined_failed_spec_ci
|
|
overwrite: true
|
|
|
|
- name: Get latest flaky tests
|
|
shell: bash
|
|
run: |
|
|
curl --request POST --url https://yatin-s-workspace-jk8ru5.us-east-1.xata.sh/db/CypressKnownFailures:main/tables/CypressKnownFailuires/query --header 'Authorization: Bearer ${{ secrets.XATA_TOKEN }}' --header 'Content-Type: application/json'|jq -r |grep Spec|cut -d ':' -f 2 2> /dev/null|sed 's/"//g'|sed 's/,//g' > ~/knownfailures
|
|
|
|
# Verify CI test failures against known failures
|
|
- name: Verify CI test failures against known failures
|
|
if: needs.ci-test.result != 'success'
|
|
shell: bash
|
|
run: |
|
|
new_failed_spec_env="$(comm -1 -3 <(sort ~/knownfailures) <(sort -u ~/combined_failed_spec_ci) | sed 's/|cypress|cypress/\n/g' | sed 's/^/<li>/')"
|
|
echo "$new_failed_spec_env"
|
|
echo "new_failed_spec_env<<EOF" >> $GITHUB_ENV
|
|
echo "$new_failed_spec_env" >> $GITHUB_ENV
|
|
echo "EOF" >> $GITHUB_ENV
|
|
|
|
- name: Modify test response in the PR with new CI failures
|
|
if: needs.ci-test.result != 'success' && steps.combine_ci.outputs.specs_failed == '1'
|
|
uses: actions/github-script@v7
|
|
env:
|
|
NODE_PATH: "${{ github.workspace }}/.github/workflows/scripts"
|
|
BODY: |
|
|
Some tests have failed.
|
|
Workflow run: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}>
|
|
Commit: ${{ github.event.pull_request.head.sha }}
|
|
<a href="https://internal.appsmith.com/app/cypress-dashboard/rundetails-65890b3c81d7400d08fa9ee5?branch=master&workflowId=${{ github.run_id }}&attempt=${{ github.run_attempt }}&selectiontype=test&testsstatus=failed&specsstatus=fail" target="_blank">Cypress dashboard</a>.
|
|
Tags: ${{ inputs.tags }}
|
|
The following are new failures, please fix them before merging the PR: <ol>
|
|
${{env.new_failed_spec_env}}</ol>
|
|
<a href="https://internal.appsmith.com/app/cypress-dashboard/identified-flaky-tests-65890b3c81d7400d08fa9ee3?branch=master" target="_blank">List of identified flaky tests</a>.
|
|
with:
|
|
script: |
|
|
require("write-cypress-status.js")({core, context, github}, "caution", process.env.BODY)
|
|
core.setFailed()
|
|
|
|
- name: Modify test response in the PR when ci-test is failed but no specs found
|
|
if: needs.ci-test.result != 'success' && steps.combine_ci.outputs.specs_failed == '0'
|
|
uses: actions/github-script@v7
|
|
env:
|
|
NODE_PATH: "${{ github.workspace }}/.github/workflows/scripts"
|
|
BODY: |
|
|
Workflow run: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}>
|
|
Commit: ${{ github.event.pull_request.head.sha }}
|
|
<a href="https://internal.appsmith.com/app/cypress-dashboard/rundetails-65890b3c81d7400d08fa9ee5?branch=master&workflowId=${{ github.run_id }}&attempt=${{ github.run_attempt }}" target="_blank">Cypress dashboard</a>.
|
|
Tags: ${{ inputs.tags }}
|
|
It seems like **no tests ran** 😔. We are not able to recognize it, please check <a href="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" target="_blank">workflow here</a>.
|
|
with:
|
|
script: |
|
|
require("write-cypress-status.js")({core, context, github}, "warning", process.env.BODY)
|
|
core.setFailed()
|
|
|
|
- name: Modify test response in the PR when ci-test is success
|
|
if: needs.ci-test.result == 'success' && steps.combine_ci.outputs.specs_failed == '0'
|
|
uses: actions/github-script@v7
|
|
env:
|
|
NODE_PATH: "${{ github.workspace }}/.github/workflows/scripts"
|
|
BODY: |
|
|
All cypress tests have passed! 🎉 🎉 🎉
|
|
Workflow run: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}>
|
|
Commit: ${{ github.event.pull_request.head.sha }}
|
|
<a href="https://internal.appsmith.com/app/cypress-dashboard/rundetails-65890b3c81d7400d08fa9ee5?branch=master&workflowId=${{ github.run_id }}&attempt=${{ github.run_attempt }}" target="_blank">Cypress dashboard</a>.
|
|
Tags: `${{ inputs.tags }}`
|
|
with:
|
|
script: |
|
|
require("write-cypress-status.js")({core, context, github}, "tip", process.env.BODY)
|
|
|
|
- name: Check ci-test set status
|
|
if: needs.ci-test.result != 'success'
|
|
run: exit 1
|