97f8facdd1
This PR fixes incorrect diff calculation between base and head of the pull request raised. This diff is used for determining if server tests should run or not. Related PR to this is : https://github.com/appsmithorg/appsmith/pull/24731
358 lines
13 KiB
YAML
358 lines
13 KiB
YAML
name: Appsmith External Integration Test Workflow
|
|
|
|
on:
|
|
# This workflow is only triggered by the ok to test command dispatch
|
|
repository_dispatch:
|
|
types: [ ok-to-test-command ]
|
|
|
|
jobs:
|
|
notify:
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
# This step creates a comment on the PR with a link to this workflow run.
|
|
- name: Add a comment on the PR with link to workflow run
|
|
uses: peter-evans/create-or-update-comment@v2
|
|
with:
|
|
issue-number: ${{ github.event.client_payload.pull_request.number }}
|
|
body: |
|
|
Tests running at: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}>.
|
|
Workflow: `${{ github.workflow }}`.
|
|
Commit: `${{ github.event.client_payload.slash_command.args.named.sha }}`.
|
|
PR: ${{ github.event.client_payload.pull_request.number }}.
|
|
Perf tests will be available at <https://app.appsmith.com/app/performance-infra-dashboard/pr-details-638dd7cd2913ba43778b915e?pr=${{ github.event.client_payload.pull_request.number }}&runId=${{ github.run_id }}_${{github.run_attempt}}>
|
|
changeset:
|
|
runs-on: ubuntu-latest
|
|
# Required permissions
|
|
permissions:
|
|
pull-requests: read
|
|
# Set job outputs to values from filter step
|
|
outputs:
|
|
backend: ${{ steps.filter.outputs.backend }}
|
|
steps:
|
|
- uses: actions/checkout@v3
|
|
with:
|
|
fetch-depth: 0
|
|
ref: ${{ github.event.client_payload.pull_request.merge.ref }}
|
|
- uses: dorny/paths-filter@v2
|
|
id: filter
|
|
with:
|
|
base: ${{ github.event.client_payload.pull_request.base.ref }}
|
|
ref: ${{ github.event.client_payload.pull_request.head.ref }}
|
|
filters: |
|
|
backend:
|
|
- 'app/server/**'
|
|
|
|
server-build:
|
|
name: server-build
|
|
needs: [changeset]
|
|
uses: ./.github/workflows/server-build.yml
|
|
secrets: inherit
|
|
with:
|
|
pr: ${{ github.event.client_payload.pull_request.number }}
|
|
skip-tests: ${{ needs.changeset.outputs.backend == 'false' }}
|
|
|
|
client-build:
|
|
name: client-build
|
|
uses: ./.github/workflows/client-build.yml
|
|
secrets: inherit
|
|
with:
|
|
pr: ${{ github.event.client_payload.pull_request.number }}
|
|
|
|
rts-build:
|
|
name: rts-build
|
|
uses: ./.github/workflows/rts-build.yml
|
|
secrets: inherit
|
|
with:
|
|
pr: ${{ github.event.client_payload.pull_request.number }}
|
|
|
|
test-shared-modules:
|
|
name: shared-modules
|
|
uses: ./.github/workflows/shared-modules.yml
|
|
secrets: inherit
|
|
with:
|
|
pr: ${{ github.event.client_payload.pull_request.number }}
|
|
|
|
test-appsmithctl:
|
|
name: appsmithctl
|
|
uses: ./.github/workflows/appsmithctl.yml
|
|
secrets: inherit
|
|
with:
|
|
pr: ${{ github.event.client_payload.pull_request.number }}
|
|
|
|
build-docker-image:
|
|
needs: [ client-build, server-build, rts-build ]
|
|
# Only run if the build step is successful
|
|
if: success()
|
|
name: build-docker-image
|
|
uses: ./.github/workflows/build-docker-image.yml
|
|
secrets: inherit
|
|
with:
|
|
pr: ${{ github.event.client_payload.pull_request.number }}
|
|
|
|
ci-test:
|
|
needs: [ build-docker-image ]
|
|
# Only run if the build step is successful
|
|
if: success()
|
|
name: ci-test
|
|
uses: ./.github/workflows/ci-test.yml
|
|
secrets: inherit
|
|
with:
|
|
pr: ${{ github.event.client_payload.pull_request.number }}
|
|
|
|
perf-test:
|
|
needs: [ build-docker-image ]
|
|
# Only run if the build step is successful
|
|
if: success()
|
|
name: perf-test
|
|
uses: ./.github/workflows/perf-test.yml
|
|
secrets: inherit
|
|
with:
|
|
pr: ${{ github.event.client_payload.pull_request.number }}
|
|
|
|
perf-test-v2:
|
|
needs: [ build-docker-image ]
|
|
# Only run if the build step is successful
|
|
if: success()
|
|
name: perf-test
|
|
uses: ./.github/workflows/perf-test-v2.yml
|
|
secrets: inherit
|
|
with:
|
|
pr: ${{ github.event.client_payload.pull_request.number }}
|
|
|
|
ci-test-result:
|
|
needs: [ci-test, perf-test ]
|
|
# Only run if the ci-test with matrices step is successful
|
|
if: always()
|
|
runs-on: ubuntu-latest
|
|
defaults:
|
|
run:
|
|
shell: bash
|
|
steps:
|
|
- name: Dump the client payload context
|
|
env:
|
|
PAYLOAD_CONTEXT: ${{ toJson(github.event.client_payload) }}
|
|
run: echo "$PAYLOAD_CONTEXT"
|
|
|
|
# Deleting the existing dir's if any
|
|
- name: Delete existing directories
|
|
if: needs.ci-test.result != 'success'
|
|
run: |
|
|
rm -f ~/failed_spec_ci
|
|
rm -f ~/combined_failed_spec_ci
|
|
|
|
# Download failed_spec list for all jobs
|
|
- uses: actions/download-artifact@v3
|
|
if: needs.ci-test.result != 'success'
|
|
id: download_ci
|
|
with:
|
|
name: failed-spec-ci
|
|
path: ~/failed_spec_ci
|
|
|
|
# In case for any ci job failure, create combined failed spec
|
|
- name: "combine all specs for CI"
|
|
if: needs.ci-test.result != 'success'
|
|
run: |
|
|
echo "Debugging: failed specs in ~/failed_spec_ci/failed_spec_ci*"
|
|
cat ~/failed_spec_ci/failed_spec_ci*
|
|
cat ~/failed_spec_ci/failed_spec_ci* | sort -u >> ~/combined_failed_spec_ci
|
|
|
|
# Force save the CI failed spec list into a cache
|
|
- name: Store the combined run result for CI
|
|
if: needs.ci-test.result != 'success'
|
|
uses: martijnhols/actions-cache/save@v3
|
|
with:
|
|
path: |
|
|
~/combined_failed_spec_ci
|
|
key: ${{ github.run_id }}-"ci-test-result"
|
|
restore-keys: |
|
|
${{ github.run_id }}-${{ github.job }}
|
|
|
|
# Upload combined failed CI spec list to a file
|
|
# This is done for debugging.
|
|
- name: upload combined failed spec
|
|
if: needs.ci-test.result != 'success'
|
|
uses: actions/upload-artifact@v3
|
|
with:
|
|
name: combined_failed_spec_ci
|
|
path: ~/combined_failed_spec_ci
|
|
|
|
- name: Get Latest flaky Tests
|
|
shell: bash
|
|
run: |
|
|
curl --request POST --url https://yatin-s-workspace-jk8ru5.us-east-1.xata.sh/db/CypressKnownFailures:main/tables/CypressKnownFailuires/query --header 'Authorization: Bearer ${{ secrets.XATA_TOKEN }}' --header 'Content-Type: application/json'|jq -r |grep Spec|cut -d ':' -f 2 2> /dev/null|sed 's/"//g'|sed 's/,//g' > ~/knownfailures
|
|
|
|
# Verify CI test failures against known failures
|
|
- name: Verify CI test failures against known failures
|
|
if: needs.ci-test.result != 'success'
|
|
shell: bash
|
|
run: |
|
|
new_failed_spec_env="<ol>$(comm -1 -3 <(sort ~/knownfailures) <(sort -u ~/combined_failed_spec_ci) | sed 's/|cypress|cypress/\n/g' | sed 's/^/<li>/')</ol>"
|
|
echo "$new_failed_spec_env"
|
|
echo "new_failed_spec_env<<EOF" >> $GITHUB_ENV
|
|
echo "$new_failed_spec_env" >> $GITHUB_ENV
|
|
echo "EOF" >> $GITHUB_ENV
|
|
|
|
- name: Add a comment on the PR with new CI failures
|
|
if: needs.ci-test.result != 'success'
|
|
uses: peter-evans/create-or-update-comment@v1
|
|
with:
|
|
issue-number: ${{ github.event.client_payload.pull_request.number }}
|
|
body: |
|
|
Workflow run: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}>.
|
|
Commit: `${{ github.event.client_payload.slash_command.args.named.sha }}`.
|
|
The following are new failures, please fix them before merging the PR: ${{env.new_failed_spec_env}}
|
|
To know the list of identified flaky tests - <a href="https://app.appsmith.com/applications/613868bedd7786286ddd4a6a/pages/63ec710e8e503f763651791a" target="_blank">Refer here</a>
|
|
|
|
- name: Add a comment on the PR when ci-test is success
|
|
if: needs.ci-test.result == 'success'
|
|
uses: peter-evans/create-or-update-comment@v1
|
|
with:
|
|
issue-number: ${{ github.event.client_payload.pull_request.number }}
|
|
body: |
|
|
Workflow run: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}>.
|
|
Commit: `${{ github.event.client_payload.slash_command.args.named.sha }}`.
|
|
All cypress tests have passed 🎉
|
|
|
|
|
|
# Update check run called "ci-test-result"
|
|
- name: Mark ci-test-result job as complete
|
|
uses: actions/github-script@v6
|
|
id: update-check-run
|
|
if: ${{ always() }}
|
|
env:
|
|
run_id: ${{ github.run_id }}
|
|
repository: ${{ github.repository }}
|
|
number: ${{ github.event.client_payload.pull_request.number }}
|
|
job: ${{ github.job }}
|
|
# Conveniently, job.status maps to https://developer.github.com/v3/checks/runs/#update-a-check-run
|
|
conclusion: ${{ job.status }}
|
|
matrix_result: ${{ toJson(needs.ci-test) }}
|
|
with:
|
|
github-token: ${{ secrets.GITHUB_TOKEN }}
|
|
script: |
|
|
const { data: pull } = await github.rest.pulls.get({
|
|
...context.repo,
|
|
pull_number: process.env.number
|
|
});
|
|
const ref = pull.head.sha;
|
|
|
|
const { data: checks } = await github.rest.checks.listForRef({
|
|
...context.repo,
|
|
ref
|
|
});
|
|
|
|
const check = checks.check_runs.filter(c => c.name === process.env.job);
|
|
|
|
if(check.length == 0) {
|
|
const head_sha = pull.head.sha;
|
|
const { data: completed_at } = await github.rest.checks.create({
|
|
owner: context.repo.owner,
|
|
repo: context.repo.repo,
|
|
head_sha: head_sha,
|
|
name: process.env.job,
|
|
status: 'completed',
|
|
conclusion: JSON.parse(process.env.matrix_result).result,
|
|
output: {
|
|
title: "Integration tests result for ok to test",
|
|
summary: "https://github.com/" + process.env.repository + "/actions/runs/" + process.env.run_id
|
|
}
|
|
});
|
|
|
|
return completed_at;
|
|
} else {
|
|
try {
|
|
const { data: result } = await github.rest.checks.update({
|
|
...context.repo,
|
|
check_run_id: check[0].id,
|
|
status: 'completed',
|
|
conclusion: JSON.parse(process.env.matrix_result).result,
|
|
output: {
|
|
title: "Integration tests result for ok to test",
|
|
summary: "https://github.com/" + process.env.repository + "/actions/runs/" + process.env.run_id
|
|
}
|
|
});
|
|
console.log({ result });
|
|
return result;
|
|
} catch(e) {
|
|
console.error({ error: e.message });
|
|
}
|
|
}
|
|
|
|
- name: Dump the client payload context
|
|
env:
|
|
PAYLOAD_CONTEXT: ${{ toJson(github.event.client_payload) }}
|
|
run: echo "$PAYLOAD_CONTEXT"
|
|
|
|
- name: Check ci-test set status
|
|
if: needs.ci-test.result != 'success' || needs.perf-test.result != 'success'
|
|
run: exit 1
|
|
|
|
package:
|
|
needs: [ ci-test ]
|
|
runs-on: ubuntu-latest
|
|
defaults:
|
|
run:
|
|
working-directory: app/client
|
|
# Run this job only if all the previous steps are a success and the reference if the release or master branch
|
|
if: success() && (github.ref == 'refs/heads/release' || github.ref == 'refs/heads/master')
|
|
|
|
steps:
|
|
# Update check run called "package"
|
|
- name: Mark package job as complete
|
|
uses: actions/github-script@v6
|
|
id: update-check-run
|
|
if: ${{ always() }}
|
|
env:
|
|
run_id: ${{ github.run_id }}
|
|
repository: ${{ github.repository }}
|
|
number: ${{ github.event.client_payload.pull_request.number }}
|
|
job: ${{ github.job }}
|
|
# Conveniently, job.status maps to https://developer.github.com/v3/checks/runs/#update-a-check-run
|
|
conclusion: ${{ job.status }}
|
|
with:
|
|
github-token: ${{ secrets.GITHUB_TOKEN }}
|
|
script: |
|
|
const { data: pull } = await github.rest.pulls.get({
|
|
...context.repo,
|
|
pull_number: process.env.number
|
|
});
|
|
const ref = pull.head.sha;
|
|
|
|
const { data: checks } = await github.rest.checks.listForRef({
|
|
...context.repo,
|
|
ref
|
|
});
|
|
|
|
const check = checks.check_runs.filter(c => c.name === process.env.job);
|
|
|
|
if(check.length == 0) {
|
|
const head_sha = pull.head.sha;
|
|
const { data: completed_at } = await github.rest.checks.create({
|
|
owner: context.repo.owner,
|
|
repo: context.repo.repo,
|
|
head_sha: head_sha,
|
|
name: process.env.job,
|
|
status: 'completed',
|
|
conclusion: process.env.conclusion,
|
|
output: {
|
|
title: "Package result for ok to test",
|
|
summary: "https://github.com/" + process.env.repository + "/actions/runs/" + process.env.run_id
|
|
}
|
|
});
|
|
|
|
return completed_at;
|
|
} else {
|
|
const { data: result } = await github.rest.checks.update({
|
|
...context.repo,
|
|
check_run_id: check[0].id,
|
|
status: 'completed',
|
|
conclusion: process.env.conclusion,
|
|
output: {
|
|
title: "Package result for ok to test",
|
|
summary: "https://github.com/" + process.env.repository + "/actions/runs/" + process.env.run_id
|
|
}
|
|
});
|
|
|
|
return result;
|
|
}
|