Protei/PromucFlow_constructor
2
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

ci: Support for limited tests run from PR description (#34736)

Browse Source 
Specify the specs we want to run, in a plain code fence, where the first
line has to exactly be `/test`. The remaining lines have to be one spec
file per line.

Like this:

```
/test
cypress/e2e/Regression/ClientSide/Fork/ForkApplication_spec.ts
cypress/e2e/Regression/ClientSide/JSLibrary/Library_spec.ts
```

1. It's not possible to run with tags and specs and the same time. This
will need a slightly larger effort. But we'll get there.
2. The status message shows spec files like they are space-separated,
and isn't very easy to read. Don't take it too seriously yet, we'll
improve.
3. Globs _should_ also work fine. The `spec` content, _almost_ directly
lands into the `--spec` argument of Cypress. See docs at
https://docs.cypress.io/guides/guides/command-line#cypress-run-spec-lt-spec-gt.
4. Multiple runners is disabled for this, have to solve it separately.
The `ci-test-limited.yml` also doesn't do multiple runners either.

<!-- This is an auto-generated comment: Cypress test results  -->
> [!WARNING]
> Workflow run:
<https://github.com/appsmithorg/appsmith/actions/runs/9803353429>
> Commit: e99ae387c9a239ae51155af45977c285474cfbaf
> <a
href="https://internal.appsmith.com/app/cypress-dashboard/rundetails-65890b3c81d7400d08fa9ee5?branch=master&workflowId=9803353429&attempt=1"
target="_blank">Cypress dashboard</a>.
> Tags: 
> Spec: cypress/e2e/Regression/ClientSide/Fork/ForkApplication_spec.ts
> cypress/e2e/Regression/ClientSide/JSLibrary/Library_spec.ts
> 
> It seems like **no tests ran** 😔. We are not able to recognize it,
please check <a
href="https://github.com/appsmithorg/appsmith/actions/runs/9803353429"
target="_blank">workflow here</a>.
> <hr>Fri, 05 Jul 2024 05:20:23 UTC
<!-- end of auto-generated comment: Cypress test results  -->


<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->

## Summary by CodeRabbit

- **Chores**
- Enhanced GitHub Actions workflows to improve test specification
handling and reporting.
- Refactored tag parsing logic in scripts for more accurate and
informative outputs.


<!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
Shrikant Sharat Kandula 2024-07-08 21:28:07 +05:30 committed by GitHub
parent ed21a8f44c
commit 3264ac3725
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 54 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
.github/workflows/pr-automation.yml vendored
View File

@ -19,6 +19,7 @@ jobs:
shell: bash
outputs:
tags: ${{ steps.parseTags.outputs.tags }}
spec: ${{ steps.parseTags.outputs.spec }}
matrix: ${{ steps.checkAll.outputs.matrix }}
steps:
@ -42,11 +43,15 @@ jobs:
- name: Check if @tag.All is present in tags
id: checkAll
run: |
tags="${{ steps.parseTags.outputs.tags }}"
if [[ $tags == "@tag.All" ]]; then
echo "matrix=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59]" >> $GITHUB_OUTPUT
if [[ -n "${{ steps.parseTags.outputs.spec }}" ]]; then
echo "matrix=[0]" >> $GITHUB_OUTPUT
else
echo "matrix=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19]" >> $GITHUB_OUTPUT
tags="${{ steps.parseTags.outputs.tags }}"
if [[ $tags == "@tag.All" ]]; then
echo "matrix=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59]" >> $GITHUB_OUTPUT
else
echo "matrix=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19]" >> $GITHUB_OUTPUT
fi
fi
# In case of a runnable command, update the PR with run details
@ -60,6 +65,7 @@ jobs:
Commit: ${{ github.event.pull_request.head.sha }}
Workflow: `${{ github.workflow }}`
Tags: `${{ steps.parseTags.outputs.tags }}`
Spec: `${{ steps.parseTags.outputs.spec }}`
with:
script: |
require("write-cypress-status.js")({core, context, github}, "important", process.env.BODY)
@ -72,5 +78,6 @@ jobs:
secrets: inherit
with:
tags: ${{ needs.parse-tags.outputs.tags}}
spec: ${{ needs.parse-tags.outputs.spec}}
matrix: ${{ needs.parse-tags.outputs.matrix}}
is-pg-build: ${{ github.event.pull_request.base.ref == 'pg' }}

3
.github/workflows/pr-cypress.yml vendored
View File

@ -194,6 +194,7 @@ jobs:
Commit: ${{ github.event.pull_request.head.sha }}
<a href="https://internal.appsmith.com/app/cypress-dashboard/rundetails-65890b3c81d7400d08fa9ee5?branch=master&workflowId=${{ github.run_id }}&attempt=${{ github.run_attempt }}&selectiontype=test&testsstatus=failed&specsstatus=fail" target="_blank">Cypress dashboard</a>.
Tags: ${{ inputs.tags }}
Spec: ${{ inputs.spec }}
The following are new failures, please fix them before merging the PR: <ol>
${{env.new_failed_spec_env}}</ol>
<a href="https://internal.appsmith.com/app/cypress-dashboard/identified-flaky-tests-65890b3c81d7400d08fa9ee3?branch=master" target="_blank">List of identified flaky tests</a>.
@ -212,6 +213,7 @@ jobs:
Commit: ${{ github.event.pull_request.head.sha }}
<a href="https://internal.appsmith.com/app/cypress-dashboard/rundetails-65890b3c81d7400d08fa9ee5?branch=master&workflowId=${{ github.run_id }}&attempt=${{ github.run_attempt }}" target="_blank">Cypress dashboard</a>.
Tags: ${{ inputs.tags }}
Spec: ${{ inputs.spec }}
It seems like **no tests ran** 😔. We are not able to recognize it, please check <a href="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" target="_blank">workflow here</a>.
with:
script: |
@ -229,6 +231,7 @@ jobs:
Commit: ${{ github.event.pull_request.head.sha }}
<a href="https://internal.appsmith.com/app/cypress-dashboard/rundetails-65890b3c81d7400d08fa9ee5?branch=master&workflowId=${{ github.run_id }}&attempt=${{ github.run_attempt }}" target="_blank">Cypress dashboard</a>.
Tags: `${{ inputs.tags }}`
Spec: ${{ inputs.spec }}
with:
script: |
require("write-cypress-status.js")({core, context, github}, "tip", process.env.BODY)

61
.github/workflows/scripts/test-tag-parser.js vendored
View File

@ -1,33 +1,37 @@
module.exports = function ({core, context, github}) {
let tags;
try {
tags = parseTags(context.payload.pull_request.body);
} catch (error) {
core.setFailed(error.message);
core.setOutput("outcome", "failure");
const body = [
"Invalid tags. Please use `/ok-to-test tags=\"@tag.All\"` or `/test all` in the PR body to run all tests.",
"[Tags documentation](https://www.notion.so/appsmith/7c0fc64d4efb4afebf53348cd6252918)",
"[List of valid tags](https://github.com/appsmithorg/appsmith/blob/release/app/client/cypress/tags.js)",
].join("\n");
require("write-cypress-status.js")({core, context, github}, "warning", body);
// Get predictable newlines in the body content. Cause for _so_ much unneeded pain in this world!
const body = context.payload.pull_request.body?.replaceAll(/\r(\n)?/g, "\n");
if (!body) {
core.setFailed("Empty payload body!");
return;
}
core.setOutput("tags", tags);
core.setOutput("outcome", "success");
let parseResult;
try {
parseResult = parseTags(body);
} catch (error) {
core.setFailed(error.message);
const body = [
"Invalid tags. Please use `/ok-to-test tags=\"@tag.All\"` or `/test all` in the PR body to run all tests.",
"[Tags documentation](https://www.notion.so/appsmith/7c0fc64d4efb4afebf53348cd6252918).",
"[List of valid tags](https://github.com/appsmithorg/appsmith/blob/release/app/client/cypress/tags.js).",
].join("\n");
require("./write-cypress-status.js")({core, context, github}, "warning", body);
return;
}
// Shouldn't be needed anymore, but remove in separate PR.
return tags;
core.setOutput("tags", parseResult.tags ?? "");
core.setOutput("spec", parseResult.spec ?? "");
}
function parseTags(body) {
const allTags = require(process.env.GITHUB_WORKSPACE + "/app/client/cypress/tags.js").Tag;
// "/ok-to-test" matcher. Takes precedence over the "/test" matcher.
const strictMatch = body.match(/\/ok-to-test tags="(.+?)"/)?.[1];
const strictMatch = body.match(/^\/ok-to-test tags="(.+?)"/m)?.[1];
if (strictMatch) {
if (strictMatch === "@tag.All") {
return strictMatch;
return { tags: strictMatch };
}
const parts = strictMatch.split(/\s*,\s*/);
for (const part of parts) {
@ -35,7 +39,14 @@ function parseTags(body) {
throw new Error("Unknown tag: " + part);
}
}
return strictMatch;
return { tags: strictMatch };
}
// "/test" code-fence matcher.
const result = matchCodeFence(body);
if (result) {
console.log("Code fence match:\n" + result.spec);
return result;
}
// "/test" matcher.
@ -43,7 +54,7 @@ function parseTags(body) {
const concreteTags = [];
if (config.toLowerCase() === "all") {
return "@tag.All"
return { tags: "@tag.All" };
}
for (const [rawTag] of config.matchAll(/\w+/g)) {
@ -78,5 +89,13 @@ function parseTags(body) {
throw new Error("Tags were not found!")
}
return concreteTags.join(", ");
return { tags: concreteTags.join(", ") };
}
function matchCodeFence(body) {
const re = /^```\n\/test\n(.+?)^```\n/ms;
const spec = body.match(re)?.[1];
return spec ? { spec } : null;
}