From 33e04b5e11af5d00da6644e81f0c4a7a5bcdbfcd Mon Sep 17 00:00:00 2001
From: Shrikant Sharat Kandula <shrikant@appsmith.com>
Date: Tue, 2 Jul 2024 12:58:42 +0530
Subject: [PATCH] ci: Fix method used to set failure (#34402)

And use script to update PR description with Cypress status

## Summary by CodeRabbit

- **Enhancements**
- Improved GitHub Actions workflows for PR automation and Cypress tests.
- Enhanced test failure messages with links to Cypress dashboard and
flaky test identification.

- **User Experience**
- Cypress test status messages now include emoji prefixes for "tip"
(green) and "caution" (red) alerts.

- **Bug Fixes**
- Corrected error handling in scripts to provide more accurate failure
statuses.

<!-- end of auto-generated comment: release notes by coderabbit.ai -->
---
 .github/workflows/pr-automation.yml           |  2 +-
 .github/workflows/pr-cypress.yml              | 89 +++++++++----------
 .github/workflows/scripts/test-tag-parser.js  |  4 +-
 .../workflows/scripts/write-cypress-status.js |  2 +
 4 files changed, 45 insertions(+), 52 deletions(-)

diff --git a/.github/workflows/pr-automation.yml b/.github/workflows/pr-automation.yml
index 4f4f7985d6..f228ba5429 100644
--- a/.github/workflows/pr-automation.yml
+++ b/.github/workflows/pr-automation.yml
@@ -36,7 +36,7 @@ jobs:
           NODE_PATH: "${{ github.workspace }}/.github/workflows/scripts"
         with:
           script: |
-            require("test-tag-parser.js")({core, context})
+            require("test-tag-parser.js")({core, context, github})
 
       # In case of a run with all test cases, allocate a larger matrix
       - name: Check if @tag.All is present in tags
diff --git a/.github/workflows/pr-cypress.yml b/.github/workflows/pr-cypress.yml
index 8234f89507..6c49b3f14b 100644
--- a/.github/workflows/pr-cypress.yml
+++ b/.github/workflows/pr-cypress.yml
@@ -173,7 +173,7 @@ jobs:
         if: needs.ci-test.result != 'success'
         shell: bash
         run: |
-          new_failed_spec_env="$(comm -1 -3 <(sort ~/knownfailures) <(sort -u ~/combined_failed_spec_ci) | sed 's/|cypress|cypress/\n/g' | sed 's/^/> <li>/')"
+          new_failed_spec_env="$(comm -1 -3 <(sort ~/knownfailures) <(sort -u ~/combined_failed_spec_ci) | sed 's/|cypress|cypress/\n/g' | sed 's/^/<li>/')"
           echo "$new_failed_spec_env"
           echo "new_failed_spec_env<<EOF" >> $GITHUB_ENV
           echo "$new_failed_spec_env" >> $GITHUB_ENV
@@ -181,63 +181,54 @@ jobs:
 
       - name: Modify test response in the PR with new CI failures
         if: needs.ci-test.result != 'success' && steps.combine_ci.outputs.specs_failed == '1'
-        uses: nefrob/pr-description@v1.1.2
+        uses: actions/github-script@v7
+        env:
+          NODE_PATH: "${{ github.workspace }}/.github/workflows/scripts"
+          BODY: |
+            Some tests have failed.
+            Workflow run: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}>
+            Commit: ${{ github.event.pull_request.head.sha }}
+            <a href="https://internal.appsmith.com/app/cypress-dashboard/rundetails-65890b3c81d7400d08fa9ee5?branch=master&workflowId=${{ github.run_id }}&attempt=${{ github.run_attempt }}&selectiontype=test&testsstatus=failed&specsstatus=fail" target="_blank">Cypress dashboard</a>.
+            Tags: ${{ inputs.tags }}
+            The following are new failures, please fix them before merging the PR: <ol>
+            ${{env.new_failed_spec_env}}</ol>
+            <a href="https://internal.appsmith.com/app/cypress-dashboard/identified-flaky-tests-65890b3c81d7400d08fa9ee3?branch=master" target="_blank">List of identified flaky tests</a>.
         with:
-          content: |
-            <!-- This is an auto-generated comment: Cypress test results  -->
-            > [!CAUTION]
-            > 🔴 🔴 🔴 Some tests have failed.
-            > Workflow run: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}>
-            > Commit: ${{ github.event.pull_request.head.sha }}
-            > <a href="https://internal.appsmith.com/app/cypress-dashboard/rundetails-65890b3c81d7400d08fa9ee5?branch=master&workflowId=${{ github.run_id }}&attempt=${{ github.run_attempt }}&selectiontype=test&testsstatus=failed&specsstatus=fail" target="_blank">Cypress dashboard</a>.
-            > Tags: `${{ inputs.tags }}`
-            > The following are new failures, please fix them before merging the PR: <ol>
-            ${{env.new_failed_spec_env}} </ol>
-            > <a href="https://internal.appsmith.com/app/cypress-dashboard/identified-flaky-tests-65890b3c81d7400d08fa9ee3?branch=master" target="_blank">List of identified flaky tests</a>.
-
-            <!-- end of auto-generated comment: Cypress test results  -->
-
-          regex: "<!-- This is an auto-generated comment: Cypress test results  -->.*?<!-- end of auto-generated comment: Cypress test results  -->"
-          regexFlags: ims
-          token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            require("write-cypress-status.js")({core, context, github}, "caution", process.env.BODY)
+            core.setFailed()
 
       - name: Modify test response in the PR when ci-test is failed but no specs found
         if: needs.ci-test.result != 'success' && steps.combine_ci.outputs.specs_failed == '0'
-        uses: nefrob/pr-description@v1.1.2
+        uses: actions/github-script@v7
+        env:
+          NODE_PATH: "${{ github.workspace }}/.github/workflows/scripts"
+          BODY: |
+            Workflow run: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}>
+            Commit: ${{ github.event.pull_request.head.sha }}
+            <a href="https://internal.appsmith.com/app/cypress-dashboard/rundetails-65890b3c81d7400d08fa9ee5?branch=master&workflowId=${{ github.run_id }}&attempt=${{ github.run_attempt }}" target="_blank">Cypress dashboard</a>.
+            Tags: ${{ inputs.tags }}
+            It seems like **no tests ran** 😔. We are not able to recognize it, please check <a href="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" target="_blank">workflow here</a>.
         with:
-          content: |
-            <!-- This is an auto-generated comment: Cypress test results  -->
-            > [!WARNING]
-            > Workflow run: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}>
-            > Commit: ${{ github.event.pull_request.head.sha }}
-            > <a href="https://internal.appsmith.com/app/cypress-dashboard/rundetails-65890b3c81d7400d08fa9ee5?branch=master&workflowId=${{ github.run_id }}&attempt=${{ github.run_attempt }}" target="_blank">Cypress dashboard</a>.
-            > Tags: `${{ inputs.tags }}`
-            > It seems like **no tests ran** 😔. We are not able to recognize it, please check <a href="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" target="_blank">workflow here</a>.
-
-            <!-- end of auto-generated comment: Cypress test results  -->
-
-          regex: "<!-- This is an auto-generated comment: Cypress test results  -->.*?<!-- end of auto-generated comment: Cypress test results  -->"
-          regexFlags: ims
-          token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            require("write-cypress-status.js")({core, context, github}, "warning", process.env.BODY)
+            core.setFailed()
 
       - name: Modify test response in the PR when ci-test is success
         if: needs.ci-test.result == 'success' && steps.combine_ci.outputs.specs_failed == '0'
-        uses: nefrob/pr-description@v1.1.2
+        uses: actions/github-script@v7
+        env:
+          NODE_PATH: "${{ github.workspace }}/.github/workflows/scripts"
+          BODY: |
+            All cypress tests have passed! 🎉 🎉 🎉
+            Workflow run: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}>
+            Commit: ${{ github.event.pull_request.head.sha }}
+            <a href="https://internal.appsmith.com/app/cypress-dashboard/rundetails-65890b3c81d7400d08fa9ee5?branch=master&workflowId=${{ github.run_id }}&attempt=${{ github.run_attempt }}" target="_blank">Cypress dashboard</a>.
+            Tags: `${{ inputs.tags }}`
         with:
-          content: |
-            <!-- This is an auto-generated comment: Cypress test results  -->
-            > [!TIP]
-            > 🟢 🟢 🟢 All cypress tests have passed! 🎉 🎉 🎉
-            > Workflow run: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}>
-            > Commit: ${{ github.event.pull_request.head.sha }}
-            > <a href="https://internal.appsmith.com/app/cypress-dashboard/rundetails-65890b3c81d7400d08fa9ee5?branch=master&workflowId=${{ github.run_id }}&attempt=${{ github.run_attempt }}" target="_blank">Cypress dashboard</a>.
-            > Tags: `${{ inputs.tags }}`
-
-            <!-- end of auto-generated comment: Cypress test results  -->
-
-          regex: "<!-- This is an auto-generated comment: Cypress test results  -->.*?<!-- end of auto-generated comment: Cypress test results  -->"
-          regexFlags: ims
-          token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            require("write-cypress-status.js")({core, context, github}, "tip", process.env.BODY)
+            core.setFailed()
 
       - name: Check ci-test set status
         if: needs.ci-test.result != 'success'
diff --git a/.github/workflows/scripts/test-tag-parser.js b/.github/workflows/scripts/test-tag-parser.js
index 20a5dd2e4f..2309de878b 100644
--- a/.github/workflows/scripts/test-tag-parser.js
+++ b/.github/workflows/scripts/test-tag-parser.js
@@ -1,9 +1,9 @@
-module.exports = function ({core, context}) {
+module.exports = function ({core, context, github}) {
   let tags;
   try {
     tags = parseTags(context.payload.pull_request.body);
   } catch (error) {
-    core.setFailure(error.message);
+    core.setFailed(error.message);
     core.setOutput("outcome", "failure");
     const body = [
       "Invalid tags. Please use `/ok-to-test tags=\"@tag.All\"` or `/test all` in the PR body to run all tests.",
diff --git a/.github/workflows/scripts/write-cypress-status.js b/.github/workflows/scripts/write-cypress-status.js
index b40fc377cb..28cbc50ff4 100644
--- a/.github/workflows/scripts/write-cypress-status.js
+++ b/.github/workflows/scripts/write-cypress-status.js
@@ -6,7 +6,9 @@ const PATTERN = new RegExp(HEADER + ".*?" + FOOTER, "ims");
 const VALID_ALERT_TYPES = ["note", "tip", "important", "warning", "caution"]
 
 const ALERT_PREFIXES = {
+  tip: "🟢 🟢 🟢 ",
   important: "🟣 🟣 🟣 ",
+  caution: "🔴 🔴 🔴 ",
 }
 
 module.exports = async function({core, context, github}, alertType, note) {