From ba9ebf1d6e52313d05a9854678080375e67db85a Mon Sep 17 00:00:00 2001 From: Saroj <43822041+sarojsarab@users.noreply.github.com> Date: Wed, 6 Sep 2023 14:40:13 +0530 Subject: [PATCH] ci: Migrate pending workflows to dime defenders (#26968) ## Description - Updated the logic to use DimeDefenders spec split - Updated ci-test-hosted, ci-test-limited and ci-test-custom-script - Changes to handle when there is no spec to run in CI #### Type of change - DimeDefender script - Workflows ## Testing - Workflow run --- .github/workflows/ci-test-custom-script.yml | 20 ++-- .github/workflows/ci-test-hosted.yml | 63 ++++++----- .github/workflows/ci-test-limited.yml | 77 ++++++++----- app/client/cypress/cypress-split.ts | 115 +++++++++----------- app/client/cypress/plugins/index.js | 11 +- app/client/cypress/scripts/cypress-hooks.js | 106 +++++++++--------- app/client/cypress/scripts/no_spec.ts | 1 + app/client/cypress_ci_hosted.config.ts | 32 +++++- 8 files changed, 232 insertions(+), 193 deletions(-) create mode 100644 app/client/cypress/scripts/no_spec.ts diff --git a/.github/workflows/ci-test-custom-script.yml b/.github/workflows/ci-test-custom-script.yml index 0714ccbfc7..45563e589b 100644 --- a/.github/workflows/ci-test-custom-script.yml +++ b/.github/workflows/ci-test-custom-script.yml @@ -368,15 +368,10 @@ jobs: CYPRESS_AIRGAPPED: false APPSMITH_DISABLE_TELEMETRY: true APPSMITH_GOOGLE_MAPS_API_KEY: ${{ secrets.APPSMITH_GOOGLE_MAPS_API_KEY }} - COMMIT_INFO_MESSAGE: ${{ env.COMMIT_INFO_MESSAGE }} CYPRESS_VERIFY_TIMEOUT: 100000 + COMMIT_INFO_MESSAGE: ${{ env.COMMIT_INFO_MESSAGE }} TOTAL_RUNNERS: ${{ strategy.job-total }} THIS_RUNNER: ${{ strategy.job-index }} - CYPRESS_SPEC_PATTERN: "cypress/e2e/**/**/*" - CYPRESS_ENV: "NODE_ENV=development" - CYPRESS_CONFIG_FILE: cypress_ci_custom.config.ts - CYPRESS_HEADLESS: "true" - CYPRESS_BROWSER: ${{ env.BROWSER_PATH }} RUNID: ${{ github.run_id }} ATTEMPT_NUMBER: ${{ github.run_attempt }} REPOSITORY: ${{ github.repository }} @@ -390,9 +385,11 @@ jobs: CYPRESS_S3_ACCESS: ${{ secrets.CYPRESS_S3_ACCESS }} CYPRESS_S3_SECRET: ${{ secrets.CYPRESS_S3_SECRET }} with: + browser: ${{ env.BROWSER_PATH }} install: false + config-file: cypress_ci_custom.config.ts working-directory: app/client - command: yarn ts-node --require esm cypress/cypress-split.ts + env: "NODE_ENV=development" # In case of second attempt only run failed specs - name: Run the cypress test with failed tests @@ -440,10 +437,7 @@ jobs: APPSMITH_DISABLE_TELEMETRY: true APPSMITH_GOOGLE_MAPS_API_KEY: ${{ secrets.APPSMITH_GOOGLE_MAPS_API_KEY }} COMMIT_INFO_MESSAGE: ${{ env.COMMIT_INFO_MESSAGE }} - CYPRESS_ENV: "NODE_ENV=development" - CYPRESS_CONFIG_FILE: cypress_ci.config.ts - CYPRESS_HEADLESS: "true" - CYPRESS_BROWSER: ${{ env.BROWSER_PATH }} + CYPRESS_SPECS: ${{ env.failed_spec_env }} RUNID: ${{ github.run_id }} ATTEMPT_NUMBER: ${{ github.run_attempt }} REPOSITORY: ${{ github.repository }} @@ -458,9 +452,11 @@ jobs: CYPRESS_S3_ACCESS: ${{ secrets.CYPRESS_S3_ACCESS }} CYPRESS_S3_SECRET: ${{ secrets.CYPRESS_S3_SECRET }} with: + browser: ${{ env.BROWSER_PATH }} install: false + config-file: cypress_ci_custom.config.ts working-directory: app/client - command: yarn cypress run --browser ${{ env.BROWSER_PATH }} --config-file cypress_ci_custom.config.ts --spec ${{ env.failed_spec_env }} + env: "NODE_ENV=development" - name: Collect CI container logs if: failure() diff --git a/.github/workflows/ci-test-hosted.yml b/.github/workflows/ci-test-hosted.yml index eb5e0c9e60..6ce21e9098 100644 --- a/.github/workflows/ci-test-hosted.yml +++ b/.github/workflows/ci-test-hosted.yml @@ -107,14 +107,24 @@ jobs: if: steps.run_result.outputs.run_result == 'failedtest' working-directory: app/client run: | - echo "failed_spec_env<> $GITHUB_ENV - while IFS= read -r line - do - spec_name=$(echo $line | awk -F'/' '{print $NF}') - failed_spec_env=$(find . -name $spec_name | sed 's|./||') - echo "$failed_spec_env" >> $GITHUB_ENV - done < ~/failed_spec_ci/failed_spec_ci - echo "EOF" >> $GITHUB_ENV + failed_spec_env="" + while IFS= read -r line || [ -n "$line" ]; do + file_path=$(echo "$line" | awk -F'/' '{print $(NF-1)"/"$NF}') + spec_name=$(echo "$file_path" | awk -F'/' '{print $NF}') + failed_spec=$(find . -name "$spec_name" | sed 's|./||') + if [ "$(echo "$failed_spec" | wc -l)" -eq 1 ]; then + failed_spec_env="$failed_spec_env,$failed_spec" + else + for file in $failed_spec; do + new_file_path=$(echo "$file" | awk -F'/' '{print $(NF-1)"/"$NF}') + if [ "$new_file_path" == "$file_path" ]; then + failed_spec_env="$failed_spec_env,$file" + fi + done + fi + done < ~/failed_spec_ci/failed_spec_ci-${{ matrix.job }} + failed_spec_env=${failed_spec_env#,} + echo "failed_spec_env=$failed_spec_env" >> $GITHUB_ENV - if: steps.run_result.outputs.run_result != 'success' && steps.run_result.outputs.run_result != 'failedtest' run: echo "Starting full run" && exit 0 @@ -256,8 +266,6 @@ jobs: uses: cypress-io/github-action@v5 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }} - CYPRESS_PROJECT_ID: ${{ secrets.CYPRESS_PROJECT_ID }} CYPRESS_USERNAME: ${{ secrets.CYPRESS_USERNAME }} CYPRESS_PASSWORD: ${{ secrets.CYPRESS_PASSWORD }} CYPRESS_TESTUSERNAME1: ${{ secrets.CYPRESS_TESTUSERNAME1 }} @@ -305,6 +313,7 @@ jobs: TAG: ${{ github.event_name }} BRANCH: ${{ env.COMMIT_INFO_BRANCH }} THIS_RUNNER: ${{ strategy.job-index }} + TOTAL_RUNNERS: ${{ strategy.job-total }} CYPRESS_DB_USER: ${{ secrets.CYPRESS_DB_USER }} CYPRESS_DB_HOST: ${{ secrets.CYPRESS_DB_HOST }} CYPRESS_DB_NAME: ${{ secrets.CYPRESS_DB_NAME }} @@ -313,17 +322,9 @@ jobs: CYPRESS_S3_SECRET: ${{ secrets.CYPRESS_S3_SECRET }} with: browser: ${{ env.BROWSER_PATH }} - record: true install: false - parallel: true config-file: cypress_ci_hosted.config.ts - group: "Chrome-Fat Container tests" - spec: | - cypress/e2e/Sanity/Datasources/Airtable_Basic_Spec.ts - cypress/e2e/GSheet/**/**/* working-directory: app/client - # tag will be either "push" or "pull_request" - tag: ${{ github.event_name }} env: "NODE_ENV=development" # In case of second attempt only run failed specs @@ -333,8 +334,6 @@ jobs: uses: cypress-io/github-action@v5 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }} - CYPRESS_PROJECT_ID: ${{ secrets.CYPRESS_PROJECT_ID }} CYPRESS_USERNAME: ${{ secrets.CYPRESS_USERNAME }} CYPRESS_PASSWORD: ${{ secrets.CYPRESS_PASSWORD }} CYPRESS_TESTUSERNAME1: ${{ secrets.CYPRESS_TESTUSERNAME1 }} @@ -381,6 +380,7 @@ jobs: TAG: ${{ github.event_name }} BRANCH: ${{ env.COMMIT_INFO_BRANCH }} THIS_RUNNER: ${{ strategy.job-index }} + CYPRESS_SPECS: ${{ env.failed_spec_env }} CYPRESS_DB_USER: ${{ secrets.CYPRESS_DB_USER }} CYPRESS_DB_HOST: ${{ secrets.CYPRESS_DB_HOST }} CYPRESS_DB_NAME: ${{ secrets.CYPRESS_DB_NAME }} @@ -389,17 +389,24 @@ jobs: CYPRESS_S3_SECRET: ${{ secrets.CYPRESS_S3_SECRET }} with: browser: ${{ env.BROWSER_PATH }} - record: true install: false - parallel: true config-file: cypress_ci_hosted.config.ts - group: "Chrome-Fat Container tests" - spec: ${{ env.failed_spec_env }} working-directory: app/client - # tag will be either "push" or "pull_request" - tag: ${{ github.event_name }} env: "NODE_ENV=development" + - name: Rename reports + if: always() + run: | + mkdir -p ~/results + mv ${{ github.workspace }}/app/client/results ~/results/${{ matrix.job }} + + - name: Upload cypress report + if: always() + uses: actions/upload-artifact@v3 + with: + name: results-${{github.run_attempt}} + path: ~/results + # Set status = failedtest - name: Set fail if there are test failures id: test_status @@ -443,9 +450,9 @@ jobs: if: always() run: | if [[ "${{steps.run_result.outputs.run_result }}" != "success" && "${{steps.run_result.outputs.run_result }}" != "failedtest" ]]; then - echo "url=${{ steps.cypress_test.outputs.resultsUrl }}" >> $GITHUB_OUTPUT + echo "" >> $GITHUB_OUTPUT elif [[ "${{steps.run_result.outputs.run_result }}" == "failedtest" ]]; then - echo "url=${{ steps.cypress_test_failedtest.outputs.resultsUrl }}" >> $GITHUB_OUTPUT + echo "" >> $GITHUB_OUTPUT fi - name: Generate slack message diff --git a/.github/workflows/ci-test-limited.yml b/.github/workflows/ci-test-limited.yml index 4a46662543..173548f7d3 100644 --- a/.github/workflows/ci-test-limited.yml +++ b/.github/workflows/ci-test-limited.yml @@ -5,20 +5,20 @@ on: workflow_dispatch: inputs: pr: - description: "This is the PR number in case the workflow is being called in a pull request" + description: "PR number" required: false type: number default: 0 previous-workflow-run-id: - description: "This is the PR number in case the workflow is being called in a pull request" + description: "Workflow ID (To Download cicontainer)" required: false type: number default: 0 matrix: - description: "This is the matrix job number" + description: "Matrix jobs" required: false type: string - default: "[0, 1, 2, 3, 4]" + default: "[0, 1, 2]" workflow_call: inputs: pr: @@ -129,29 +129,40 @@ jobs: if: steps.run_result.outputs.run_result == 'failedtest' working-directory: app/client run: | - echo "failed_spec_env<> $GITHUB_ENV - while IFS= read -r line - do - spec_name=$(echo $line | awk -F'/' '{print $NF}') - failed_spec_env=$(find . -name $spec_name | sed 's|./||') - echo "$failed_spec_env" >> $GITHUB_ENV - done < ~/combined_failed_spec_ci/combined_failed_spec_ci - echo "EOF" >> $GITHUB_ENV + failed_spec_env="" + while IFS= read -r line || [ -n "$line" ]; do + file_path=$(echo "$line" | awk -F'/' '{print $(NF-1)"/"$NF}') + spec_name=$(echo "$file_path" | awk -F'/' '{print $NF}') + failed_spec=$(find . -name "$spec_name" | sed 's|./||') + if [ "$(echo "$failed_spec" | wc -l)" -eq 1 ]; then + failed_spec_env="$failed_spec_env,$failed_spec" + else + for file in $failed_spec; do + new_file_path=$(echo "$file" | awk -F'/' '{print $(NF-1)"/"$NF}') + if [ "$new_file_path" == "$file_path" ]; then + failed_spec_env="$failed_spec_env,$file" + fi + done + fi + done < ~/failed_spec_ci/failed_spec_ci-${{ matrix.job }} + failed_spec_env=${failed_spec_env#,} + echo "failed_spec_env=$failed_spec_env" >> $GITHUB_ENV # Get specs to run - name: Get specs to run if: steps.run_result.outputs.run_result != 'success' run: | - echo "specs_to_run<> $GITHUB_ENV + specs_to_run="" while IFS= read -r line do if [[ $line =~ ^#|^\/\/ ]]; then continue else - echo "$line" >> $GITHUB_ENV + specs_to_run="$specs_to_run,$line" fi done < app/client/cypress/limited-tests.txt - echo "EOF" >> $GITHUB_ENV + specs_to_run=${specs_to_run#,} + echo "specs_to_run=$specs_to_run" >> $GITHUB_ENV # In case of run-id provided download the artifact from the previous run - name: Download Docker image artifact @@ -407,7 +418,9 @@ jobs: COMMITTER: ${{ env.COMMIT_INFO_AUTHOR }} TAG: ${{ github.event_name }} BRANCH: ${{ env.COMMIT_INFO_BRANCH }} + TOTAL_RUNNERS: ${{ strategy.job-total }} THIS_RUNNER: ${{ strategy.job-index }} + CYPRESS_SPECS: ${{ env.specs_to_run }} CYPRESS_DB_USER: ${{ secrets.CYPRESS_DB_USER }} CYPRESS_DB_HOST: ${{ secrets.CYPRESS_DB_HOST }} CYPRESS_DB_NAME: ${{ secrets.CYPRESS_DB_NAME }} @@ -416,15 +429,10 @@ jobs: CYPRESS_S3_SECRET: ${{ secrets.CYPRESS_S3_SECRET }} with: browser: ${{ env.BROWSER_PATH }} - record: true install: false - parallel: true - config-file: cypress_ci.config.ts - group: "Chrome-Fat Container tests" + config-file: cypress_ci_custom.config.ts spec: ${{ env.specs_to_run }} working-directory: app/client - # tag will be either "push" or "pull_request" - tag: ${{ github.event_name }} env: "NODE_ENV=development" # In case of second attempt only run failed specs @@ -482,7 +490,9 @@ jobs: COMMITTER: ${{ env.COMMIT_INFO_AUTHOR }} TAG: ${{ github.event_name }} BRANCH: ${{ env.COMMIT_INFO_BRANCH }} + TOTAL_RUNNERS: ${{ strategy.job-total }} THIS_RUNNER: ${{ strategy.job-index }} + CYPRESS_SPECS: ${{ env.failed_spec_env }} CYPRESS_DB_USER: ${{ secrets.CYPRESS_DB_USER }} CYPRESS_DB_HOST: ${{ secrets.CYPRESS_DB_HOST }} CYPRESS_DB_NAME: ${{ secrets.CYPRESS_DB_NAME }} @@ -491,15 +501,9 @@ jobs: CYPRESS_S3_SECRET: ${{ secrets.CYPRESS_S3_SECRET }} with: browser: ${{ env.BROWSER_PATH }} - record: true install: false - parallel: true - config-file: cypress_ci.config.ts - group: "Chrome-Fat Container tests" - spec: ${{ env.failed_spec_env }} + config-file: cypress_ci_custom.config.ts working-directory: app/client - # tag will be either "push" or "pull_request" - tag: ${{ github.event_name }} env: "NODE_ENV=development" - name: Collect CI container logs @@ -517,6 +521,19 @@ jobs: name: dockerlogs path: ~/dockerlogs + - name: Rename reports + if: always() + run: | + mkdir -p ~/results + mv ${{ github.workspace }}/app/client/results ~/results/${{ matrix.job }} + + - name: Upload cypress report + if: always() + uses: actions/upload-artifact@v3 + with: + name: results-${{github.run_attempt}} + path: ~/results + # Set status = failedtest - name: Set fail if there are test failures if: failure() @@ -570,9 +587,9 @@ jobs: if: always() run: | if [[ "${{steps.run_result.outputs.run_result }}" != "success" && "${{steps.run_result.outputs.run_result }}" != "failedtest" ]]; then - echo ${{ steps.cypress_test.outputs.resultsUrl }} >> ~/cypress_url + echo "" >> ~/cypress_url elif [[ "${{steps.run_result.outputs.run_result }}" == "failedtest" ]]; then - echo ${{ steps.cypress_test_failedtest.outputs.resultsUrl }} >> ~/cypress_url + echo "" >> ~/cypress_url fi # Force store previous run result to cache diff --git a/app/client/cypress/cypress-split.ts b/app/client/cypress/cypress-split.ts index e8b84df417..2b6fbace72 100644 --- a/app/client/cypress/cypress-split.ts +++ b/app/client/cypress/cypress-split.ts @@ -9,17 +9,14 @@ type GetEnvOptions = { required?: boolean; }; -// some default properties -const ignoreTestFiles = [ - "cypress/e2e/**/spec_utility.ts", - "cypress/e2e/GSheet/**/*", -]; - // used to roughly determine how many tests are in a file const testPattern = /(^|\s)(it)\(/g; // This function will get all the spec paths using the pattern -async function getSpecFilePaths(specPattern: any): Promise { +async function getSpecFilePaths( + specPattern: any, + ignoreTestFiles: any, +): Promise { const files = globby.sync(specPattern, { ignore: ignoreTestFiles, }); @@ -81,27 +78,26 @@ function splitSpecs( async function getSpecsToRun( totalRunnersCount = 0, currentRunner = 0, - specPattern = "cypress/e2e/**/**/*.{js,ts}", -): Promise { - { - try { - const specFilePaths = await sortSpecFilesByTestCount( - await getSpecFilePaths(specPattern), - ); + specPattern: string | string[] = "cypress/e2e/**/**/*.{js,ts}", + ignorePattern: string | string[], +): Promise { + try { + const specFilePaths = await sortSpecFilesByTestCount( + await getSpecFilePaths(specPattern, ignorePattern), + ); - if (!specFilePaths.length) { - throw Error("No spec files found."); - } - const specsToRun = splitSpecs( - specFilePaths, - totalRunnersCount, - currentRunner, - ); - return specsToRun.join(","); - } catch (err) { - console.error(err); - process.exit(1); + if (!specFilePaths.length) { + throw Error("No spec files found."); } + const specsToRun = splitSpecs( + specFilePaths, + totalRunnersCount, + currentRunner, + ); + return specsToRun; + } catch (err) { + console.error(err); + process.exit(1); } } @@ -135,53 +131,42 @@ function getEnvValue( // This is to fetch the env variables from CI function getArgs() { return { - totalRunners: getEnvNumber("TOTAL_RUNNERS", { required: true }), - thisRunner: getEnvNumber("THIS_RUNNER", { required: true }), - specsPattern: getEnvValue("CYPRESS_SPEC_PATTERN", { required: true }), - env: getEnvValue("CYPRESS_ENV", { required: false }), - configFile: getEnvValue("CYPRESS_CONFIG_FILE", { required: false }), - headless: getEnvValue("CYPRESS_HEADLESS", { required: false }), - browser: getEnvValue("CYPRESS_BROWSER", { required: false }), + totalRunners: getEnvValue("TOTAL_RUNNERS", { required: false }), + thisRunner: getEnvValue("THIS_RUNNER", { required: false }), + cypressSpecs: getEnvValue("CYPRESS_SPECS", { required: false }), }; } -// This will finally segregate the specs and run the command to execute cypress -(async () => { +export async function cypressSplit(on: any, config: any) { try { - const { - browser, - configFile, - env, - headless, - specsPattern, - thisRunner, - totalRunners, - } = getArgs(); + let currentRunner = 0; + let allRunners = 1; + let specPattern = await config.specPattern; + const ignorePattern = await config.excludeSpecPattern; + const { cypressSpecs, thisRunner, totalRunners } = getArgs(); - let command = "yarn cypress run "; - command += browser != "" ? `--browser ${browser} ` : ""; - command += configFile != "" ? `--config-file ${configFile} ` : ""; - command += env != "" ? `--env ${env} ` : ""; - command += headless == "false" ? `--headed ` : ""; + if (cypressSpecs != "") + specPattern = cypressSpecs?.split(",").filter((val) => val !== ""); - const specs = await getSpecsToRun(totalRunners, thisRunner, specsPattern); - const final_command = `${command} --spec "${specs}"`; - const commandProcess = exec(final_command); - - // pipe output because we want to see the results of the run - if (commandProcess.stdout) { - commandProcess.stdout.pipe(process.stdout); + if (totalRunners != "") { + currentRunner = Number(thisRunner); + allRunners = Number(totalRunners); } - if (commandProcess.stderr) { - commandProcess.stderr.pipe(process.stderr); - } + const specs = await getSpecsToRun( + allRunners, + currentRunner, + specPattern, + ignorePattern, + ); - commandProcess.on("exit", (code) => { - process.exit(code || 0); - }); + if (specs.length > 0) { + config.specPattern = specs.length == 1 ? specs[0] : specs; + } else { + config.specPattern = "cypress/scripts/no_spec.ts"; + } + return config; } catch (err) { - console.error(err); - process.exit(1); + console.log(err); } -})(); +} diff --git a/app/client/cypress/plugins/index.js b/app/client/cypress/plugins/index.js index 26737e8c26..eb6a9f156a 100644 --- a/app/client/cypress/plugins/index.js +++ b/app/client/cypress/plugins/index.js @@ -12,6 +12,7 @@ const { } = require("cypress-image-snapshot/plugin"); const { tagify } = require("cypress-tags"); const { cypressHooks } = require("../scripts/cypress-hooks"); +const { cypressSplit } = require("../cypress-split"); // *********************************************************** // This example plugins/index.js can be used to load plugins // @@ -29,7 +30,7 @@ const { cypressHooks } = require("../scripts/cypress-hooks"); * @type {Cypress.PluginConfig} */ -module.exports = (on, config) => { +module.exports = async (on, config) => { // on("task", { // isFileExist, // }); @@ -45,9 +46,6 @@ module.exports = (on, config) => { on("file:preprocessor", tagify(config)); addMatchImageSnapshotPlugin(on, config); - if (process.env["RUNID"]) { - cypressHooks(on, config); - } on("before:browser:launch", (browser = {}, launchOptions) => { /* @@ -217,5 +215,10 @@ module.exports = (on, config) => { }, }); + if (process.env["RUNID"]) { + config = await cypressSplit(on, config); + cypressHooks(on, config); + } + return config; }; diff --git a/app/client/cypress/scripts/cypress-hooks.js b/app/client/cypress/scripts/cypress-hooks.js index 372db2e27a..fb509f6516 100644 --- a/app/client/cypress/scripts/cypress-hooks.js +++ b/app/client/cypress/scripts/cypress-hooks.js @@ -135,11 +135,13 @@ async function cypressHooks(on, config) { specData.matrixId = matrix.id; const dbClient = await configureDbClient().connect(); try { - const specResponse = await dbClient.query( - 'INSERT INTO public.specs ("name", "matrixId") VALUES ($1, $2) RETURNING id', - [specData.name, matrix.id], - ); - specData.specId = specResponse.rows[0].id; // Save the inserted spec ID for later updates + if (!specData.name.includes("no_spec.ts")) { + const specResponse = await dbClient.query( + 'INSERT INTO public.specs ("name", "matrixId") VALUES ($1, $2) RETURNING id', + [specData.name, matrix.id], + ); + specData.specId = specResponse.rows[0].id; // Save the inserted spec ID for later updates + } } catch (err) { console.log(err); } finally { @@ -154,62 +156,66 @@ async function cypressHooks(on, config) { specData.pending = results.stats.pending; specData.skipped = results.stats.skipped; specData.status = results.stats.failures > 0 ? "fail" : "pass"; + specData.duration = results.stats.wallClockDuration; const dbClient = await configureDbClient().connect(); try { - await dbClient.query( - 'UPDATE public.specs SET "testCount" = $1, "passes" = $2, "failed" = $3, "skipped" = $4, "pending" = $5, "status" = $6 WHERE id = $7', - [ - results.stats.tests, - results.stats.passes, - results.stats.failures, - results.stats.skipped, - results.stats.pending, - specData.status, - specData.specId, - ], - ); - for (const test of results.tests) { - const testResponse = await dbClient.query( - `INSERT INTO public.tests ("name", "specId", "status", "retries", "retryData") VALUES ($1, $2, $3, $4, $5) RETURNING id`, + if (!specData.name.includes("no_spec.ts")) { + await dbClient.query( + 'UPDATE public.specs SET "testCount" = $1, "passes" = $2, "failed" = $3, "skipped" = $4, "pending" = $5, "status" = $6, "duration" = $7 WHERE id = $8', [ - test.title[1], + results.stats.tests, + results.stats.passes, + results.stats.failures, + results.stats.skipped, + results.stats.pending, + specData.status, + specData.duration, specData.specId, - test.state, - test.attempts.length, - JSON.stringify(test.attempts), ], ); - if ( - test.attempts.some((attempt) => attempt.state === "failed") && - results.screenshots - ) { - const out = results.screenshots.filter( - (scr) => scr.testId === test.testId, + for (const test of results.tests) { + const testResponse = await dbClient.query( + `INSERT INTO public.tests ("name", "specId", "status", "retries", "retryData") VALUES ($1, $2, $3, $4, $5) RETURNING id`, + [ + test.title[1], + specData.specId, + test.state, + test.attempts.length, + JSON.stringify(test.attempts), + ], ); - console.log("Uploading screenshots..."); - for (const scr of out) { - const key = `${testResponse.rows[0].id}_${specData.specId}_${ - scr.testAttemptIndex + 1 - }`; - Promise.all([uploadToS3(s3, scr.path, key)]).catch((error) => { - console.log("Error in uploading screenshots:", error); - }); + if ( + test.attempts.some((attempt) => attempt.state === "failed") && + results.screenshots + ) { + const out = results.screenshots.filter( + (scr) => scr.testId === test.testId, + ); + console.log("Uploading screenshots..."); + for (const scr of out) { + const key = `${testResponse.rows[0].id}_${specData.specId}_${ + scr.testAttemptIndex + 1 + }`; + Promise.all([uploadToS3(s3, scr.path, key)]).catch((error) => { + console.log("Error in uploading screenshots:", error); + }); + } } } - } - if ( - results.tests.some((test) => - test.attempts.some((attempt) => attempt.state === "failed"), - ) && - results.video - ) { - console.log("Uploading video..."); - const key = `${specData.specId}`; - Promise.all([uploadToS3(s3, results.video, key)]).catch((error) => { - console.log("Error in uploading video:", error); - }); + if ( + results.tests.some((test) => + test.attempts.some((attempt) => attempt.state === "failed"), + ) && + results.video + ) { + console.log("Uploading video..."); + const key = `${specData.specId}`; + Promise.all([uploadToS3(s3, results.video, key)]).catch((error) => { + console.log("Error in uploading video:", error); + }); + } } } catch (err) { console.log(err); diff --git a/app/client/cypress/scripts/no_spec.ts b/app/client/cypress/scripts/no_spec.ts new file mode 100644 index 0000000000..98f3e69d0a --- /dev/null +++ b/app/client/cypress/scripts/no_spec.ts @@ -0,0 +1 @@ +// This file is to run when no cypress specs found. diff --git a/app/client/cypress_ci_hosted.config.ts b/app/client/cypress_ci_hosted.config.ts index aadf98709b..d42ea55e8b 100644 --- a/app/client/cypress_ci_hosted.config.ts +++ b/app/client/cypress_ci_hosted.config.ts @@ -1,4 +1,5 @@ import { defineConfig } from "cypress"; +import fs from "fs"; export default defineConfig({ defaultCommandTimeout: 30000, @@ -9,11 +10,15 @@ export default defineConfig({ videoUploadOnPasses: false, numTestsKeptInMemory: 5, experimentalMemoryManagement: true, + reporter: "cypress-mochawesome-reporter", reporterOptions: { reportDir: "results", - overwrite: false, - html: true, - json: false, + charts: true, + reportPageTitle: "Cypress-report", + videoOnFailOnly: true, + embeddedScreenshots: true, + inlineAssets: true, + saveAllAttempts: true, }, chromeWebSecurity: false, viewportHeight: 1100, @@ -26,9 +31,28 @@ export default defineConfig({ e2e: { baseUrl: "https://regression.test.appsmith.com/", setupNodeEvents(on, config) { + require("cypress-mochawesome-reporter/plugin")(on); + on( + "after:spec", + (spec: Cypress.Spec, results: CypressCommandLine.RunResult) => { + if (results && results.video) { + // Do we have failures for any retry attempts? + const failures = results.tests.some((test) => + test.attempts.some((attempt) => attempt.state === "failed"), + ); + if (!failures) { + // delete the video if the spec passed and no tests retried + fs.unlinkSync(results.video); + } + } + }, + ); return require("./cypress/plugins/index.js")(on, config); }, - specPattern: "cypress/e2e/**/*.{js,ts}", + specPattern: [ + "cypress/e2e/Sanity/Datasources/Airtable_Basic_Spec.ts", + "cypress/e2e/GSheet/**/**/*", + ], testIsolation: false, excludeSpecPattern: ["cypress/e2e/**/spec_utility.ts"], },