diff --git a/.github/workflows/ci-test-custom-script.yml b/.github/workflows/ci-test-custom-script.yml index ef0e376a56..7149de364b 100644 --- a/.github/workflows/ci-test-custom-script.yml +++ b/.github/workflows/ci-test-custom-script.yml @@ -370,6 +370,7 @@ jobs: CYPRESS_VERIFY_TIMEOUT: 100000 COMMIT_INFO_MESSAGE: ${{ env.COMMIT_INFO_MESSAGE }} THIS_RUNNER: ${{ strategy.job-index }} + TOTAL_RUNNERS: ${{ strategy.job-total }} RUNID: ${{ github.run_id }} ATTEMPT_NUMBER: ${{ github.run_attempt }} REPOSITORY: ${{ github.repository }} @@ -385,6 +386,7 @@ jobs: CYPRESS_S3_SECRET: ${{ secrets.CYPRESS_S3_SECRET }} CYPRESS_grepTags: ${{ inputs.tags }} # This is a comma separated list of tags to run a subset of the suite CYPRESS_SKIP_FLAKY: true + CYPRESS_STATIC_ALLOCATION: true DEBUG: ${{secrets.CYPRESS_GREP_DEBUG }} # This is derived from secrets so that we can toggle it without having to change any workflow. Only acceptable value is: @cypress/grep with: browser: ${{ env.BROWSER_PATH }} diff --git a/.github/workflows/ci-test-hosted.yml b/.github/workflows/ci-test-hosted.yml index 3006ea4586..59ab1b42bb 100644 --- a/.github/workflows/ci-test-hosted.yml +++ b/.github/workflows/ci-test-hosted.yml @@ -243,6 +243,7 @@ jobs: CYPRESS_DB_PWD: ${{ secrets.CYPRESS_DB_PWD }} CYPRESS_S3_ACCESS: ${{ secrets.CYPRESS_S3_ACCESS }} CYPRESS_S3_SECRET: ${{ secrets.CYPRESS_S3_SECRET }} + CYPRESS_STATIC_ALLOCATION: true with: install: false config-file: cypress_ci_hosted.config.ts diff --git a/.github/workflows/ci-test-limited.yml b/.github/workflows/ci-test-limited.yml index a43c783813..aa5b4fbb6a 100644 --- a/.github/workflows/ci-test-limited.yml +++ b/.github/workflows/ci-test-limited.yml @@ -343,6 +343,7 @@ jobs: TAG: ${{ github.event_name }} BRANCH: ${{ env.COMMIT_INFO_BRANCH }} THIS_RUNNER: ${{ strategy.job-index }} + TOTAL_RUNNERS: ${{ strategy.job-total }} CYPRESS_SPECS: ${{ env.specs_to_run }} CYPRESS_RERUN: ${{steps.run_result.outputs.rerun}} CYPRESS_DB_USER: ${{ secrets.CYPRESS_DB_USER }} @@ -351,6 +352,7 @@ jobs: CYPRESS_DB_PWD: ${{ secrets.CYPRESS_DB_PWD }} CYPRESS_S3_ACCESS: ${{ secrets.CYPRESS_S3_ACCESS }} CYPRESS_S3_SECRET: ${{ secrets.CYPRESS_S3_SECRET }} + CYPRESS_STATIC_ALLOCATION: true with: browser: ${{ env.BROWSER_PATH }} install: false diff --git a/.github/workflows/ci-test-with-documentdb.yml b/.github/workflows/ci-test-with-documentdb.yml index 808b7e47d9..8ee0dbe541 100644 --- a/.github/workflows/ci-test-with-documentdb.yml +++ b/.github/workflows/ci-test-with-documentdb.yml @@ -445,6 +445,7 @@ jobs: CYPRESS_DB_PWD: ${{ secrets.CYPRESS_DB_PWD }} CYPRESS_S3_ACCESS: ${{ secrets.CYPRESS_S3_ACCESS }} CYPRESS_S3_SECRET: ${{ secrets.CYPRESS_S3_SECRET }} + CYPRESS_STATIC_ALLOCATION: true with: install: false working-directory: app/client diff --git a/app/client/cypress/plugins/index.js b/app/client/cypress/plugins/index.js index 6ceb683e84..ccd03ec796 100644 --- a/app/client/cypress/plugins/index.js +++ b/app/client/cypress/plugins/index.js @@ -10,7 +10,8 @@ const { } = require("cypress-image-snapshot/plugin"); const { tagify } = require("cypress-tags"); const { cypressHooks } = require("../scripts/cypress-hooks"); -const { cypressSplit } = require("../scripts/cypress-split"); +const { dynamicSplit } = require("../scripts/cypress-split-dynamic"); +const { staticSplit } = require("../scripts/cypress-split-static"); // *********************************************************** // This example plugins/index.js can be used to load plugins // @@ -235,7 +236,10 @@ module.exports = async (on, config) => { console.log("config.specPattern:", config.specPattern); if (process.env["RUNID"]) { - config = await new cypressSplit().splitSpecs(on, config); + config = + process.env["CYPRESS_STATIC_ALLOCATION"] == "true" + ? await new staticSplit().splitSpecs(config) + : await new dynamicSplit().splitSpecs(config); cypressHooks(on, config); } diff --git a/app/client/cypress/scripts/cypress-split.ts b/app/client/cypress/scripts/cypress-split-dynamic.ts similarity index 89% rename from app/client/cypress/scripts/cypress-split.ts rename to app/client/cypress/scripts/cypress-split-dynamic.ts index 96fa8299f1..f79b89cea6 100644 --- a/app/client/cypress/scripts/cypress-split.ts +++ b/app/client/cypress/scripts/cypress-split-dynamic.ts @@ -1,36 +1,9 @@ -/* eslint-disable no-console */ import util from "./util"; -import globby from "globby"; -import minimatch from "minimatch"; -export class cypressSplit { +export class dynamicSplit { util = new util(); dbClient = this.util.configureDbClient(); - // This function will get all the spec paths using the pattern - private async getSpecFilePaths( - specPattern: any, - ignoreTestFiles: any, - ): Promise { - const files = globby.sync(specPattern, { - ignore: ignoreTestFiles, - }); - - // ignore the files that doesn't match - const ignorePatterns = [...(ignoreTestFiles || [])]; - - // a function which returns true if the file does NOT match - const doesNotMatchAllIgnoredPatterns = (file: string) => { - // using {dot: true} here so that folders with a '.' in them are matched - const MINIMATCH_OPTIONS = { dot: true, matchBase: true }; - return ignorePatterns.every((pattern) => { - return !minimatch(file, pattern, MINIMATCH_OPTIONS); - }); - }; - const filtered = files.filter(doesNotMatchAllIgnoredPatterns); - return filtered; - } - private async getSpecsWithTime(specs: string[], attemptId: number) { const client = await this.dbClient.connect(); const defaultDuration = 180000; @@ -68,7 +41,7 @@ export class cypressSplit { attemptId: number, ): Promise { try { - const specFilePaths = await this.getSpecFilePaths( + const specFilePaths = await this.util.getSpecFilePaths( specPattern, ignorePattern, ); @@ -306,10 +279,7 @@ export class cypressSplit { return new Promise((resolve) => setTimeout(resolve, ms)); } - public async splitSpecs( - on: Cypress.PluginEvents, - config: Cypress.PluginConfigOptions, - ) { + public async splitSpecs(config: Cypress.PluginConfigOptions) { try { let specPattern = config.specPattern; let ignorePattern: string | string[] = config.excludeSpecPattern; diff --git a/app/client/cypress/scripts/cypress-split-static.ts b/app/client/cypress/scripts/cypress-split-static.ts new file mode 100644 index 0000000000..5e75b51901 --- /dev/null +++ b/app/client/cypress/scripts/cypress-split-static.ts @@ -0,0 +1,237 @@ +import util from "./util"; + +export class staticSplit { + util = new util(); + dbClient = this.util.configureDbClient(); + + private async getSpecsWithTime(specs: string[], attemptId: number) { + const client = await this.dbClient.connect(); + const defaultDuration = 180000; + const specsMap = new Map(); + try { + const queryRes = await client.query( + 'SELECT * FROM public."spec_avg_duration" ORDER BY duration DESC', + ); + + queryRes.rows.forEach((obj) => { + specsMap.set(obj.name, obj); + }); + + const allSpecsWithDuration = specs.map((spec) => { + const match = specsMap.get(spec); + return match ? match : { name: spec, duration: defaultDuration }; + }); + + return await this.util.divideSpecsIntoBalancedGroups( + allSpecsWithDuration, + Number(this.util.getVars().totalRunners), + ); + } catch (err) { + console.log(err); + } finally { + client.release(); + } + } + + // This function will finally get the specs as a comma separated string to pass the specs to the command + private async getSpecsToRun( + specPattern: string | string[] = "cypress/e2e/**/**/*.{js,ts}", + ignorePattern: string | string[], + attemptId: number, + ): Promise { + try { + const specFilePaths = await this.util.getSpecFilePaths( + specPattern, + ignorePattern, + ); + + if (this.util.getVars().cypressRerun === "true") { + return specFilePaths; + } else { + const specsToRun = await this.getSpecsWithTime( + specFilePaths, + attemptId, + ); + return specsToRun === undefined || specsToRun.length === 0 + ? [] + : specsToRun[Number(this.util.getVars().thisRunner)].map( + (spec) => spec.name, + ); + } + } catch (err) { + console.error(err); + process.exit(1); + } + } + + private async createAttempt() { + const client = await this.dbClient.connect(); + try { + const runResponse = await client.query( + `INSERT INTO public."attempt" ("workflowId", "attempt", "repo", "committer", "type", "commitMsg", "branch") + VALUES ($1, $2, $3, $4, $5, $6, $7) + ON CONFLICT ("workflowId", attempt) DO NOTHING + RETURNING id;`, + [ + this.util.getVars().runId, + this.util.getVars().attempt_number, + this.util.getVars().repository, + this.util.getVars().committer, + this.util.getVars().tag, + this.util.getVars().commitMsg, + this.util.getVars().branch, + ], + ); + + if (runResponse.rows.length > 0) { + return runResponse.rows[0].id; + } else { + const res = await client.query( + `SELECT id FROM public."attempt" WHERE "workflowId" = $1 AND attempt = $2`, + [this.util.getVars().runId, this.util.getVars().attempt_number], + ); + return res.rows[0].id; + } + } catch (err) { + console.log(err); + } finally { + client.release(); + } + } + + private async createMatrix(attemptId: number) { + const client = await this.dbClient.connect(); + try { + const matrixResponse = await client.query( + `INSERT INTO public."matrix" ("workflowId", "matrixId", "status", "attemptId") + VALUES ($1, $2, $3, $4) + ON CONFLICT ("matrixId", "attemptId") DO NOTHING + RETURNING id;`, + [ + this.util.getVars().runId, + this.util.getVars().thisRunner, + "started", + attemptId, + ], + ); + return matrixResponse.rows[0].id; + } catch (err) { + console.log(err); + } finally { + client.release(); + } + } + + private async getFailedSpecsFromPreviousRun( + runnerId = Number(this.util.getVars().thisRunner), + workflowId = Number(this.util.getVars().runId), + attempt_number = Number(this.util.getVars().attempt_number) - 1, + ) { + const client = await this.dbClient.connect(); + try { + const dbRes = await client.query( + `SELECT name FROM public."specs" + WHERE "matrixId" = + (SELECT id FROM public."matrix" + WHERE "attemptId" = ( + SELECT id FROM public."attempt" WHERE "workflowId" = $1 and "attempt" = $2 + ) AND "matrixId" = $3 + ) AND status IN ('fail', 'queued', 'in-progress')`, + [workflowId, attempt_number, runnerId], + ); + const specs: string[] = + dbRes.rows.length > 0 ? dbRes.rows.map((row) => row.name) : []; + return specs; + } catch (err) { + console.log(err); + } finally { + client.release(); + } + } + + private async addSpecsToMatrix(matrixId: number, specs: string[]) { + const client = await this.dbClient.connect(); + try { + for (const spec of specs) { + const res = await client.query( + `INSERT INTO public."specs" ("name", "matrixId", "status") VALUES ($1, $2, $3) RETURNING id`, + [spec, matrixId, "queued"], + ); + } + } catch (err) { + console.log(err); + } finally { + client.release(); + } + } + + private async updateTheSpecsForMatrix(attemptId: number, specs: string[]) { + const client = await this.dbClient.connect(); + try { + if (specs.length > 0) { + const matrixRes = await this.createMatrix(attemptId); + await this.addSpecsToMatrix(matrixRes, specs); + } + } catch (err) { + console.log(err); + } finally { + client.release(); + } + } + + private async getFlakySpecs() { + const client = await this.dbClient.connect(); + try { + const dbRes = await client.query( + `SELECT spec FROM public."flaky_specs" WHERE skip=true`, + ); + const specs: string[] = + dbRes.rows.length > 0 ? dbRes.rows.map((row) => row.spec) : []; + return specs; + } catch (err) { + console.log(err); + } finally { + client.release(); + } + } + + public async splitSpecs(config: Cypress.PluginConfigOptions) { + try { + let specPattern = config.specPattern; + let ignorePattern: string | string[] = config.excludeSpecPattern; + const cypressSpecs = this.util.getVars().cypressSpecs; + const defaultSpec = "cypress/scripts/no_spec.ts"; + + if (cypressSpecs != "") { + specPattern = cypressSpecs?.split(",").filter((val) => val !== ""); + } + + if (this.util.getVars().cypressRerun === "true") { + specPattern = + (await this.getFailedSpecsFromPreviousRun()) ?? defaultSpec; + } + + if (this.util.getVars().cypressSkipFlaky === "true") { + let specsToSkip = (await this.getFlakySpecs()) ?? []; + ignorePattern = [...ignorePattern, ...specsToSkip]; + } + + const attempt = await this.createAttempt(); + const specs = + (await this.getSpecsToRun(specPattern, ignorePattern, attempt)) ?? []; + console.log("SPECS TO RUN ----------> :", specs); + if (specs.length > 0 && !specs.includes(defaultSpec)) { + config.specPattern = specs.length == 1 ? specs[0] : specs; + } else { + config.specPattern = defaultSpec; + } + await this.updateTheSpecsForMatrix(attempt, specs); + + return config; + } catch (err) { + console.log(err); + } finally { + this.dbClient.end(); + } + } +} diff --git a/app/client/cypress/scripts/util.ts b/app/client/cypress/scripts/util.ts index 181ed45ef6..44321cf4be 100644 --- a/app/client/cypress/scripts/util.ts +++ b/app/client/cypress/scripts/util.ts @@ -4,6 +4,8 @@ import { Upload } from "@aws-sdk/lib-storage"; import fs from "fs"; import { Octokit } from "@octokit/rest"; import fetch from "node-fetch"; +import globby from "globby"; +import minimatch from "minimatch"; export interface DataItem { name: string; @@ -37,6 +39,9 @@ export default class util { cypressSkipFlaky: this.getEnvValue("CYPRESS_SKIP_FLAKY", { required: false, }), + staticAllocation: this.getEnvValue("CYPRESS_STATIC_ALLOCATION", { + required: false, + }), }; } @@ -69,6 +74,30 @@ export default class util { return groups; } + // This function will get all the spec paths using the pattern + public async getSpecFilePaths( + specPattern: any, + ignoreTestFiles: any, + ): Promise { + const files = globby.sync(specPattern, { + ignore: ignoreTestFiles, + }); + + // ignore the files that doesn't match + const ignorePatterns = [...(ignoreTestFiles || [])]; + + // a function which returns true if the file does NOT match + const doesNotMatchAllIgnoredPatterns = (file: string) => { + // using {dot: true} here so that folders with a '.' in them are matched + const MINIMATCH_OPTIONS = { dot: true, matchBase: true }; + return ignorePatterns.every((pattern) => { + return !minimatch(file, pattern, MINIMATCH_OPTIONS); + }); + }; + const filtered = files.filter(doesNotMatchAllIgnoredPatterns); + return filtered; + } + public getEnvValue(varName: string, { required = true }): string { if (required && process.env[varName] === undefined) { throw Error(