ci: Allow static split of cypress specs (#29653)

## Description
- Allowing static split of cypress specs in CI runs.
- Optimised the spec allocation by considering the duration history for
each spec.
- Updated the affecting workflows to utilise the static split

#### Type of change
- Workflows
- Cypress-split pugin
## Testing
- Workflow run

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

- **New Features**
- Introduced a new environment variable to optimize test resource
allocation during CI/CD processes.
- Enhanced Cypress testing with dynamic and static test splitting
strategies.

- **Refactor**
- Refactored Cypress plugin configuration to support conditional test
splitting based on the environment variable.

- **Chores**
- Updated GitHub Actions workflows to include new environment variables
for test runs.

- **Documentation**
- Updated internal documentation to reflect changes in test setup and
execution strategies.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

.github/workflows/ci-test-custom-script.yml

@@ -370,6 +370,7 @@ jobs:
           CYPRESS_VERIFY_TIMEOUT: 100000
           COMMIT_INFO_MESSAGE: ${{ env.COMMIT_INFO_MESSAGE }}
           THIS_RUNNER: ${{ strategy.job-index }}
+          TOTAL_RUNNERS: ${{ strategy.job-total }}
           RUNID: ${{ github.run_id }}
           ATTEMPT_NUMBER: ${{ github.run_attempt }}
           REPOSITORY: ${{ github.repository }}
@@ -385,6 +386,7 @@ jobs:
           CYPRESS_S3_SECRET: ${{ secrets.CYPRESS_S3_SECRET }}
           CYPRESS_grepTags: ${{ inputs.tags }} # This is a comma separated list of tags to run a subset of the suite
           CYPRESS_SKIP_FLAKY: true
+          CYPRESS_STATIC_ALLOCATION: true
           DEBUG: ${{secrets.CYPRESS_GREP_DEBUG }} # This is derived from secrets so that we can toggle it without having to change any workflow. Only acceptable value is: @cypress/grep
         with:
           browser: ${{ env.BROWSER_PATH }}

.github/workflows/ci-test-hosted.yml

@@ -243,6 +243,7 @@ jobs:
           CYPRESS_DB_PWD: ${{ secrets.CYPRESS_DB_PWD }}
           CYPRESS_S3_ACCESS: ${{ secrets.CYPRESS_S3_ACCESS }}
           CYPRESS_S3_SECRET: ${{ secrets.CYPRESS_S3_SECRET }}
+          CYPRESS_STATIC_ALLOCATION: true
         with:
           install: false
           config-file: cypress_ci_hosted.config.ts

.github/workflows/ci-test-limited.yml

@@ -343,6 +343,7 @@ jobs:
           TAG: ${{ github.event_name }}
           BRANCH: ${{ env.COMMIT_INFO_BRANCH }}
           THIS_RUNNER: ${{ strategy.job-index }}
+          TOTAL_RUNNERS: ${{ strategy.job-total }}
           CYPRESS_SPECS: ${{ env.specs_to_run }}
           CYPRESS_RERUN: ${{steps.run_result.outputs.rerun}}
           CYPRESS_DB_USER: ${{ secrets.CYPRESS_DB_USER }}
@@ -351,6 +352,7 @@ jobs:
           CYPRESS_DB_PWD: ${{ secrets.CYPRESS_DB_PWD }}
           CYPRESS_S3_ACCESS: ${{ secrets.CYPRESS_S3_ACCESS }}
           CYPRESS_S3_SECRET: ${{ secrets.CYPRESS_S3_SECRET }}
+          CYPRESS_STATIC_ALLOCATION: true
         with:
           browser: ${{ env.BROWSER_PATH }}
           install: false

.github/workflows/ci-test-with-documentdb.yml

@@ -445,6 +445,7 @@ jobs:
           CYPRESS_DB_PWD: ${{ secrets.CYPRESS_DB_PWD }}
           CYPRESS_S3_ACCESS: ${{ secrets.CYPRESS_S3_ACCESS }}
           CYPRESS_S3_SECRET: ${{ secrets.CYPRESS_S3_SECRET }}
+          CYPRESS_STATIC_ALLOCATION: true
         with:
           install: false
           working-directory: app/client

app/client/cypress/plugins/index.js

@@ -10,7 +10,8 @@ const {
 } = require("cypress-image-snapshot/plugin");
 const { tagify } = require("cypress-tags");
 const { cypressHooks } = require("../scripts/cypress-hooks");
-const { cypressSplit } = require("../scripts/cypress-split");
+const { dynamicSplit } = require("../scripts/cypress-split-dynamic");
+const { staticSplit } = require("../scripts/cypress-split-static");
 // ***********************************************************
 // This example plugins/index.js can be used to load plugins
 //
@@ -235,7 +236,10 @@ module.exports = async (on, config) => {
   console.log("config.specPattern:", config.specPattern);
 
   if (process.env["RUNID"]) {
-    config = await new cypressSplit().splitSpecs(on, config);
+    config =
+      process.env["CYPRESS_STATIC_ALLOCATION"] == "true"
+        ? await new staticSplit().splitSpecs(config)
+        : await new dynamicSplit().splitSpecs(config);
     cypressHooks(on, config);
   }
 

app/client/cypress/scripts/cypress-split-dynamic.ts

@@ -1,36 +1,9 @@
-/* eslint-disable no-console */
 import util from "./util";
-import globby from "globby";
-import minimatch from "minimatch";
 
-export class cypressSplit {
+export class dynamicSplit {
   util = new util();
   dbClient = this.util.configureDbClient();
 
-  // This function will get all the spec paths using the pattern
-  private async getSpecFilePaths(
-    specPattern: any,
-    ignoreTestFiles: any,
-  ): Promise<string[]> {
-    const files = globby.sync(specPattern, {
-      ignore: ignoreTestFiles,
-    });
-
-    // ignore the files that doesn't match
-    const ignorePatterns = [...(ignoreTestFiles || [])];
-
-    // a function which returns true if the file does NOT match
-    const doesNotMatchAllIgnoredPatterns = (file: string) => {
-      // using {dot: true} here so that folders with a '.' in them are matched
-      const MINIMATCH_OPTIONS = { dot: true, matchBase: true };
-      return ignorePatterns.every((pattern) => {
-        return !minimatch(file, pattern, MINIMATCH_OPTIONS);
-      });
-    };
-    const filtered = files.filter(doesNotMatchAllIgnoredPatterns);
-    return filtered;
-  }
-
   private async getSpecsWithTime(specs: string[], attemptId: number) {
     const client = await this.dbClient.connect();
     const defaultDuration = 180000;
@@ -68,7 +41,7 @@ export class cypressSplit {
     attemptId: number,
   ): Promise<string[]> {
     try {
-      const specFilePaths = await this.getSpecFilePaths(
+      const specFilePaths = await this.util.getSpecFilePaths(
         specPattern,
         ignorePattern,
       );
@@ -306,10 +279,7 @@ export class cypressSplit {
     return new Promise((resolve) => setTimeout(resolve, ms));
   }
 
-  public async splitSpecs(
-    on: Cypress.PluginEvents,
-    config: Cypress.PluginConfigOptions,
-  ) {
+  public async splitSpecs(config: Cypress.PluginConfigOptions) {
     try {
       let specPattern = config.specPattern;
       let ignorePattern: string | string[] = config.excludeSpecPattern;

app/client/cypress/scripts/cypress-split-static.ts

@@ -0,0 +1,237 @@
+import util from "./util";
+
+export class staticSplit {
+  util = new util();
+  dbClient = this.util.configureDbClient();
+
+  private async getSpecsWithTime(specs: string[], attemptId: number) {
+    const client = await this.dbClient.connect();
+    const defaultDuration = 180000;
+    const specsMap = new Map();
+    try {
+      const queryRes = await client.query(
+        'SELECT * FROM public."spec_avg_duration" ORDER BY duration DESC',
+      );
+
+      queryRes.rows.forEach((obj) => {
+        specsMap.set(obj.name, obj);
+      });
+
+      const allSpecsWithDuration = specs.map((spec) => {
+        const match = specsMap.get(spec);
+        return match ? match : { name: spec, duration: defaultDuration };
+      });
+
+      return await this.util.divideSpecsIntoBalancedGroups(
+        allSpecsWithDuration,
+        Number(this.util.getVars().totalRunners),
+      );
+    } catch (err) {
+      console.log(err);
+    } finally {
+      client.release();
+    }
+  }
+
+  // This function will finally get the specs as a comma separated string to pass the specs to the command
+  private async getSpecsToRun(
+    specPattern: string | string[] = "cypress/e2e/**/**/*.{js,ts}",
+    ignorePattern: string | string[],
+    attemptId: number,
+  ): Promise<string[]> {
+    try {
+      const specFilePaths = await this.util.getSpecFilePaths(
+        specPattern,
+        ignorePattern,
+      );
+
+      if (this.util.getVars().cypressRerun === "true") {
+        return specFilePaths;
+      } else {
+        const specsToRun = await this.getSpecsWithTime(
+          specFilePaths,
+          attemptId,
+        );
+        return specsToRun === undefined || specsToRun.length === 0
+          ? []
+          : specsToRun[Number(this.util.getVars().thisRunner)].map(
+              (spec) => spec.name,
+            );
+      }
+    } catch (err) {
+      console.error(err);
+      process.exit(1);
+    }
+  }
+
+  private async createAttempt() {
+    const client = await this.dbClient.connect();
+    try {
+      const runResponse = await client.query(
+        `INSERT INTO public."attempt" ("workflowId", "attempt", "repo", "committer", "type", "commitMsg", "branch")
+          VALUES ($1, $2, $3, $4, $5, $6, $7)
+          ON CONFLICT ("workflowId", attempt) DO NOTHING
+          RETURNING id;`,
+        [
+          this.util.getVars().runId,
+          this.util.getVars().attempt_number,
+          this.util.getVars().repository,
+          this.util.getVars().committer,
+          this.util.getVars().tag,
+          this.util.getVars().commitMsg,
+          this.util.getVars().branch,
+        ],
+      );
+
+      if (runResponse.rows.length > 0) {
+        return runResponse.rows[0].id;
+      } else {
+        const res = await client.query(
+          `SELECT id FROM public."attempt" WHERE "workflowId" = $1 AND attempt = $2`,
+          [this.util.getVars().runId, this.util.getVars().attempt_number],
+        );
+        return res.rows[0].id;
+      }
+    } catch (err) {
+      console.log(err);
+    } finally {
+      client.release();
+    }
+  }
+
+  private async createMatrix(attemptId: number) {
+    const client = await this.dbClient.connect();
+    try {
+      const matrixResponse = await client.query(
+        `INSERT INTO public."matrix" ("workflowId", "matrixId", "status", "attemptId")
+          VALUES ($1, $2, $3, $4)
+          ON CONFLICT ("matrixId", "attemptId") DO NOTHING
+          RETURNING id;`,
+        [
+          this.util.getVars().runId,
+          this.util.getVars().thisRunner,
+          "started",
+          attemptId,
+        ],
+      );
+      return matrixResponse.rows[0].id;
+    } catch (err) {
+      console.log(err);
+    } finally {
+      client.release();
+    }
+  }
+
+  private async getFailedSpecsFromPreviousRun(
+    runnerId = Number(this.util.getVars().thisRunner),
+    workflowId = Number(this.util.getVars().runId),
+    attempt_number = Number(this.util.getVars().attempt_number) - 1,
+  ) {
+    const client = await this.dbClient.connect();
+    try {
+      const dbRes = await client.query(
+        `SELECT name FROM public."specs" 
+      WHERE "matrixId" = 
+      (SELECT id FROM public."matrix" 
+       WHERE "attemptId" = (
+         SELECT id FROM public."attempt" WHERE "workflowId" = $1 and "attempt" = $2
+       ) AND "matrixId" = $3
+      ) AND status IN ('fail', 'queued', 'in-progress')`,
+        [workflowId, attempt_number, runnerId],
+      );
+      const specs: string[] =
+        dbRes.rows.length > 0 ? dbRes.rows.map((row) => row.name) : [];
+      return specs;
+    } catch (err) {
+      console.log(err);
+    } finally {
+      client.release();
+    }
+  }
+
+  private async addSpecsToMatrix(matrixId: number, specs: string[]) {
+    const client = await this.dbClient.connect();
+    try {
+      for (const spec of specs) {
+        const res = await client.query(
+          `INSERT INTO public."specs" ("name", "matrixId", "status") VALUES ($1, $2, $3) RETURNING id`,
+          [spec, matrixId, "queued"],
+        );
+      }
+    } catch (err) {
+      console.log(err);
+    } finally {
+      client.release();
+    }
+  }
+
+  private async updateTheSpecsForMatrix(attemptId: number, specs: string[]) {
+    const client = await this.dbClient.connect();
+    try {
+      if (specs.length > 0) {
+        const matrixRes = await this.createMatrix(attemptId);
+        await this.addSpecsToMatrix(matrixRes, specs);
+      }
+    } catch (err) {
+      console.log(err);
+    } finally {
+      client.release();
+    }
+  }
+
+  private async getFlakySpecs() {
+    const client = await this.dbClient.connect();
+    try {
+      const dbRes = await client.query(
+        `SELECT spec FROM public."flaky_specs" WHERE skip=true`,
+      );
+      const specs: string[] =
+        dbRes.rows.length > 0 ? dbRes.rows.map((row) => row.spec) : [];
+      return specs;
+    } catch (err) {
+      console.log(err);
+    } finally {
+      client.release();
+    }
+  }
+
+  public async splitSpecs(config: Cypress.PluginConfigOptions) {
+    try {
+      let specPattern = config.specPattern;
+      let ignorePattern: string | string[] = config.excludeSpecPattern;
+      const cypressSpecs = this.util.getVars().cypressSpecs;
+      const defaultSpec = "cypress/scripts/no_spec.ts";
+
+      if (cypressSpecs != "") {
+        specPattern = cypressSpecs?.split(",").filter((val) => val !== "");
+      }
+
+      if (this.util.getVars().cypressRerun === "true") {
+        specPattern =
+          (await this.getFailedSpecsFromPreviousRun()) ?? defaultSpec;
+      }
+
+      if (this.util.getVars().cypressSkipFlaky === "true") {
+        let specsToSkip = (await this.getFlakySpecs()) ?? [];
+        ignorePattern = [...ignorePattern, ...specsToSkip];
+      }
+
+      const attempt = await this.createAttempt();
+      const specs =
+        (await this.getSpecsToRun(specPattern, ignorePattern, attempt)) ?? [];
+      console.log("SPECS TO RUN ----------> :", specs);
+      if (specs.length > 0 && !specs.includes(defaultSpec)) {
+        config.specPattern = specs.length == 1 ? specs[0] : specs;
+      } else {
+        config.specPattern = defaultSpec;
+      }
+      await this.updateTheSpecsForMatrix(attempt, specs);
+
+      return config;
+    } catch (err) {
+      console.log(err);
+    } finally {
+      this.dbClient.end();
+    }
+  }
+}

app/client/cypress/scripts/util.ts

@@ -4,6 +4,8 @@ import { Upload } from "@aws-sdk/lib-storage";
 import fs from "fs";
 import { Octokit } from "@octokit/rest";
 import fetch from "node-fetch";
+import globby from "globby";
+import minimatch from "minimatch";
 
 export interface DataItem {
   name: string;
@@ -37,6 +39,9 @@ export default class util {
       cypressSkipFlaky: this.getEnvValue("CYPRESS_SKIP_FLAKY", {
         required: false,
       }),
+      staticAllocation: this.getEnvValue("CYPRESS_STATIC_ALLOCATION", {
+        required: false,
+      }),
     };
   }
 
@@ -69,6 +74,30 @@ export default class util {
     return groups;
   }
 
+  // This function will get all the spec paths using the pattern
+  public async getSpecFilePaths(
+    specPattern: any,
+    ignoreTestFiles: any,
+  ): Promise<string[]> {
+    const files = globby.sync(specPattern, {
+      ignore: ignoreTestFiles,
+    });
+
+    // ignore the files that doesn't match
+    const ignorePatterns = [...(ignoreTestFiles || [])];
+
+    // a function which returns true if the file does NOT match
+    const doesNotMatchAllIgnoredPatterns = (file: string) => {
+      // using {dot: true} here so that folders with a '.' in them are matched
+      const MINIMATCH_OPTIONS = { dot: true, matchBase: true };
+      return ignorePatterns.every((pattern) => {
+        return !minimatch(file, pattern, MINIMATCH_OPTIONS);
+      });
+    };
+    const filtered = files.filter(doesNotMatchAllIgnoredPatterns);
+    return filtered;
+  }
+
   public getEnvValue(varName: string, { required = true }): string {
     if (required && process.env[varName] === undefined) {
       throw Error(