feat: Add app import utility and perform the initial setup on CI (#10050)

- Run all files in the tests folder in sequence
- Better error handling and saving of screenshots
- Organise and refactor code. WIP
- Improve the summary generator
- Add utility method to import an app
- Add a basic performance test on imported app

Co-authored-by: Satish Gandham <satish@appsmith.com>

.gitignore

@@ -17,4 +17,5 @@ app/client/cypress/locators/Widgets.json
 deploy/ansible/appsmith_playbook/inventory
 
 # performance tests
-app/client/perf/traces/*
+app/client/perf/traces/*
+.history

app/client/perf/package.json

@@ -9,7 +9,9 @@
   "author": "",
   "license": "ISC",
   "dependencies": {
+    "node-stdev": "^1.0.1",
     "puppeteer": "^12.0.1",
+    "sanitize-filename": "^1.6.3",
     "tracelib": "^1.0.1"
   }
 }

app/client/perf/src/index.js

@@ -0,0 +1,21 @@
+const glob = require("glob");
+const path = require("path");
+const { summaries } = require("./summary");
+var cp = require("child_process");
+var fs = require("fs");
+
+// Create the directory
+global.APP_ROOT = path.join(__dirname, ".."); //Going back one level from src folder to /perf
+const dir = `${APP_ROOT}/traces/reports`;
+if (!fs.existsSync(dir)) {
+  fs.mkdirSync(dir, { recursive: true });
+}
+
+glob("./tests/*.perf.js", {}, async function(er, files) {
+  // Initial setup
+  await cp.execSync(`node ./tests/initial-setup.js`, { stdio: "inherit" });
+  files.forEach(async (file) => {
+    await cp.execSync(`node ${file}`, { stdio: "inherit" }); // Logging to terminal, log it to a file in future?
+  });
+  summaries(`${APP_ROOT}/traces/reports`);
+});

app/client/perf/src/perf.js

@@ -1,21 +1,59 @@
 const Tracelib = require("tracelib");
 const puppeteer = require("puppeteer");
+var sanitize = require("sanitize-filename");
 const fs = require("fs");
+const path = require("path");
+
 const {
   delay,
   login,
   getFormattedTime,
   sortObjectKeys,
 } = require("./utils/utils");
-
+const selectors = {
+  appMoreIcon: "span.t--options-icon",
+  orgImportAppOption: '[data-cy*="t--org-import-app"]',
+  fileInput: "#fileInput",
+  importButton: '[data-cy*="t--org-import-app-button"]',
+  createNewApp: ".createnew",
+};
 module.exports = class Perf {
   constructor(launchOptions = {}) {
     this.launchOptions = {
+      defaultViewport: null,
+      args: ["--window-size=1920,1080"],
+      ignoreHTTPSErrors: true, // @todo Remove it after initial testing
       ...launchOptions,
     };
+
+    if (process.env.PERF_TEST_ENV === "dev") {
+      this.launchOptions.executablePath =
+        "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome";
+      this.launchOptions.devtools = true;
+      this.launchOptions.headless = false;
+    }
+
     this.traces = [];
-    this.traceInProgress = false;
+    this.currentTrace = null;
     this.browser = null;
+
+    // Initial setup
+    this.currentTestFile = process.argv[1]
+      .split("/")
+      .pop()
+      .replace(".perf.js", "");
+    global.APP_ROOT = path.join(__dirname, ".."); //Going back one level from src folder to /perf
+    process.on("unhandledRejection", async (reason, p) => {
+      console.error("Unhandled Rejection at: Promise", p, "reason:", reason);
+      const fileName = sanitize(
+        `${this.currentTestFile}__${this.currentTrace}`,
+      );
+      const screenshotPath = `${APP_ROOT}/traces/reports/${fileName}-${getFormattedTime()}.png`;
+      await this.page.screenshot({
+        path: screenshotPath,
+      });
+      this.browser.close();
+    });
   }
   /**
    * Launches the browser and, gives you the page
@@ -33,12 +71,12 @@ module.exports = class Perf {
   };
 
   startTrace = async (action = "foo") => {
-    if (this.traceInProgress) {
+    if (this.currentTrace) {
       console.warn("Trace progress. You can run only one trace at a time");
       return;
     }
 
-    this.traceInProgress = true;
+    this.currentTrace = action;
     await delay(3000, `before starting trace ${action}`);
     const path = `${APP_ROOT}/traces/${action}-${getFormattedTime()}-chrome-profile.json`;
     await this.page.tracing.start({
@@ -49,8 +87,8 @@ module.exports = class Perf {
   };
 
   stopTrace = async () => {
-    this.traceInProgress = false;
-    await delay(5000, "before stoping the trace");
+    this.currentTrace = null;
+    await delay(3000, "before stopping the trace");
     await this.page.tracing.stop();
   };
 
@@ -60,7 +98,7 @@ module.exports = class Perf {
   };
 
   loadDSL = async (dsl) => {
-    const selector = ".createnew";
+    const selector = selectors.createNewApp;
     await this.page.waitForSelector(selector);
     await this.page.click(selector);
     // We goto the newly created app.
@@ -106,6 +144,20 @@ module.exports = class Perf {
     });
   };
 
+  importApplication = async (jsonPath) => {
+    await this.page.waitForSelector(selectors.appMoreIcon);
+    await this.page.click(selectors.appMoreIcon);
+    await this.page.waitForSelector(selectors.orgImportAppOption);
+    await this.page.click(selectors.orgImportAppOption);
+
+    const elementHandle = await this.page.$(selectors.fileInput);
+    await elementHandle.uploadFile(jsonPath);
+    await this.page.click(selectors.importButton);
+
+    await this.page.waitForNavigation();
+    await this.page.reload();
+  };
+
   generateReport = async () => {
     const report = {};
     this.traces.forEach(({ path, action }) => {

app/client/perf/src/summary.js

@@ -1,5 +1,8 @@
 const fs = require("fs");
 const path = require("path");
+const sd = require("node-stdev");
+
+global.APP_ROOT = path.resolve(__dirname);
 
 exports.summaries = async (directory) => {
   const files = await fs.promises.readdir(directory);
@@ -14,62 +17,79 @@ exports.summaries = async (directory) => {
         if (!results[key]?.scripting) {
           results[key].scripting = [];
         }
-        results[key].scripting.push(content[key].summary.scripting);
+        results[key].scripting.push(
+          parseFloat(content[key].summary.scripting.toFixed(2)),
+        );
 
         if (!results[key]?.painting) {
           results[key].painting = [];
         }
-        results[key].painting.push(content[key].summary.painting);
+        results[key].painting.push(
+          parseFloat(content[key].summary.painting.toFixed(2)),
+        );
 
         if (!results[key]?.rendering) {
           results[key].rendering = [];
         }
-        results[key].rendering.push(content[key].summary.rendering);
+        results[key].rendering.push(
+          parseFloat(content[key].summary.rendering.toFixed(2)),
+        );
       });
     }
   });
-  generateReport(results);
+  generateMarkdown(results);
 };
 
-const generateReport = (results) => {
-  var size = 5;
+const getMaxSize = (results) => {
+  let size = 0;
   Object.keys(results).forEach((key) => {
     const action = results[key];
-    Object.keys(action).forEach((key) => {
-      size = action[key].length;
-      const sum = action[key].reduce((sum, val) => sum + val, 0);
-      const avg = (sum / action[key].length).toFixed(2);
-      action[key].push(avg);
-    });
+    size = Math.max(action["scripting"].length, size);
   });
 
-  generateMarkdown(results, size);
+  return size;
 };
 
-const generateMarkdown = (results, size = 5) => {
+const generateMarkdown = (results) => {
+  const size = getMaxSize(results);
   let markdown = `<details><summary>Click to view performance test results</summary>\n\n| `;
   for (let i = 0; i < size; i++) {
-    markdown = markdown + `| Run #${i + 1} `;
+    markdown = markdown + `| Run ${i + 1} `;
   }
-  markdown = markdown + `| Avg `;
+  markdown = markdown + `| Mean | SD.Sample | SD.Population`;
 
   markdown += "|\n";
 
-  for (let i = 0; i <= size + 1; i++) {
+  for (let i = 0; i <= size + 3; i++) {
     markdown = markdown + `| ------------- `;
   }
   markdown += "|\n";
 
   Object.keys(results).forEach((key) => {
     const action = results[key];
-    markdown = markdown + key;
+    markdown += `**${key}**`;
     for (let i = 0; i <= size; i++) {
-      markdown = markdown + `| `;
+      markdown += `| `;
     }
     markdown += "|\n";
+
     Object.keys(action).forEach((key) => {
+      const length = action[key].length;
       markdown += `| ${key} | `;
       markdown += action[key].reduce((sum, val) => `${sum} | ${val} `);
+      if (length < size) {
+        for (let i = 0; i < size - action[key].length; i++) {
+          markdown += " | ";
+        }
+      }
+      // Add average
+      const avg = parseFloat(
+        (action[key].reduce((sum, val) => sum + val, 0) / length).toFixed(2),
+      );
+      markdown += `| ${avg} | ${((sd.sample(action[key]) / avg) * 100).toFixed(
+        2,
+      )} | ${((sd.population(action[key]) / avg) * 100).toFixed(2)}`;
+
       markdown += "| \n";
     });
   });

app/client/perf/src/utils/utils.js

@@ -40,7 +40,7 @@ exports.startReactProfile = async (reactProfiler) => {
     "#container > div > div > div > div > div.Toolbar___30kHu > button.Button___1-PiG.InactiveRecordToggle___2CUtF";
   await reactProfiler.waitForSelector(recordButton);
   const container = await reactProfiler.$(recordButton);
-  console.log("Satring recording");
+  console.log("Starting recording");
   await reactProfiler.evaluate((el) => el.click(), container);
   console.log("Recording started");
 };
@@ -61,7 +61,7 @@ exports.downloadReactProfile = async (reactProfiler) => {
   await reactProfiler.waitForSelector(saveProfileButton);
   const container = await reactProfiler.$(saveProfileButton);
   await reactProfiler.evaluate((el) => el.click(), container);
-  console.log("Downlaoded the profile");
+  console.log("Downloaded the profile");
 };
 
 exports.saveProfile = async (reactProfiler, name) => {
@@ -92,7 +92,7 @@ exports.login = async (page) => {
   const url = "https://dev.appsmith.com/user/login";
 
   await page.goto(url);
-  await page.setViewport({ width: 1440, height: 714 });
+  await page.setViewport({ width: 1920, height: 1080 });
 
   await delay(1000, "before login");
 
@@ -100,23 +100,12 @@ exports.login = async (page) => {
   const passwordSelector = "input[name='password']";
   const buttonSelector = "button[type='submit']";
 
-  try {
-    await page.waitForSelector(emailSelector);
-    await page.waitForSelector(passwordSelector);
-    await page.waitForSelector(buttonSelector);
-  } catch (e) {
-    console.error(e);
-    console.log(
-      "Screenshot:",
-      `${APP_ROOT}/traces/reports/login-selector-error.png`,
-    );
-    await page.screenshot({
-      path: `${APP_ROOT}/traces/reports/login-selector-error.png`,
-    });
-  }
+  await page.waitForSelector(emailSelector);
+  await page.waitForSelector(passwordSelector);
+  await page.waitForSelector(buttonSelector);
 
-  await page.type(emailSelector, process.env.CYPRESS_TESTUSERNAME1);
-  await page.type(passwordSelector, process.env.CYPRESS_TESTPASSWORD1);
+  await page.type(emailSelector, "hello@myemail.com");
+  await page.type(passwordSelector, "qwerty1234");
   delay(1000, "before clicking login button");
   await page.click(buttonSelector);
 };

app/client/perf/start-test.sh

@@ -1 +1 @@
-node index.js
+node ./src/index.js

app/client/perf/tests/import-application.perf.js

@@ -0,0 +1,30 @@
+const path = require("path");
+const Perf = require("../src/perf.js");
+var fs = require("fs");
+const { delay } = require("../src/utils/utils");
+
+process.env["NODE_TLS_REJECT_UNAUTHORIZED"] = 0;
+
+async function importApplication() {
+  const perf = new Perf();
+
+  await perf.launch();
+  const page = perf.getPage();
+  await perf.importApplication(`${APP_ROOT}/tests/dsl/ImportTest.json`);
+  await page.waitForSelector("#tablezjf167vmt5 div.tr:nth-child(4)");
+  await perf.startTrace("Click on table row");
+  await page.click("#tablezjf167vmt5 div.tr:nth-child(4)");
+  await delay(3000);
+  await perf.stopTrace();
+  await perf.generateReport();
+
+  perf.close();
+}
+async function runTests() {
+  await importApplication();
+  await importApplication();
+  await importApplication();
+  await importApplication();
+  await importApplication();
+}
+runTests();

app/client/perf/tests/initial-setup.js

@@ -0,0 +1,43 @@
+const puppeteer = require("puppeteer");
+
+process.env["NODE_TLS_REJECT_UNAUTHORIZED"] = 0;
+
+(async () => {
+  const browser = await puppeteer.launch({
+    args: ["--window-size=1920,1080"],
+    ignoreHTTPSErrors: true,
+  });
+  let page = await browser.newPage();
+  await page.goto("https://dev.appsmith.com/setup/welcome");
+  // await page.goto("http://localhost/setup/welcome");
+  // Since we are not testing the initial setup, just send the post request directly.
+  // Could be moved to bash script as well.
+  await page.evaluate(async () => {
+    const url = "https://dev.appsmith.com/api/v1/users/super";
+    // const url = "http://localhost/api/v1/users/super";
+    await fetch(url, {
+      headers: {
+        accept:
+          "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
+        "accept-language": "en-US,en;q=0.9,fr-CA;q=0.8,fr;q=0.7",
+        "cache-control": "no-cache",
+        "content-type": "application/x-www-form-urlencoded",
+      },
+
+      referrerPolicy: "strict-origin-when-cross-origin",
+      body:
+        "name=Im+Puppeteer&email=hello%40myemail.com&password=qwerty1234&allowCollectingAnonymousData=true&signupForNewsletter=true&role=engineer&useCase=just+exploring",
+      method: "POST",
+      mode: "cors",
+      credentials: "include",
+    })
+      .then((res) =>
+        console.log("Save page with new DSL response:", res.json()),
+      )
+      .catch((err) => {
+        console.log("Save page with new DSL error:", err);
+      });
+  });
+  console.log("Initial setup is successful");
+  await browser.close();
+})();

app/client/perf/tests/typing.perf.js

@@ -1,24 +1,11 @@
 const path = require("path");
-const Perf = require("./perf.js");
+const Perf = require("../src/perf.js");
 const dsl = require("./dsl/simple-typing").dsl;
-var fs = require("fs");
-const { summaries } = require("./summary");
-
-// Set the perf directory as APP_ROOT on the global level
-global.APP_ROOT = path.resolve(__dirname);
-
-// Create the directory
-const dir = `${APP_ROOT}/traces/reports`;
-if (!fs.existsSync(dir)) {
-  fs.mkdirSync(dir, { recursive: true });
-}
 
 process.env["NODE_TLS_REJECT_UNAUTHORIZED"] = 0;
 
 async function testTyping() {
-  const perf = new Perf({
-    ignoreHTTPSErrors: true, // @todo Remove it after initial testing
-  });
+  const perf = new Perf();
   await perf.launch();
   const page = perf.getPage();
   await perf.loadDSL(dsl);
@@ -50,6 +37,5 @@ async function runTests() {
   await testTyping();
   await testTyping();
   await testTyping();
-  summaries(`${APP_ROOT}/traces/reports`);
 }
 runTests();

app/client/perf/yarn.lock

@@ -185,6 +185,11 @@ locate-path@^5.0.0:
   dependencies:
     p-locate "^4.1.0"
 
+lodash@^4.17.2:
+  version "4.17.21"
+  resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c"
+  integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==
+
 minimatch@^3.0.4:
   version "3.0.4"
   resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083"
@@ -209,6 +214,13 @@ node-fetch@2.6.5:
   dependencies:
     whatwg-url "^5.0.0"
 
+node-stdev@^1.0.1:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/node-stdev/-/node-stdev-1.0.1.tgz#7a4ba4ae44123683b9f4f06a25e0ec88b1ff1c54"
+  integrity sha1-ekukrkQSNoO59PBqJeDsiLH/HFQ=
+  dependencies:
+    lodash "^4.17.2"
+
 once@^1.3.0, once@^1.3.1, once@^1.4.0:
   version "1.4.0"
   resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1"
@@ -314,6 +326,13 @@ safe-buffer@~5.2.0:
   resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6"
   integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==
 
+sanitize-filename@^1.6.3:
+  version "1.6.3"
+  resolved "https://registry.yarnpkg.com/sanitize-filename/-/sanitize-filename-1.6.3.tgz#755ebd752045931977e30b2025d340d7c9090378"
+  integrity sha512-y/52Mcy7aw3gRm7IrcGDFx/bCk4AhRh2eI9luHOQM86nZsqwiRkkq2GekHXBBD+SmPidc8i2PqtYZl+pWJ8Oeg==
+  dependencies:
+    truncate-utf8-bytes "^1.0.0"
+
 string_decoder@^1.1.1:
   version "1.3.0"
   resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.3.0.tgz#42f114594a46cf1a8e30b0a84f56c78c3edac21e"
@@ -357,6 +376,13 @@ tracelib@^1.0.1:
   resolved "https://registry.yarnpkg.com/tracelib/-/tracelib-1.0.1.tgz#bb44ea96c19b8d7a6c85a6ee1cac9945c5b75c64"
   integrity sha512-T2Vkpa/7Vdm3sV8nXRn8vZ0tnq6wlnO4Zx7Pux+JA1W6DMlg5EtbNcPZu/L7XRTPc9S0eAKhEFR4p/u0GcsDpQ==
 
+truncate-utf8-bytes@^1.0.0:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/truncate-utf8-bytes/-/truncate-utf8-bytes-1.0.2.tgz#405923909592d56f78a5818434b0b78489ca5f2b"
+  integrity sha1-QFkjkJWS1W94pYGENLC3hInKXys=
+  dependencies:
+    utf8-byte-length "^1.0.1"
+
 unbzip2-stream@1.4.3:
   version "1.4.3"
   resolved "https://registry.yarnpkg.com/unbzip2-stream/-/unbzip2-stream-1.4.3.tgz#b0da04c4371311df771cdc215e87f2130991ace7"
@@ -365,6 +391,11 @@ unbzip2-stream@1.4.3:
     buffer "^5.2.1"
     through "^2.3.8"
 
+utf8-byte-length@^1.0.1:
+  version "1.0.4"
+  resolved "https://registry.yarnpkg.com/utf8-byte-length/-/utf8-byte-length-1.0.4.tgz#f45f150c4c66eee968186505ab93fcbb8ad6bf61"
+  integrity sha1-9F8VDExm7uloGGUFq5P8u4rWv2E=
+
 util-deprecate@^1.0.1:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf"