ci: Fix golden app tests and more (#15172)

* - Save the DOM on test failures.
- Handle promise rejection in tests.
- Save console logs to a file instead of stdout

* - Fix golden app tests
- Upgrade widgets in golden app to latest version

* - Removing logging to file temporarily
- Add test index to the file names

* Some final cleanup

Co-authored-by: Satish Gandham <satish@appsmith.com>

app/client/perf/readme.md

@@ -0,0 +1,33 @@
+### Adding credentials to app export
+- In the exported app add this property under `datasourceList` in the item corresponding to the plugin you are adding credentials for.
+
+    ```
+    "datasourceConfiguration": {
+                "connection": {
+                    "mode": "READ_WRITE",
+                    "ssl": {
+                        "authType": "DEFAULT"
+                    }
+                },
+                "endpoints": [{
+                    "host": "localhost",
+                    "port": 5432
+                }],
+                "sshProxyEnabled": false
+            },
+    ```
+- Add this key at the top level
+ 	```
+    "decryptedFields": {
+		"PostgresGolden": {
+			"password": "********",
+			"authType": "com.appsmith.external.models.DBAuth",
+			"dbAuth": {
+				"authenticationType": "dbAuth",
+				"authenticationType": "dbAuth",
+				"username": "********",
+				"databaseName": "db_name"
+			}
+		}
+	},
+    ```

app/client/perf/src/ci/supabase.js

@@ -18,7 +18,7 @@ const metricsToLog = [
   "LongTask",
 ];
 
-const supabaseKey = process.env.APPSMITH_PERF_SUPABASE_SECRET;
+const supabaseKey = process.env.APPSMITH_PERF_SUPABASE_SECRET || "empty";
 const supabase = createClient(supabaseUrl, supabaseKey);
 
 const actionRows = Object.keys(actions).map((action) => ({

app/client/perf/src/index.js

@@ -15,6 +15,7 @@ if (!fs.existsSync(dir)) {
 glob("./tests/*.perf.js", {}, async function(er, files) {
   // Initial setup
   await cp.execSync(`node ./tests/initial-setup.js`, { stdio: "inherit" });
+
   files.forEach(async (file) => {
     await cp.execSync(`node ${file}`, { stdio: "inherit" }); // Logging to terminal, log it to a file in future?
   });

app/client/perf/src/perf.js

@@ -26,6 +26,7 @@ const selectors = {
 
 module.exports = class Perf {
   constructor(launchOptions = {}) {
+    this.iteration = launchOptions.iteration || 0; // Current iteration number
     this.launchOptions = {
       defaultViewport: null,
       args: ["--window-size=1920,1080"],
@@ -50,26 +51,48 @@ module.exports = class Perf {
       .pop()
       .replace(".perf.js", "");
     global.APP_ROOT = path.join(__dirname, ".."); //Going back one level from src folder to /perf
-    process.on("unhandledRejection", async (reason, p) => {
+
-      console.error("Unhandled Rejection at: Promise", p, "reason:", reason);
+    process.on("unhandledRejection", this.handleRejections);
-      const fileName = sanitize(
-        `${this.currentTestFile}__${this.currentTrace}`,
-      );
-      const screenshotPath = `${APP_ROOT}/traces/reports/${fileName}-${getFormattedTime()}.png`;
-      await this.page.screenshot({
-        path: screenshotPath,
-      });
-      if (this.currentTrace) {
-        await this.stopTrace();
-      }
-      this.browser.close();
-    });
   }
+
+  handleRejections = async (reason = "", p = "") => {
+    console.error("Unhandled Rejection at: Promise", p, "reason:", reason);
+    const fileName = sanitize(`${this.currentTestFile}__${this.currentTrace}`);
+
+    if (!this.page) {
+      console.warn("No page instance was found", this.currentTestFile);
+      return;
+    }
+    const screenshotPath = `${APP_ROOT}/traces/reports/${fileName}-${getFormattedTime()}.png`;
+    await this.page.screenshot({
+      path: screenshotPath,
+    });
+
+    const pageContent = await this.page.evaluate(() => {
+      return document.querySelector("body").innerHTML;
+    });
+
+    fs.writeFile(
+      `${APP_ROOT}/traces/reports/${fileName}-${getFormattedTime()}.html`,
+      pageContent,
+      (err) => {
+        if (err) {
+          console.error(err);
+        }
+      },
+    );
+
+    if (this.currentTrace) {
+      await this.stopTrace();
+    }
+    this.browser.close();
+  };
   /**
    * Launches the browser and, gives you the page
    */
   launch = async () => {
     await cleanTheHost();
+    await delay(3000);
     this.browser = await puppeteer.launch(this.launchOptions);
     const pages_ = await this.browser.pages();
     this.page = pages_[0];
@@ -97,7 +120,9 @@ module.exports = class Perf {
     await this.page._client.send("HeapProfiler.collectGarbage");
     await delay(1000, `After clearing memory`);
 
-    const path = `${APP_ROOT}/traces/${action}-${getFormattedTime()}-chrome-profile.json`;
+    const path = `${APP_ROOT}/traces/${action}-${
+      this.iteration
+    }-${getFormattedTime()}-chrome-profile.json`;
 
     await this.page.tracing.start({
       path: path,

app/client/perf/tests/golden-app.perf.js

@@ -6,19 +6,19 @@ const { delay, makeid } = require("../src/utils/utils");
 process.env["NODE_TLS_REJECT_UNAUTHORIZED"] = 0;
 
 const SEL = {
-  category: "div[label=Uncategorized]",
+  category: "div.rc-select-item[title=Uncategorized]",
   multiSelect: ".rc-select",
   table: "#tablejabdu9f16g",
   tableData: ".t--property-control-tabledata textarea",
   tableRow:
     "#tablejabdu9f16g > div.tableWrap > div > div:nth-child(1) > div > div.tbody.no-scroll > div:nth-child(6) > div:nth-child(2)",
-  titleInput: ".appsmith_widget_in8e51pg3y  input",
+  titleInput: ".appsmith_widget_armli8hauj  input",
   updateButton:
     "#comment-overlay-wrapper-4gnygu5jew > div > div > div > div > button",
   tableRowCell:
     "#tablejabdu9f16g > div.tableWrap > div > div:nth-child(1) > div > div.tbody.no-scroll > div:nth-child(6) > div:nth-child(2) > div > span > span > span",
   deletePostButton:
-    "#tablejabdu9f16g > div.tableWrap > div > div:nth-child(1) > div > div.tbody.no-scroll > div:nth-child(1) > div:nth-child(27) > div > div > button",
+    "#tablejabdu9f16g > div.tableWrap > div > div:nth-child(1) > div > div.tbody.no-scroll > div:nth-child(1) > div:last-child > div > div > button",
   modalTitle: "#reyoxo4oec",
   closeModal:
     "#comment-overlay-wrapper-lryg8kw537 > div > div > div > div > button",
@@ -26,94 +26,93 @@ const SEL = {
   commentsTableTitle: "#urzv99hdc8",
 };
 
-async function testTyping() {
+async function testGoldenApp(iteration) {
-  const perf = new Perf();
+  const perf = new Perf({ iteration });
-  await perf.launch();
+  try {
-  const page = perf.getPage();
+    await perf.launch();
+    const page = perf.getPage();
 
-  await perf.importApplication(
+    await perf.importApplication(
-    `${APP_ROOT}/tests/dsl/blog-admin-app-postgres.json`,
+      `${APP_ROOT}/tests/dsl/blog-admin-app-postgres.json`,
-  );
+    );
 
-  await delay(5000, "for newly created page to settle down");
+    await delay(5000, "for newly created page to settle down");
-  // Make the elements of the dropdown render
+    // Make the elements of the dropdown render
-  await page.waitForSelector(SEL.multiSelect);
+    await page.waitForSelector(SEL.multiSelect);
-  await page.click(SEL.multiSelect);
+    await page.click(SEL.multiSelect);
 
-  await perf.startTrace(actions.SELECT_CATEGORY);
+    await perf.startTrace(actions.SELECT_CATEGORY);
-  await page.waitForSelector(SEL.category);
+    await page.waitForSelector(SEL.category);
-  await page.click(SEL.category);
+    await page.click(SEL.category);
 
-  await perf.stopTrace();
+    await perf.stopTrace();
 
-  // Focus on the table widget
+    // Focus on the table widget
-  await page.waitForSelector(SEL.table);
+    await page.waitForSelector(SEL.table);
-  await page.click(SEL.table);
 
-  // Profile table Data binding
+    // Not sure why it needs two clicks to focus
-  await perf.startTrace(actions.BIND_TABLE_DATA);
+    await page.click(SEL.table);
-  await page.waitForSelector(SEL.tableData);
+    await page.click(SEL.table);
-  await page.type(SEL.tableData, "{{SelectQuery.data}}");
-  await page.waitForSelector(SEL.tableRow);
-  await perf.stopTrace();
 
-  // Click on table row
+    // Profile table Data binding
-  await perf.startTrace(actions.CLICK_ON_TABLE_ROW);
+    await perf.startTrace(actions.BIND_TABLE_DATA);
-  await page.click(SEL.tableRow);
+    await page.waitForSelector(SEL.tableData);
-  await page.waitForFunction(
+    await page.type(SEL.tableData, "{{SelectQuery.data}}");
-    `document.querySelector("${SEL.titleInput}").value.includes("Template: Comments")`,
+    await page.waitForSelector(SEL.tableRow);
-  );
+    await perf.stopTrace();
 
-  await perf.stopTrace();
+    // Click on table row
+    await perf.startTrace(actions.CLICK_ON_TABLE_ROW);
+    await page.click(SEL.tableRow);
+    await page.waitForFunction(
+      `document.querySelector("${SEL.titleInput}").value.includes("Template: Comments")`,
+    );
 
-  // Edit title
+    await perf.stopTrace();
-  await page.waitForSelector(SEL.titleInput);
-  await perf.startTrace(actions.UPDATE_POST_TITLE);
 
-  const randomString = makeid();
+    // Edit title
-  await page.type(SEL.titleInput, randomString);
+    await page.waitForSelector(SEL.titleInput);
-  await delay(5000, "For the evaluations to comeback?");
+    await perf.startTrace(actions.UPDATE_POST_TITLE);
 
-  await page.waitForSelector(SEL.updateButton);
+    const randomString = makeid();
-  await page.click(SEL.updateButton);
+    await page.type(SEL.titleInput, randomString);
-  // When the row is updated, selected row changes.
+    await delay(5000, "For the evaluations to comeback?");
-  // await page.waitForSelector(SEL.tableRowCell);
-  await page.waitForFunction(
-    `document.querySelector("${SEL.table}").textContent.includes("${randomString}")`,
-  );
-  await perf.stopTrace();
 
-  // Open modal
+    await page.waitForSelector(SEL.updateButton);
-  await page.waitForSelector(SEL.deletePostButton);
+    await page.click(SEL.updateButton);
-  await perf.startTrace(actions.OPEN_MODAL);
+    // When the row is updated, selected row changes.
-  await page.click(SEL.deletePostButton);
+    // await page.waitForSelector(SEL.tableRowCell);
-  await page.waitForSelector(SEL.modalTitle);
+    await page.waitForFunction(
-  await perf.stopTrace();
+      `document.querySelector("${SEL.table}").textContent.includes("${randomString}")`,
+    );
+    await perf.stopTrace();
 
-  // Close modal
+    // Open modal
-  await page.waitForSelector(SEL.closeModal);
+    await page.waitForSelector(SEL.deletePostButton);
-  await perf.startTrace(actions.CLOSE_MODAL);
+    await perf.startTrace(actions.OPEN_MODAL);
-  await page.click(SEL.closeModal);
+    await page.click(SEL.deletePostButton);
-  await delay(3000, "wait after closing modal");
+    await page.waitForSelector(SEL.modalTitle);
-  await perf.stopTrace();
+    await perf.stopTrace();
 
-  /* Enable this after the new entity explorer
+    // Close modal
-  // Navigate to a page
+    await page.waitForSelector(SEL.closeModal);
-  await page.waitForSelector(SEL.commentsPageLink);
+    await perf.startTrace(actions.CLOSE_MODAL);
-  await perf.startTrace("Switch page");
+    await page.click(SEL.closeModal);
-  await page.click(SEL.commentsPageLink);
+    await delay(3000, "wait after closing modal");
-  await page.waitForSelector(SEL.commentsTableTitle);
+    await perf.stopTrace();
-  await perf.stopTrace();
+
-*/
+    await perf.generateReport();
-  await perf.generateReport();
+    await perf.close();
-  await perf.close();
+  } catch (e) {
+    await perf.handleRejections(e);
+    await perf.close();
+  }
 }
 
 async function runTests() {
-  await testTyping();
+  for (let i = 0; i < 5; i++) {
-  await testTyping();
+    await testGoldenApp(i + 1);
-  await testTyping();
+  }
-  await testTyping();
-  await testTyping();
 }
+
 runTests();

app/client/perf/tests/sample-test.js

@@ -4,38 +4,43 @@ const dsl = require("./dsl/simple-typing").dsl;
 
 process.env["NODE_TLS_REJECT_UNAUTHORIZED"] = 0;
 
-async function testTyping() {
+async function sampleTest(iteration) {
-  const perf = new Perf();
+  const perf = new Perf({ iteration });
-  await perf.launch();
+  try {
-  const page = perf.getPage();
+    await perf.launch();
-  await perf.loadDSL(dsl);
 
-  const selector = "input.bp3-input"; // Input selector
+    const page = perf.getPage();
-  await page.waitForSelector(selector);
+    await perf.loadDSL(dsl);
-  const input = await page.$(selector);
 
-  await perf.startTrace("Edit input");
+    const selector = "input.bp3-input"; // Input selector
-  await page.type(selector, "Hello Appsmith");
+    await page.waitForSelector(selector);
-  await perf.stopTrace();
+    const input = await page.$(selector);
 
-  await perf.startTrace("Clear input");
+    await perf.startTrace("Edit input");
-  await input.click({ clickCount: 3 });
+    await page.type(selector, "Hello Appsmith");
-  await input.press("Backspace");
+    await perf.stopTrace();
-  await perf.stopTrace();
 
-  await perf.startTrace("Edit input again");
+    await perf.startTrace("Clear input");
-  await page.type(selector, "Howdy satish");
+    await input.click({ clickCount: 3 });
-  await perf.stopTrace();
+    await input.press("Backspace");
+    await perf.stopTrace();
 
-  await perf.generateReport();
+    await perf.startTrace("Edit input again");
-  await perf.close();
+    await page.type(selector, "Howdy satish");
+    await perf.stopTrace();
+
+    await perf.generateReport();
+    await perf.close();
+  } catch (e) {
+    await perf.handleRejections(e);
+    await perf.close();
+  }
 }
 
 async function runTests() {
-  await testTyping();
+  for (let i = 0; i < 5; i++) {
-  await testTyping();
+    await sampleTest(i + 1);
-  await testTyping();
+  }
-  await testTyping();
-  await testTyping();
 }
+
 runTests();

app/client/perf/tests/select-widget.perf.js

@@ -10,9 +10,9 @@ const SEL = {
   first_option_item: ".menu-item-text:nth-child(1)",
 };
 
-async function testSelectOptionsRender() {
+async function testSelectOptionsRender(iteration) {
+  const perf = new Perf({ iteration });
   try {
-    const perf = new Perf();
     await perf.launch();
     const page = perf.getPage();
 
@@ -34,15 +34,15 @@ async function testSelectOptionsRender() {
     await perf.generateReport();
     await perf.close();
   } catch (e) {
-    console.log(e);
+    await perf.handleRejections(e);
+    await perf.close();
   }
 }
 
 async function runTests() {
-  await testSelectOptionsRender();
+  for (let i = 0; i < 5; i++) {
-  await testSelectOptionsRender();
+    await testSelectOptionsRender(i + 1);
-  await testSelectOptionsRender();
+  }
-  await testSelectOptionsRender();
-  await testSelectOptionsRender();
 }
+
 runTests();