feat: Syntax parsing through AST (#9115)

* ast changes

* ast fix for cyclic dependency

* jest test updates for ast

* feat: Extract references in code with AST (#8617)

Co-authored-by: Nidhi <nidhi@appsmith.com>

* undo debugger changes

* code clean up and comments

* update type checks for literal nodes

* include tests for IIFE and direct object access

* fix - dependency map not updated on IIFE/direct object access

* update tslib

* unescape on AST parsing

Co-authored-by: Hetu Nandu <hetunandu@gmail.com>
Co-authored-by: Nidhi <nidhi@appsmith.com>

app/client/package.json

@@ -34,6 +34,7 @@
     "@uppy/url": "^1.5.16",
     "@uppy/webcam": "^1.8.4",
     "@welldone-software/why-did-you-render": "^4.2.5",
+    "acorn-walk": "^8.2.0",
     "algoliasearch": "^4.2.0",
     "astring": "^1.7.5",
     "axios": "^0.21.1",
@@ -198,25 +199,8 @@
     "@testing-library/react": "^11.2.6",
     "@testing-library/react-hooks": "^7.0.2",
     "@testing-library/user-event": "^13.1.1",
-    "@types/codemirror": "^0.0.96",
     "@types/chance": "^1.0.7",
-    "@types/lodash": "^4.14.120",
+    "@types/codemirror": "^0.0.96",
-    "@types/moment-timezone": "^0.5.10",
-    "@types/nanoid": "^2.0.0",
-    "@types/node": "^10.12.18",
-    "@types/prismjs": "^1.16.1",
-    "@types/react": "^16.8.2",
-    "@types/react-custom-scrollbars": "^4.0.7",
-    "@types/react-dom": "^16.8.0",
-    "@types/react-helmet": "^5.0.14",
-    "@types/react-instantsearch-dom": "^6.3.0",
-    "@types/react-redux": "^7.0.1",
-    "@types/react-router-dom": "^5.1.2",
-    "@types/react-syntax-highlighter": "^13.5.2",
-    "@types/react-table": "^7.0.13",
-    "@types/styled-components": "^5.1.3",
-    "@types/tinycolor2": "^1.4.2",
-    "@types/zipcelx": "^1.5.0",
     "@types/deep-diff": "^1.0.0",
     "@types/downloadjs": "^1.4.2",
     "@types/draft-js": "^0.11.1",
@@ -224,20 +208,37 @@
     "@types/jest": "^24.0.22",
     "@types/js-beautify": "^1.13.2",
     "@types/jshint": "^2.12.0",
+    "@types/lodash": "^4.14.120",
     "@types/marked": "^1.2.2",
+    "@types/moment-timezone": "^0.5.10",
+    "@types/nanoid": "^2.0.0",
+    "@types/node": "^10.12.18",
     "@types/node-forge": "^0.10.0",
+    "@types/prismjs": "^1.16.1",
+    "@types/react": "^16.8.2",
     "@types/react-beautiful-dnd": "^11.0.4",
+    "@types/react-custom-scrollbars": "^4.0.7",
+    "@types/react-dom": "^16.8.0",
     "@types/react-google-recaptcha": "^2.1.1",
+    "@types/react-helmet": "^5.0.14",
+    "@types/react-instantsearch-dom": "^6.3.0",
+    "@types/react-redux": "^7.0.1",
+    "@types/react-router-dom": "^5.1.2",
     "@types/react-select": "^3.0.5",
+    "@types/react-syntax-highlighter": "^13.5.2",
+    "@types/react-table": "^7.0.13",
     "@types/react-tabs": "^2.3.1",
     "@types/react-test-renderer": "^17.0.1",
     "@types/react-window": "^1.8.2",
     "@types/redux-form": "^8.1.9",
     "@types/redux-mock-store": "^1.0.2",
     "@types/resize-observer-browser": "^0.1.5",
+    "@types/styled-components": "^5.1.3",
     "@types/styled-system": "^5.1.9",
     "@types/tern": "0.22.0",
+    "@types/tinycolor2": "^1.4.2",
     "@types/toposort": "^2.0.3",
+    "@types/zipcelx": "^1.5.0",
     "@typescript-eslint/eslint-plugin": "^4.15.0",
     "@typescript-eslint/parser": "^4.15.0",
     "autoprefixer": "^9",

app/client/src/sagas/PostEvaluationSagas.ts

@@ -288,12 +288,12 @@ export function* evalErrorHandler(
         });
         break;
       }
-      // case EvalErrorTypes.EXTRACT_DEPENDENCY_ERROR: {
+      case EvalErrorTypes.EXTRACT_DEPENDENCY_ERROR: {
-      //   Sentry.captureException(new Error(error.message), {
+        Sentry.captureException(new Error(error.message), {
-      //     extra: error.context,
+          extra: error.context,
-      //   });
+        });
-      //   break;
+        break;
-      // }
+      }
       default: {
         Sentry.captureException(error);
         log.debug(error);

app/client/src/utils/DynamicBindingUtils.ts

@@ -118,6 +118,7 @@ export enum EvalErrorTypes {
   EVAL_TRIGGER_ERROR = "EVAL_TRIGGER_ERROR",
   PARSE_JS_ERROR = "PARSE_JS_ERROR",
   CLONE_ERROR = "CLONE_ERROR",
+  EXTRACT_DEPENDENCY_ERROR = "EXTRACT_DEPENDENCY_ERROR",
 }
 
 export type EvalError = {

app/client/src/widgets/InputWidget/widget/index.tsx

@@ -508,7 +508,7 @@ class InputWidget extends BaseWidget<InputWidgetProps, WidgetState> {
   static getDerivedPropertiesMap(): DerivedPropertiesMap {
     return {
       isValid: `{{
-        function(){
+        (function(){
           if (!this.isRequired && !this.text) {
             return true
           }
@@ -577,7 +577,7 @@ class InputWidget extends BaseWidget<InputWidgetProps, WidgetState> {
           } else {
             return true;
           }
-        }()
+        })()
       }}`,
       value: `{{this.text}}`,
     };

app/client/src/workers/DataTreeEvaluator.ts

@@ -67,6 +67,7 @@ import { Severity } from "entities/AppsmithConsole";
 import { getLintingErrors } from "workers/lint";
 import { JSUpdate } from "utils/JSPaneUtils";
 import { error as logError } from "loglevel";
+import { extractIdentifiersFromCode } from "workers/ast";
 
 export default class DataTreeEvaluator {
   dependencyMap: DependencyMap = {};
@@ -448,9 +449,20 @@ export default class DataTreeEvaluator {
     });
     Object.keys(dependencyMap).forEach((key) => {
       dependencyMap[key] = _.flatten(
-        dependencyMap[key].map((path) =>
+        dependencyMap[key].map((path) => {
-          extractReferencesFromBinding(path, this.allKeys),
+          try {
-        ),
+            return extractReferencesFromBinding(path, this.allKeys);
+          } catch (e) {
+            this.errors.push({
+              type: EvalErrorTypes.EXTRACT_DEPENDENCY_ERROR,
+              message: e.message,
+              context: {
+                script: path,
+              },
+            });
+            return [];
+          }
+        }),
       );
     });
     dependencyMap = makeParentsDependOnChildren(dependencyMap);
@@ -1321,9 +1333,20 @@ export default class DataTreeEvaluator {
       Object.keys(this.dependencyMap).forEach((key) => {
         this.dependencyMap[key] = _.uniq(
           _.flatten(
-            this.dependencyMap[key].map((path) =>
+            this.dependencyMap[key].map((path) => {
-              extractReferencesFromBinding(path, this.allKeys),
+              try {
-            ),
+                return extractReferencesFromBinding(path, this.allKeys);
+              } catch (e) {
+                this.errors.push({
+                  type: EvalErrorTypes.EXTRACT_DEPENDENCY_ERROR,
+                  message: e.message,
+                  context: {
+                    script: path,
+                  },
+                });
+                return [];
+              }
+            }),
           ),
         );
       });
@@ -1462,9 +1485,22 @@ export default class DataTreeEvaluator {
         Object.keys(entityPropertyBindings).forEach((path) => {
           const propertyBindings = entityPropertyBindings[path];
           const references = _.flatten(
-            propertyBindings.map((binding) =>
+            propertyBindings.map((binding) => {
-              extractReferencesFromBinding(binding, this.allKeys),
+              {
-            ),
+                try {
+                  return extractReferencesFromBinding(binding, this.allKeys);
+                } catch (e) {
+                  this.errors.push({
+                    type: EvalErrorTypes.EXTRACT_DEPENDENCY_ERROR,
+                    message: e.message,
+                    context: {
+                      script: binding,
+                    },
+                  });
+                  return [];
+                }
+              }
+            }),
           );
           references.forEach((value) => {
             if (isChildPropertyPath(propertyPath, value)) {
@@ -1545,18 +1581,17 @@ export default class DataTreeEvaluator {
   }
 }
 
-const extractReferencesFromBinding = (
+export const extractReferencesFromBinding = (
-  dependentPath: string,
+  script: string,
-  all: Record<string, true>,
+  allPaths: Record<string, true>,
-): Array<string> => {
+): string[] => {
-  const subDeps: Array<string> = [];
+  const references: Set<string> = new Set<string>();
-  const identifiers = dependentPath.match(/[a-zA-Z_$][a-zA-Z_$0-9.\[\]]*/g) || [
+  const identifiers = extractIdentifiersFromCode(script);
-    dependentPath,
+
-  ];
   identifiers.forEach((identifier: string) => {
     // If the identifier exists directly, add it and return
-    if (all.hasOwnProperty(identifier)) {
+    if (allPaths.hasOwnProperty(identifier)) {
-      subDeps.push(identifier);
+      references.add(identifier);
       return;
     }
     const subpaths = _.toPath(identifier);
@@ -1568,14 +1603,14 @@ const extractReferencesFromBinding = (
     while (subpaths.length > 1) {
       current = convertPathToString(subpaths);
       // We've found the dep, add it and return
-      if (all.hasOwnProperty(current)) {
+      if (allPaths.hasOwnProperty(current)) {
-        subDeps.push(current);
+        references.add(current);
         return;
       }
       subpaths.pop();
     }
   });
-  return _.uniq(subDeps);
+  return Array.from(references);
 };
 
 // TODO cryptic comment below. Dont know if we still need this. Duplicate function

app/client/src/workers/ast.test.ts

@@ -0,0 +1,230 @@
+import { extractIdentifiersFromCode } from "workers/ast";
+
+describe("getAllIdentifiers", () => {
+  it("works properly", () => {
+    const cases: { script: string; expectedResults: string[] }[] = [
+      {
+        // Entity reference
+        script: "DirectTableReference",
+        expectedResults: ["DirectTableReference"],
+      },
+      {
+        // One level nesting
+        script: "TableDataReference.data",
+        expectedResults: ["TableDataReference.data"],
+      },
+      {
+        // Deep nesting
+        script: "TableDataDetailsReference.data.details",
+        expectedResults: ["TableDataDetailsReference.data.details"],
+      },
+      {
+        // Deep nesting
+        script: "TableDataDetailsMoreReference.data.details.more",
+        expectedResults: ["TableDataDetailsMoreReference.data.details.more"],
+      },
+      {
+        // Deep optional chaining
+        script: "TableDataOptionalReference.data?.details.more",
+        expectedResults: ["TableDataOptionalReference.data"],
+      },
+      {
+        // Deep optional chaining with logical operator
+        script:
+          "TableDataOptionalWithLogical.data?.details.more || FallbackTableData.data",
+        expectedResults: [
+          "TableDataOptionalWithLogical.data",
+          "FallbackTableData.data",
+        ],
+      },
+      {
+        // null coalescing
+        script: "TableDataOptionalWithLogical.data ?? FallbackTableData.data",
+        expectedResults: [
+          "TableDataOptionalWithLogical.data",
+          "FallbackTableData.data",
+        ],
+      },
+      {
+        // Basic map function
+        script: "Table5.data.map(c => ({ name: c.name }))",
+        expectedResults: ["Table5.data.map", "c.name"],
+      },
+      {
+        // Literal property search
+        script: "Table6['data']",
+        expectedResults: ["Table6"],
+      },
+      {
+        // Deep literal property search
+        script: "TableDataOptionalReference['data'].details",
+        expectedResults: ["TableDataOptionalReference"],
+      },
+      {
+        // Array index search
+        script: "array[8]",
+        expectedResults: ["array[8]"],
+      },
+      {
+        // Deep array index search
+        script: "Table7.data[4]",
+        expectedResults: ["Table7.data[4]"],
+      },
+      {
+        // Deep array index search
+        script: "Table7.data[4].value",
+        expectedResults: ["Table7.data[4].value"],
+      },
+      {
+        // string literal and array index search
+        script: "Table['data'][9]",
+        expectedResults: ["Table"],
+      },
+      {
+        // array index and string literal search
+        script: "Array[9]['data']",
+        expectedResults: ["Array[9]"],
+      },
+      {
+        // Index identifier search
+        script: "Table8.data[row][name]",
+        expectedResults: ["Table8.data", "row", "name"],
+      },
+      {
+        // Index identifier search with global
+        script: "Table9.data[appsmith.store.row]",
+        expectedResults: ["Table9.data", "appsmith.store.row"],
+      },
+      {
+        // Index literal with further nested lookups
+        script: "Table10.data[row].name",
+        expectedResults: ["Table10.data", "row"],
+      },
+      {
+        // IIFE and if conditions
+        script:
+          "(function(){ if(Table11.isVisible) { return Api1.data } else { return Api2.data } })()",
+        expectedResults: ["Table11.isVisible", "Api1.data", "Api2.data"],
+      },
+      {
+        // Functions and arguments
+        script: "JSObject1.run(Api1.data, Api2.data)",
+        expectedResults: ["JSObject1.run", "Api1.data", "Api2.data"],
+      },
+      {
+        // IIFE - without braces
+        script: `function() {
+          const index = Input1.text   
+         
+          const obj = {
+              "a": 123
+          }
+      
+          return obj[index]
+      
+      }()`,
+        expectedResults: ["Input1.text"],
+      },
+      {
+        // IIFE
+        script: `(function() {
+          const index = Input2.text   
+         
+          const obj = {
+              "a": 123
+          }
+      
+          return obj[index]
+      
+      })()`,
+        expectedResults: ["Input2.text"],
+      },
+      {
+        // arrow IIFE - without braces - will fail
+        script: `() => {
+          const index = Input3.text   
+         
+          const obj = {
+              "a": 123
+          }
+      
+          return obj[index]
+      
+      }()`,
+        expectedResults: [],
+      },
+      {
+        // arrow IIFE
+        script: `(() => {
+          const index = Input4.text   
+         
+          const obj = {
+              "a": 123
+          }
+      
+          return obj[index]
+      
+      })()`,
+        expectedResults: ["Input4.text"],
+      },
+      {
+        // Direct object access
+        script: `{ "a": 123 }[Input5.text]`,
+        expectedResults: ["Input5.text"],
+      },
+      {
+        // Function declaration and default arguments
+        script: `function run(apiData = Api1.data) {
+          return apiData;
+        }`,
+        expectedResults: ["Api1.data"],
+      },
+      {
+        // Function declaration with arguments
+        script: `function run(data) {
+          return data;
+        }`,
+        expectedResults: [],
+      },
+      {
+        // anonymous function with variables
+        script: `() => {
+          let row = 0;
+          const data = {};
+          while(row < 10) {
+            data["test__" + row] = Table12.data[row];
+            row = row += 1;
+          }
+        }`,
+        expectedResults: ["Table12.data"],
+      },
+      {
+        // function with variables
+        script: `function myFunction() {
+          let row = 0;
+          const data = {};
+          while(row < 10) {
+            data["test__" + row] = Table13.data[row];
+            row = row += 1;
+          }
+        }`,
+        expectedResults: ["Table13.data"],
+      },
+      {
+        // expression with arithmetic operations
+        script: `Table14.data + 15`,
+        expectedResults: ["Table14.data"],
+      },
+      {
+        // expression with logical operations
+        script: `Table15.data || [{}]`,
+        expectedResults: ["Table15.data"],
+      },
+    ];
+
+    cases.forEach((perCase) => {
+      const references = extractIdentifiersFromCode(perCase.script);
+      expect(references).toStrictEqual(perCase.expectedResults);
+    });
+  });
+});

app/client/src/workers/ast.ts

@@ -1,11 +1,301 @@
-import { generate } from "astring";
+import { parse, Node } from "acorn";
-import { parse } from "acorn";
+import { ancestor } from "acorn-walk";
+import _ from "lodash";
+import { ECMA_VERSION } from "workers/constants";
+import { unEscapeScript } from "./evaluate";
 
-export const getAST = (code: string) => parse(code, { ecmaVersion: 2020 });
+/*
-export const getFnContents = (code: string) => {
+ * Valuable links:
-  const ast = getAST(code);
+ *
-  // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+ * * ESTree spec: Javascript AST is called ESTree.
-  // @ts-ignore: ast has body, please believe me!
+ * Each es version has its md file in the repo to find features
-  const fnBlock = generate(ast.body[0].body);
+ * implemented and their node type
-  return fnBlock.substring(1, fnBlock.length - 1);
+ * https://github.com/estree/estree
+ *
+ * * Acorn: The parser we use to get the AST
+ * https://github.com/acornjs/acorn
+ *
+ * * Acorn walk: The walker we use to traverse the AST
+ * https://github.com/acornjs/acorn/tree/master/acorn-walk
+ *
+ * * AST Explorer: Helpful web tool to see ASTs and its parts
+ * https://astexplorer.net/
+ *
+ */
+
+// Each node has an attached type property which further defines
+// what all properties can the node have.
+// We will just define the ones we are working with
+enum NodeTypes {
+  MemberExpression = "MemberExpression",
+  Identifier = "Identifier",
+  VariableDeclarator = "VariableDeclarator",
+  FunctionDeclaration = "FunctionDeclaration",
+  FunctionExpression = "FunctionExpression",
+  AssignmentPattern = "AssignmentPattern",
+  Literal = "Literal",
+}
+
+type Pattern = IdentifierNode | AssignmentPatternNode;
+
+// doc: https://github.com/estree/estree/blob/master/es5.md#memberexpression
+interface MemberExpressionNode extends Node {
+  type: NodeTypes.MemberExpression;
+  object: MemberExpressionNode | IdentifierNode;
+  property: IdentifierNode | LiteralNode;
+  computed: boolean;
+  // doc: https://github.com/estree/estree/blob/master/es2020.md#chainexpression
+  optional?: boolean;
+}
+
+// doc: https://github.com/estree/estree/blob/master/es5.md#identifier
+interface IdentifierNode extends Node {
+  type: NodeTypes.Identifier;
+  name: string;
+}
+
+// doc: https://github.com/estree/estree/blob/master/es5.md#variabledeclarator
+interface VariableDeclaratorNode extends Node {
+  type: NodeTypes.VariableDeclarator;
+  id: IdentifierNode;
+}
+
+// doc: https://github.com/estree/estree/blob/master/es5.md#functions
+interface Function extends Node {
+  id: IdentifierNode | null;
+  params: Pattern[];
+}
+
+// doc: https://github.com/estree/estree/blob/master/es5.md#functiondeclaration
+interface FunctionDeclarationNode extends Node, Function {
+  type: NodeTypes.FunctionDeclaration;
+}
+
+// doc: https://github.com/estree/estree/blob/master/es5.md#functionexpression
+interface FunctionExpressionNode extends Node, Function {
+  type: NodeTypes.FunctionExpression;
+}
+
+// doc: https://github.com/estree/estree/blob/master/es2015.md#assignmentpattern
+interface AssignmentPatternNode extends Node {
+  type: NodeTypes.AssignmentPattern;
+  left: Pattern;
+}
+
+// doc: https://github.com/estree/estree/blob/master/es5.md#literal
+interface LiteralNode extends Node {
+  type: NodeTypes.Literal;
+  value: string | boolean | null | number | RegExp;
+}
+
+/* We need these functions to typescript casts the nodes with the correct types */
+const isIdentifierNode = (node: Node): node is IdentifierNode => {
+  return node.type === NodeTypes.Identifier;
+};
+
+const isMemberExpressionNode = (node: Node): node is MemberExpressionNode => {
+  return node.type === NodeTypes.MemberExpression;
+};
+
+const isVariableDeclarator = (node: Node): node is VariableDeclaratorNode => {
+  return node.type === NodeTypes.VariableDeclarator;
+};
+
+const isFunctionDeclaration = (node: Node): node is FunctionDeclarationNode => {
+  return node.type === NodeTypes.FunctionDeclaration;
+};
+
+const isFunctionExpression = (node: Node): node is FunctionExpressionNode => {
+  return node.type === NodeTypes.FunctionExpression;
+};
+
+const isAssignmentPatternNode = (node: Node): node is AssignmentPatternNode => {
+  return node.type === NodeTypes.AssignmentPattern;
+};
+
+const isLiteralNode = (node: Node): node is LiteralNode => {
+  return node.type === NodeTypes.Literal;
+};
+
+const isArrayAccessorNode = (node: Node): node is MemberExpressionNode => {
+  return (
+    isMemberExpressionNode(node) &&
+    node.computed &&
+    isLiteralNode(node.property) &&
+    _.isFinite(node.property.value)
+  );
+};
+
+const wrapCode = (code: string) => {
+  return `
+    (function() {
+      return ${code}
+    })
+  `;
+};
+
+export const getAST = (code: string) =>
+  parse(code, { ecmaVersion: ECMA_VERSION });
+
+/**
+ * An AST based extractor that fetches all possible identifiers in a given
+ * piece of code. We use this to get any references to the global entities in Appsmith
+ * and create dependencies on them. If the reference was updated, the given piece of code
+ * should run again.
+ * @param code: The piece of script where identifiers need to be extracted from
+ */
+export const extractIdentifiersFromCode = (code: string): string[] => {
+  // List of all identifiers found
+  const identifiers = new Set<string>();
+  // List of variables declared within the script. This will be removed from identifier list
+  const variableDeclarations = new Set<string>();
+  // List of functionalParams found. This will be removed from the identifier list
+  let functionalParams = new Set<string>();
+  let ast: Node = { end: 0, start: 0, type: "" };
+  try {
+    const unEscapedCode = unEscapeScript(code);
+    /* wrapCode - Wrapping code in a function, since all code/script get wrapped with a function during evaluation.
+       Some syntaxes won't be valid unless they're at the RHS of a statement.
+       Since we're assigning all code/script to RHS during evaluation, we do the same here.
+       So that during ast parse, those errors are neglected.
+    */
+    /* e.g. IIFE without braces
+      function() { return 123; }() -> is invalid
+      let result = function() { return 123; }() -> is valid 
+    */
+    const wrappedCode = wrapCode(unEscapedCode);
+    ast = getAST(wrappedCode);
+  } catch (e) {
+    if (e instanceof SyntaxError) {
+      // Syntax error. Ignore and return 0 identifiers
+      return [];
+    }
+    throw e;
+  }
+
+  /*
+   * We do an ancestor walk on the AST to get all identifiers. Since we need to know
+   * what surrounds the identifier, ancestor walk will give that information in the callback
+   * doc: https://github.com/acornjs/acorn/tree/master/acorn-walk
+   */
+  ancestor(ast, {
+    Identifier(node: Node, ancestors: Node[]) {
+      /*
+       * We are interested in identifiers. Due to the nature of AST, Identifier nodes can
+       * also be nested inside MemberExpressions. For deeply nested object references, there
+       * could be nesting of many MemberExpressions. To find the final reference, we will
+       * try to find the top level MemberExpression that does not have a MemberExpression parent.
+       * */
+      let candidateTopLevelNode:
+        | IdentifierNode
+        | MemberExpressionNode = node as IdentifierNode;
+      let depth = ancestors.length - 2; // start "depth" with first parent
+      while (depth > 0) {
+        const parent = ancestors[depth];
+        if (
+          isMemberExpressionNode(parent) &&
+          /* Member expressions that are "computed" (with [ ] search)
+             and the ones that have optional chaining ( a.b?.c )
+             will be considered top level node. 
+             We will stop looking for further parents */
+          /* "computed" exception - isArrayAccessorNode
+             Member expressions that are array accessors with static index - [9]
+             will not be considered top level.
+             We will continue looking further. */
+          (!parent.computed || isArrayAccessorNode(parent)) &&
+          !parent.optional
+        ) {
+          candidateTopLevelNode = parent;
+          depth = depth - 1;
+        } else {
+          // Top level found
+          break;
+        }
+      }
+      if (isIdentifierNode(candidateTopLevelNode)) {
+        // If the node is an Identifier, just save that
+        identifiers.add(candidateTopLevelNode.name);
+      } else {
+        // For MemberExpression Nodes, we will construct a final reference string and then add
+        // it to the identifier list
+        const memberExpIdentifier = constructFinalMemberExpIdentifier(
+          candidateTopLevelNode,
+        );
+        identifiers.add(memberExpIdentifier);
+      }
+    },
+    VariableDeclarator(node: Node) {
+      // keep a track of declared variables so they can be
+      // subtracted from the final list of identifiers
+      if (isVariableDeclarator(node)) {
+        variableDeclarations.add(node.id.name);
+      }
+    },
+    FunctionDeclaration(node: Node) {
+      // params in function declarations are also counted as identifiers so we keep
+      // track of them and remove them from the final list of identifiers
+      if (!isFunctionDeclaration(node)) return;
+      functionalParams = new Set([
+        ...functionalParams,
+        ...getFunctionalParamsFromNode(node),
+      ]);
+    },
+    FunctionExpression(node: Node) {
+      // params in function experssions are also counted as identifiers so we keep
+      // track of them and remove them from the final list of identifiers
+      if (!isFunctionExpression(node)) return;
+      functionalParams = new Set([
+        ...functionalParams,
+        ...getFunctionalParamsFromNode(node),
+      ]);
+    },
+  });
+
+  // Remove declared variables and function params
+  variableDeclarations.forEach((variable) => identifiers.delete(variable));
+  functionalParams.forEach((param) => identifiers.delete(param));
+
+  return Array.from(identifiers);
+};
+
+const getFunctionalParamsFromNode = (
+  node: FunctionDeclarationNode | FunctionExpressionNode,
+): Set<string> => {
+  const functionalParams = new Set<string>();
+  node.params.forEach((paramNode) => {
+    if (isIdentifierNode(paramNode)) {
+      functionalParams.add(paramNode.name);
+    } else if (isAssignmentPatternNode(paramNode)) {
+      if (isIdentifierNode(paramNode.left)) {
+        functionalParams.add(paramNode.left.name);
+      }
+    }
+  });
+  return functionalParams;
+};
+
+const constructFinalMemberExpIdentifier = (
+  node: MemberExpressionNode,
+  child = "",
+): string => {
+  const propertyAccessor = getPropertyAccessor(node.property);
+  if (isIdentifierNode(node.object)) {
+    return `${node.object.name}${propertyAccessor}${child}`;
+  } else {
+    const propertyAccessor = getPropertyAccessor(node.property);
+    const nestedChild = `${propertyAccessor}${child}`;
+    return constructFinalMemberExpIdentifier(node.object, nestedChild);
+  }
+};
+
+const getPropertyAccessor = (propertyNode: IdentifierNode | LiteralNode) => {
+  if (isIdentifierNode(propertyNode)) {
+    return `.${propertyNode.name}`;
+  } else if (isLiteralNode(propertyNode) && _.isString(propertyNode.value)) {
+    // is string literal search a['b']
+    return `.${propertyNode.value}`;
+  } else if (isLiteralNode(propertyNode) && _.isFinite(propertyNode.value)) {
+    // is array index search - a[9]
+    return `[${propertyNode.value}]`;
+  }
 };

app/client/src/workers/constants.ts

@@ -0,0 +1 @@
+export const ECMA_VERSION = 11;

app/client/src/workers/evaluate.ts

@@ -128,6 +128,16 @@ export const createGlobalData = (
   return GLOBAL_DATA;
 };
 
+export function unEscapeScript(js: string) {
+  // We remove any line breaks from the beginning of the script because that
+  // makes the final function invalid. We also unescape any escaped characters
+  // so that eval can happen
+  const trimmedJS = js.replace(beginsWithLineBreakRegex, "");
+  const unescapedJS =
+    self.evaluationVersion > 1 ? trimmedJS : unescapeJS(trimmedJS);
+  return unescapedJS;
+}
+
 export default function evaluate(
   js: string,
   data: DataTree,
@@ -135,12 +145,7 @@ export default function evaluate(
   evalArguments?: Array<any>,
   isTriggerBased = false,
 ): EvalResult {
-  // We remove any line breaks from the beginning of the script because that
+  const unescapedJS = unEscapeScript(js);
-  // makes the final function invalid. We also unescape any escaped characters
-  // so that eval can happen
-  const trimmedJS = js.replace(beginsWithLineBreakRegex, "");
-  const unescapedJS =
-    self.evaluationVersion > 1 ? trimmedJS : unescapeJS(trimmedJS);
   const scriptType = getScriptType(evalArguments, isTriggerBased);
   const script = getScriptToEval(unescapedJS, scriptType);
   // We are linting original js binding,

app/client/src/workers/evaluation.test.ts

@@ -228,6 +228,7 @@ const BASE_WIDGET: DataTreeWidget = {
 };
 
 const BASE_ACTION: DataTreeAction = {
+  clear: {},
   logBlackList: {},
   actionId: "randomId",
   name: "randomActionName",

app/client/src/workers/lint.ts

@@ -12,6 +12,7 @@ import {
   EvaluationScriptType,
   ScriptTemplate,
 } from "workers/evaluate";
+import { ECMA_VERSION } from "workers/constants";
 
 export const getPositionInEvaluationScript = (
   type: EvaluationScriptType,
@@ -61,7 +62,7 @@ export const getLintingErrors = (
 
   const options = {
     indent: 2,
-    esversion: 11, // For optional chaining and null coalescing support
+    esversion: ECMA_VERSION,
     eqeqeq: false, // Not necessary to use ===
     curly: false, // Blocks can be added without {}, eg if (x) return true
     freeze: true, // Overriding inbuilt classes like Array is not allowed

app/client/yarn.lock

@@ -3885,6 +3885,11 @@ acorn-walk@^7.0.0, acorn-walk@^7.1.1:
   version "7.2.0"
   resolved "https://registry.npmjs.org/acorn-walk/-/acorn-walk-7.2.0.tgz"
 
+acorn-walk@^8.2.0:
+  version "8.2.0"
+  resolved "https://registry.yarnpkg.com/acorn-walk/-/acorn-walk-8.2.0.tgz#741210f2e2426454508853a2f44d0ab83b7f69c1"
+  integrity sha512-k+iyHEuPgSw6SbuDpGQM+06HQUa04DZ3o+F6CSzXMvvI5KMvnaEqXe+YVe555R9nn6GPt404fos4wcgpw12SDA==
+
 acorn@^4.0.9:
   version "4.0.13"
   resolved "https://registry.npmjs.org/acorn/-/acorn-4.0.13.tgz"
@@ -16193,8 +16198,9 @@ tslib@^1.10.0, tslib@^1.8.1, tslib@^1.9.0, tslib@^1.9.3:
   resolved "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz"
 
 tslib@^2.1.0:
-  version "2.1.0"
+  version "2.3.1"
-  resolved "https://registry.npmjs.org/tslib/-/tslib-2.1.0.tgz"
+  resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.3.1.tgz#e8a335add5ceae51aa261d32a490158ef042ef01"
+  integrity sha512-77EbyPPpMz+FRFRuAFlWMtmgUWGe9UOG2Z25NqCwiIjRhOf5iKGuzSe5P2w1laq+FkRy4p+PCuVkJSGkzTEKVw==
 
 tslib@~1.13.0:
   version "1.13.0"