Protei/PromucFlow_constructor
2
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

feat: Linting in entity properties and methods (#16171)

Browse Source 
* Initial commit

* Remove arrow function params from identifiers

* Remove invalid identifiers from extracted identifiers

* Remove invalid identifiers which are derived from function params and variable declarations

* Fix typo error

* Correctly remove invalid identifiers

* Remove invalid names from identifier list

* fix build failure

* Add Promise to list of unacceptable entity name

* Keep track of unreferenced identifiers in bindings

* Add Global scope object names as unusable entity names

* Keep track of unreferenced identifiers

* Prevent traversal of data tree for addition of new paths and entities

* Sync linting in trigger fields

* Support linting of invalid properties

* Fix linting reactivity bug in trigger field

* Remove unused objects

* Fix conflict in merging

* Lint jsobject body for function change

* Remove unused map from tests

* Code cleanup

* Modify jest tests

* Update jest tests

* Fix cypress tests

* Code cleanup

* Support  linting of multiple bindings

* Set squiggle line as long as invalid property length

* Add jest tests

* Minor code refactor

* Move ast to shared repo

* Rename confusing identifiers

* Improve naming of functions and their return values

* move shared widget validation utils and constants to shared folder

* Add jest test for invalid entity names

* Add cypress tests

* Modify test comment

* Extend list of dedicated worker scope identifiers

* Resolve code review comments

* Resolve review comments

* Annonate code where necessary

* Code refactor

* Improve worker global scope object

* Code refactor

* Fix merge conflict

* Code refactor

* Minor bug fix

* Redundant commit to retrigger vercel build

* Add null checks to dependecy chain
This commit is contained in:
Favour Ohanekwu 2022-09-17 10:40:28 -07:00 committed by GitHub
parent 5e9cb1e447
commit d6fbdb15b9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
32 changed files with 1979 additions and 1905 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
app/client/cypress/integration/Smoke_TestSuite/ClientSideTests/Linting/BasicLint_spec.ts
View File

@ -75,9 +75,7 @@ describe("Linting", () => {
clickButtonAndAssertLintError(true);
// create Api1
apiPage.CreateAndFillApi(
"https://jsonplaceholder.typicode.com/"
);
apiPage.CreateAndFillApi("https://jsonplaceholder.typicode.com/");
clickButtonAndAssertLintError(false);
@ -88,9 +86,7 @@ describe("Linting", () => {
clickButtonAndAssertLintError(true);
// Re-create Api1
apiPage.CreateAndFillApi(
"https://jsonplaceholder.typicode.com/"
);
apiPage.CreateAndFillApi("https://jsonplaceholder.typicode.com/");
clickButtonAndAssertLintError(false);
});
@ -272,9 +268,7 @@ describe("Linting", () => {
shouldCreateNewJSObj: true,
},
);
apiPage.CreateAndFillApi(
"https://jsonplaceholder.typicode.com/"
);
apiPage.CreateAndFillApi("https://jsonplaceholder.typicode.com/");
createMySQLDatasourceQuery();

111
app/client/cypress/integration/Smoke_TestSuite/ClientSideTests/Linting/EntityPropertiesLint_spec.ts Normal file
View File

@ -0,0 +1,111 @@
import { ObjectsRegistry } from "../../../../support/Objects/Registry";
const jsEditor = ObjectsRegistry.JSEditor,
locator = ObjectsRegistry.CommonLocators,
ee = ObjectsRegistry.EntityExplorer,
apiPage = ObjectsRegistry.ApiPage,
agHelper = ObjectsRegistry.AggregateHelper,
propPane = ObjectsRegistry.PropertyPane;
describe("Linting of entity properties", () => {
before(() => {
ee.DragDropWidgetNVerify("buttonwidget", 300, 300);
ee.NavigateToSwitcher("explorer");
});
it("1. Shows correct lint error when wrong Api property is binded", () => {
const invalidProperty = "unknownProperty";
// create Api1
apiPage.CreateAndFillApi("https://jsonplaceholder.typicode.com/");
// Edit Button onclick property
ee.SelectEntityByName("Button1", "Widgets");
propPane.EnterJSContext(
"onClick",
`{{function(){
console.log(Api1.${invalidProperty})
}()}}`,
);
propPane.UpdatePropertyFieldValue("Label", `{{Api1.${invalidProperty}}}`);
cy.get(locator._lintErrorElement)
.should("have.length", 2)
.first()
.trigger("mouseover");
agHelper
.AssertContains(`"${invalidProperty}" doesn't exist in Api1`)
.should("exist");
});
it("2. Shows correct lint error when wrong JSObject property is binded", () => {
// create JSObject
jsEditor.CreateJSObject(
`export default {
myFun1: () => {
console.log("JSOBJECT 1")
}
}`,
{
paste: true,
completeReplace: true,
toRun: false,
shouldCreateNewJSObj: true,
},
);
const invalidProperty = "unknownFunction";
// Edit Button onclick and text property
ee.SelectEntityByName("Button1", "Widgets");
propPane.EnterJSContext(
"onClick",
`{{function(){
console.log(JSObject1.${invalidProperty})
}()}}`,
);
propPane.UpdatePropertyFieldValue(
"Label",
`{{JSObject1.${invalidProperty}}}`,
);
// Assert lint errors
cy.get(locator._lintErrorElement)
.should("have.length", 2)
.first()
.trigger("mouseover");
agHelper.AssertContains(`"${invalidProperty}" doesn't exist in JSObject1`);
// Edit JS Object and add "unknown" function
ee.SelectEntityByName("JSObject1", "Queries/JS");
jsEditor.EditJSObj(`export default {
${invalidProperty}: () => {
console.log("JSOBJECT 1")
}
}`);
// select button, and assert that no lint is present
ee.SelectEntityByName("Button1", "Widgets");
agHelper.AssertElementAbsence(locator._lintErrorElement);
// delete JSObject
ee.ActionContextMenuByEntityName("JSObject1", "Delete", "Are you sure?");
// select button, and assert that lint error is present
ee.SelectEntityByName("Button1", "Widgets");
cy.get(locator._lintErrorElement)
.should("have.length", 2)
.first()
.trigger("mouseover");
agHelper.AssertContains(`'JSObject1' is not defined`);
// create js object
jsEditor.CreateJSObject(
`export default {
${invalidProperty}: () => {
console.log("JSOBJECT 1")
}
}`,
{
paste: true,
completeReplace: true,
toRun: false,
shouldCreateNewJSObj: true,
},
);
// select button, and assert that no lint error is present
ee.SelectEntityByName("Button1", "Widgets");
cy.get(locator._lintErrorElement).should("not.exist");
});
});

7
app/client/cypress/integration/Smoke_TestSuite/ServerSideTests/ApiTests/API_Unique_name_spec.js
View File

@ -15,10 +15,7 @@ describe("Name uniqueness test", function() {
cy.CreationOfUniqueAPIcheck("download");
});
it("Validate window object property apiname check", () => {
cy.CreationOfUniqueAPIcheck("localStorage");
});
it("Validate window object method apiname check", () => {
cy.CreationOfUniqueAPIcheck("resizeTo");
it("Validate dedicated worker scope object property(Blob)apiname check", () => {
cy.CreationOfUniqueAPIcheck("Blob");
});
});

20
app/client/src/components/editorComponents/ActionCreator/Fields.tsx
View File

@ -25,6 +25,10 @@ import { NavigationTargetType } from "sagas/ActionExecution/NavigateActionSaga";
import DividerComponent from "widgets/DividerWidget/component";
import store from "store";
import { getPageList } from "selectors/entitiesSelector";
import {
APPSMITH_GLOBAL_FUNCTIONS,
APPSMITH_NAMESPACED_FUNCTIONS,
} from "./constants";
/* eslint-disable @typescript-eslint/ban-types */
/* TODO: Function and object types need to be updated to enable the lint rule */
@ -236,21 +240,11 @@ const enumTypeGetter = (
export const ActionType = {
none: "none",
integration: "integration",
showModal: "showModal",
closeModal: "closeModal",
navigateTo: "navigateTo",
showAlert: "showAlert",
storeValue: "storeValue",
download: "download",
copyToClipboard: "copyToClipboard",
resetWidget: "resetWidget",
jsFunction: "jsFunction",
setInterval: "setInterval",
clearInterval: "clearInterval",
getGeolocation: "appsmith.geolocation.getCurrentPosition",
watchGeolocation: "appsmith.geolocation.watchPosition",
stopWatchGeolocation: "appsmith.geolocation.clearWatch",
...APPSMITH_GLOBAL_FUNCTIONS,
...APPSMITH_NAMESPACED_FUNCTIONS,
};
type ActionType = typeof ActionType[keyof typeof ActionType];
const ViewTypes = {

18
app/client/src/components/editorComponents/ActionCreator/constants.ts Normal file
View File

@ -0,0 +1,18 @@
export const APPSMITH_GLOBAL_FUNCTIONS = {
navigateTo: "navigateTo",
showAlert: "showAlert",
showModal: "showModal",
closeModal: "closeModal",
storeValue: "storeValue",
download: "download",
copyToClipboard: "copyToClipboard",
resetWidget: "resetWidget",
setInterval: "setInterval",
clearInterval: "clearInterval",
};
export const APPSMITH_NAMESPACED_FUNCTIONS = {
getGeolocation: "appsmith.geolocation.getCurrentPosition",
watchGeolocation: "appsmith.geolocation.watchPosition",
stopWatchGeolocation: "appsmith.geolocation.clearWatch",
};

12
app/client/src/components/editorComponents/CodeEditor/constants.ts
View File

@ -40,3 +40,15 @@ export const SUPPORTED_WEB_APIS = {
console: true,
crypto: true,
};
export enum CustomLintErrorCode {
INVALID_ENTITY_PROPERTY = "INVALID_ENTITY_PROPERTY",
}
export const CUSTOM_LINT_ERRORS: Record<
CustomLintErrorCode,
(...args: any[]) => string
> = {
[CustomLintErrorCode.INVALID_ENTITY_PROPERTY]: (
entityName: string,
propertyName: string,
) => `"${propertyName}" doesn't exist in ${entityName}`,
};

18
app/client/src/components/editorComponents/CodeEditor/lintHelpers.ts
View File

@ -8,6 +8,7 @@ import {
import { Severity } from "entities/AppsmithConsole";
import {
CODE_EDITOR_START_POSITION,
CUSTOM_LINT_ERRORS,
IDENTIFIER_NOT_DEFINED_LINT_ERROR_CODE,
INVALID_JSOBJECT_START_STATEMENT,
JS_OBJECT_START_STATEMENT,
@ -31,13 +32,20 @@ interface LintAnnotationOptions {
contextData: AdditionalDynamicDataTree;
}
/**
*
* @param error
* @param contextData
* @returns A boolean signifying the presence of an identifier which the linter records as been "not defined"
* but is passed to the editor as additional dynamic data
*/
const hasUndefinedIdentifierInContextData = (
error: EvaluationError,
contextData: LintAnnotationOptions["contextData"],
) => {
/**
* W117: "'{a}' is not defined.",
* error has only one variable "a"
* error has only one variable "a", which is the name of the variable which is not defined.
* */
return (
error.code === IDENTIFIER_NOT_DEFINED_LINT_ERROR_CODE &&
@ -187,8 +195,12 @@ export const getLintAnnotations = (
}
// Jshint counts \t as two characters and codemirror counts it as 1.
// So we need to subtract number of tabs to get accurate position
const tabs = lineContent.slice(0, currentCh).match(/\t/g)?.length || 0;
// So we need to subtract number of tabs to get accurate position.
// This is not needed for custom lint errors, since they are not generated by JSHint
const tabs =
error.code && error.code in CUSTOM_LINT_ERRORS
? 0
: lineContent.slice(0, currentCh).match(/\t/g)?.length || 0;
const from = {
line: currentLine,
ch: currentCh - tabs - 1,

2
app/client/src/components/editorComponents/Debugger/helpers.tsx
View File

@ -127,7 +127,7 @@ export function getDependencyChain(
let currentChain: string[] = [];
const dependents = inverseMap[propertyPath];
if (!dependents.length) return currentChain;
if (!dependents || !dependents.length) return currentChain;
const { entityName } = getEntityNameAndPropertyPath(propertyPath);

473
app/client/src/constants/WidgetValidation.ts
View File

@ -34,76 +34,6 @@ export type Validator = (
export const ISO_DATE_FORMAT = "YYYY-MM-DDTHH:mm:ss.sssZ";
export const JAVASCRIPT_KEYWORDS = {
Array: "Array",
await: "await",
Boolean: "Boolean",
break: "break",
case: "case",
catch: "catch",
class: "class",
const: "const",
continue: "continue",
Date: "Date",
debugger: "debugger",
default: "default",
delete: "delete",
do: "do",
else: "else",
enum: "enum",
eval: "eval",
export: "export",
extends: "extends",
false: "false",
finally: "finally",
for: "for",
function: "function",
Function: "Function",
hasOwnProperty: "hasOwnProperty",
if: "if",
implements: "implements",
import: "import",
in: "in",
Infinity: "Infinity",
instanceof: "instanceof",
interface: "interface",
isFinite: "isFinite",
isNaN: "isNaN",
isPrototypeOf: "isPrototypeOf",
JSON: "JSON",
length: "length",
let: "let",
Math: "Math",
name: "name",
NaN: "NaN",
new: "new",
null: "null",
Number: "Number",
Object: "Object",
package: "package",
private: "private",
protected: "protected",
public: "public",
return: "return",
static: "static",
String: "String",
super: "super",
switch: "switch",
this: "this",
throw: "throw",
toString: "toString",
true: "true",
try: "try",
typeof: "typeof",
undefined: "undefined",
valueOf: "valueOf",
var: "var",
void: "void",
while: "while",
with: "with",
yield: "yield",
};
export const DATA_TREE_KEYWORDS = {
actionPaths: "actionPaths",
appsmith: "appsmith",
@ -111,64 +41,369 @@ export const DATA_TREE_KEYWORDS = {
[EXECUTION_PARAM_KEY]: EXECUTION_PARAM_KEY,
};
export const WINDOW_OBJECT_PROPERTIES = {
closed: "closed",
console: "console",
defaultStatus: "defaultStatus",
document: "document",
frameElement: "frameElement",
frames: "frames",
history: "history",
innerHeight: "innerHeight",
innerWidth: "innerWidth",
length: "length",
localStorage: "localStorage",
location: "location",
name: "name",
navigator: "navigator",
opener: "opener",
outerHeight: "outerHeight",
outerWidth: "outerWidth",
pageXOffset: "pageXOffset",
pageYOffset: "pageYOffset",
parent: "parent",
screen: "screen",
screenLeft: "screenLeft",
screenTop: "screenTop",
screenY: "screenY",
scrollX: "scrollX",
scrollY: "scrollY",
export const JAVASCRIPT_KEYWORDS = {
abstract: "abstract",
arguments: "arguments",
await: "await",
boolean: "boolean",
break: "break",
byte: "byte",
case: "case",
catch: "catch",
char: "char",
class: "class",
const: "const",
continue: "continue",
debugger: "debugger",
default: "default",
delete: "delete",
do: "do",
double: "double",
else: "else",
enum: "enum",
eval: "eval",
export: "export",
extends: "extends",
false: "false",
final: "final",
finally: "finally",
float: "float",
for: "for",
function: "function",
goto: "goto",
if: "if",
implements: "implements",
import: "import",
in: "in",
instanceof: "instanceof",
int: "int",
interface: "interface",
let: "let",
long: "long",
native: "native",
new: "new",
null: "null",
package: "package",
private: "private",
protected: "protected",
public: "public",
return: "return",
self: "self",
status: "status",
top: "top",
evaluationVersion: "evaluationVersion",
short: "short",
static: "static",
super: "super",
switch: "switch",
synchronized: "synchronized",
this: "this",
throw: "throw",
throws: "throws",
transient: "transient",
true: "true",
try: "try",
typeof: "typeof",
var: "var",
void: "void",
volatile: "volatile",
while: "while",
with: "with",
yield: "yield",
};
export const WINDOW_OBJECT_METHODS = {
alert: "alert",
/**
* Global scope Identifiers in the worker context, accessible via the "self" keyword.
* These identifiers are already present in the worker context and shouldn't represent any valid identifier within Appsmith, as no entity should have
* same name as them to prevent unexpected behaviour during evaluation(which happens on the worker thread) in the worker.
* Check if an identifier (or window object/property) is available in the worker context here => https://worker-playground.glitch.me/
*/
export const DEDICATED_WORKER_GLOBAL_SCOPE_IDENTIFIERS = {
AbortController: "AbortController",
AbortSignal: "AbortSignal",
AggregateError: "AggregateError",
Array: "Array",
ArrayBuffer: "ArrayBuffer",
atob: "atob",
blur: "blur",
Atomics: "Atomics",
AudioData: "AudioData",
AudioDecoder: "AudioDecoder",
AudioEncoder: "AudioEncoder",
BackgroundFetchManager: "BackgroundFetchManager",
BackgroundFetchRecord: "BackgroundFetchRecord",
BackgroundFetchRegistration: "BackgroundFetchRegistration",
BarcodeDetector: "BarcodeDetector",
BigInt: "BigInt",
BigInt64Array: "BigInt64Array",
BigUint64Array: "BigUint64Array",
Blob: "Blob",
Boolean: "Boolean",
btoa: "btoa",
BroadcastChannel: "BroadcastChannel",
ByteLengthQueuingStrategy: "ByteLengthQueuingStrategy",
caches: "caches",
CSSSkewX: "CSSSkewX",
CSSSkewY: "CSSSkewY",
Cache: "Cache",
CacheStorage: "CacheStorage",
cancelAnimationFrame: "cancelAnimationFrame",
CanvasFilter: "CanvasFilter",
CanvasGradient: "CanvasGradient",
CanvasPattern: "CanvasPattern",
clearInterval: "clearInterval",
clearTimeout: "clearTimeout",
close: "close",
confirm: "confirm",
focus: "focus",
getComputedStyle: "getComputedStyle",
getSelection: "getSelection",
matchMedia: "matchMedia",
moveBy: "moveBy",
moveTo: "moveTo",
open: "open",
print: "print",
prompt: "prompt",
CloseEvent: "CloseEvent",
CompressionStream: "CompressionStream",
console: "console",
CountQueuingStrategy: "CountQueuingStrategy",
createImageBitmap: "createImageBitmap",
CropTarget: "CropTarget",
crossOriginIsolated: "crossOriginIsolated",
Crypto: "Crypto",
CryptoKey: "CryptoKey",
CustomEvent: "CustomEvent",
decodeURI: "decodeURI",
decodeURIComponent: "decodeURIComponent",
DOMException: "DOMException",
DOMMatrix: "DOMMatrix",
DOMMatrixReadOnly: "DOMMatrixReadOnly",
DOMPoint: "DOMPoint",
DOMPointReadOnly: "DOMPointReadOnly",
DOMQuad: "DOMQuad",
DOMRect: "DOMRect",
DOMRectReadOnly: "DOMRectReadOnly",
DOMStringList: "DOMStringList",
DataView: "DataView",
Date: "Date",
DecompressionStream: "DecompressionStream",
DedicatedWorkerGlobalScope: "DedicatedWorkerGlobalScope",
encodeURI: "encodeURI",
encodeURIComponent: "encodeURIComponent",
EncodedAudioChunk: "EncodedAudioChunk",
EncodedVideoChunk: "EncodedVideoChunk",
Error: "Error",
ErrorEvent: "ErrorEvent",
escape: "escape",
eval: "eval",
EvalError: "EvalError",
Event: "Event",
EventSource: "EventSource",
EventTarget: "EventTarget",
fetch: "fetch",
File: "File",
FileList: "FileList",
FileReader: "FileReader",
FileReaderSync: "FileReaderSync",
FileSystemDirectoryHandle: "FileSystemDirectoryHandle",
FileSystemFileHandle: "FileSystemFileHandle",
FileSystemHandle: "FileSystemHandle",
FileSystemSyncAccessHandle: "FileSystemSyncAccessHandle",
FileSystemWritableFileStream: "FileSystemWritableFileStream",
FinalizationRegistry: "FinalizationRegistry",
Float32Array: "Float32Array",
Float64Array: "Float64Array",
FontFace: "FontFace",
FormData: "FormData",
Function: "Function",
globalThis: "globalThis",
hasOwnProperty: "hasOwnProperty",
Headers: "Headers",
IDBCursor: "IDBCursor",
IDBCursorWithValue: "IDBCursorWithValue",
IDBDatabase: "IDBDatabase",
IDBFactory: "IDBFactory",
IDBIndex: "IDBIndex",
IDBKeyRange: "IDBKeyRange",
IDBObjectStore: "IDBObjectStore",
IDBOpenDBRequest: "IDBOpenDBRequest",
IDBRequest: "IDBRequest",
IDBTransaction: "IDBTransaction",
IDBVersionChangeEvent: "IDBVersionChangeEvent",
IdleDetector: "IdleDetector",
ImageBitmap: "ImageBitmap",
ImageBitmapRenderingContext: "ImageBitmapRenderingContext",
ImageData: "ImageData",
ImageDecoder: "ImageDecoder",
ImageTrack: "ImageTrack",
ImageTrackList: "ImageTrackList",
importScripts: "importScripts",
indexedDB: "indexedDB",
Infinity: "Infinity",
Int8Array: "Int8Array",
Int16Array: "Int16Array",
Int32Array: "Int32Array",
Intl: "Intl",
isFinite: "isFinite",
isNaN: "isNaN",
isPrototypeOf: "isPrototypeOf",
isSecureContext: "isSecureContext",
JSON: "JSON",
Lock: "Lock",
LockManager: "LockManager",
location: "location",
Map: "Map",
Math: "Math",
MediaCapabilities: "MediaCapabilities",
MessageChannel: "MessageChannel",
MessageEvent: "MessageEvent",
MessagePort: "MessagePort",
NaN: "NaN",
name: "name",
navigator: "navigator",
NavigationPreloadManager: "NavigationPreloadManager",
NavigatorUAData: "NavigatorUAData",
NetworkInformation: "NetworkInformation",
Notification: "Notification",
Number: "Number",
onmessage: "onmessage",
onmessageerror: "onmessageerror",
origin: "origin",
Object: "Object",
OffscreenCanvas: "OffscreenCanvas",
OffscreenCanvasRenderingContext2D: "OffscreenCanvasRenderingContext2D",
parseFloat: "parseFloat",
parseInt: "parseInt",
Path2D: "Path2D",
PaymentInstruments: "PaymentInstruments",
Performance: "Performance",
PerformanceEntry: "PerformanceEntry",
PerformanceMark: "PerformanceMark",
PerformanceMeasure: "PerformanceMeasure",
PerformanceObserver: "PerformanceObserver",
PerformanceObserverEntryList: "PerformanceObserverEntryList",
PerformanceResourceTiming: "PerformanceResourceTiming",
PerformanceServerTiming: "PerformanceServerTiming",
PeriodicSyncManager: "PeriodicSyncManager",
PermissionStatus: "PermissionStatus",
Permissions: "Permissions",
postMessage: "postMessage",
ProgressEvent: "ProgressEvent",
Promise: "Promise",
PromiseRejectionEvent: "PromiseRejectionEvent",
Proxy: "Proxy",
PushManager: "PushManager",
PushSubscription: "PushSubscription",
PushSubscriptionOptions: "PushSubscriptionOptions",
queueMicrotask: "queueMicrotask",
RTCEncodedAudioFrame: "RTCEncodedAudioFrame",
RTCEncodedVideoFrame: "RTCEncodedVideoFrame",
RangeError: "RangeError",
ReadableByteStreamController: "ReadableByteStreamController",
ReadableStream: "ReadableStream",
ReadableStreamBYOBReader: "ReadableStreamBYOBReader",
ReadableStreamBYOBRequest: "ReadableStreamBYOBRequest",
ReadableStreamDefaultController: "ReadableStreamDefaultController",
ReadableStreamDefaultReader: "ReadableStreamDefaultReader",
ReferenceError: "ReferenceError",
Reflect: "Reflect",
RegExp: "RegExp",
reportError: "reportError",
ReportingObserver: "ReportingObserver",
Request: "Request",
requestAnimationFrame: "requestAnimationFrame",
resizeBy: "resizeBy",
resizeTo: "resizeTo",
scroll: "scroll",
scrollBy: "scrollBy",
scrollTo: "scrollBy",
Response: "Response",
Scheduler: "Scheduler",
SecurityPolicyViolationEvent: "SecurityPolicyViolationEvent",
Serial: "Serial",
SerialPort: "SerialPort",
ServiceWorkerRegistration: "ServiceWorkerRegistration",
Set: "Set",
setInterval: "setInterval",
setTimeout: "setTimeout",
stop: "stop",
StorageManager: "StorageManager",
String: "String",
structuredClone: "structuredClone",
SubtleCrypto: "SubtleCrypto",
Symbol: "Symbol",
SyncManager: "SyncManager",
SyntaxError: "SyntaxError",
TaskController: "TaskController",
TaskPriorityChangeEvent: "TaskPriorityChangeEvent",
TaskSignal: "TaskSignal",
TextDecoder: "TextDecoder",
TextDecoderStream: "TextDecoderStream",
TextEncoder: "TextEncoder",
TextEncoderStream: "TextEncoderStream",
TextMetrics: "TextMetrics",
toString: "toString",
TransformStream: "TransformStream",
TransformStreamDefaultController: "TransformStreamDefaultController",
TrustedHTML: "TrustedHTML",
TrustedScript: "TrustedScript",
TrustedScriptURL: "TrustedScriptURL",
trustedTypes: "trustedTypes",
TrustedTypePolicy: "TrustedTypePolicy",
TrustedTypePolicyFactory: "TrustedTypePolicyFactory",
TypeError: "TypeError",
undefined: "undefined",
unescape: "unescape",
URIError: "URIError",
URL: "URL",
URLPattern: "URLPattern",
URLSearchParams: "URLSearchParams",
USB: "USB",
USBAlternateInterface: "USBAlternateInterface",
USBConfiguration: "USBConfiguration",
USBConnectionEvent: "USBConnectionEvent",
USBDevice: "USBDevice",
USBEndpoint: "USBEndpoint",
USBInTransferResult: "USBInTransferResult",
USBInterface: "USBInterface",
USBIsochronousInTransferPacket: "USBIsochronousInTransferPacket",
USBIsochronousInTransferResult: "USBIsochronousInTransferResult",
USBIsochronousOutTransferPacket: "USBIsochronousOutTransferPacket",
USBIsochronousOutTransferResult: "USBIsochronousOutTransferResult",
USBOutTransferResult: "USBOutTransferResult",
Uint8Array: "Uint8Array",
Uint8ClampedArray: "Uint8ClampedArray",
Uint16Array: "Uint16Array",
Uint32Array: "Uint32Array",
UserActivation: "UserActivation",
VideoColorSpace: "VideoColorSpace",
VideoDecoder: "VideoDecoder",
VideoEncoder: "VideoEncoder",
VideoFrame: "VideoFrame",
WeakMap: "WeakMap",
WeakRef: "WeakRef",
WeakSet: "WeakSet",
WebAssembly: "WebAssembly",
WebGL2RenderingContext: "WebGL2RenderingContext",
WebGLActiveInfo: "WebGLActiveInfo",
WebGLBuffer: "WebGLBuffer",
WebGLFramebuffer: "WebGLFramebuffer",
WebGLProgram: "WebGLProgram",
WebGLQuery: "WebGLQuery",
WebGLRenderbuffer: "WebGLRenderbuffer",
WebGLRenderingContext: "WebGLRenderingContext",
WebGLSampler: "WebGLSampler",
WebGLShader: "WebGLShader",
WebGLShaderPrecisionFormat: "WebGLShaderPrecisionFormat",
WebGLSync: "WebGLSync",
WebGLTexture: "WebGLTexture",
WebGLTransformFeedback: "WebGLTransformFeedback",
WebGLUniformLocation: "WebGLUniformLocation",
WebGLVertexArrayObject: "WebGLVertexArrayObject",
webkitRequestFileSystem: "webkitRequestFileSystem",
webkitRequestFileSystemSync: "webkitRequestFileSystemSync",
webkitResolveLocalFileSystemSyncURL: "webkitResolveLocalFileSystemSyncURL",
webkitResolveLocalFileSystemURL: "webkitResolveLocalFileSystemURL",
WebSocket: "WebSocket",
WebTransport: "WebTransport",
WebTransportBidirectionalStream: "WebTransportBidirectionalStream",
WebTransportDatagramDuplexStream: "WebTransportDatagramDuplexStream",
WebTransportError: "WebTransportError",
Worker: "Worker",
WorkerGlobalScope: "WorkerGlobalScope",
WorkerLocation: "WorkerLocation",
WorkerNavigator: "WorkerNavigator",
WritableStream: "WritableStream",
WritableStreamDefaultController: "WritableStreamDefaultController",
WritableStreamDefaultWriter: "WritableStreamDefaultWriter",
XMLHttpRequest: "XMLHttpRequest",
XMLHttpRequestEventTarget: "XMLHttpRequestEventTarget",
XMLHttpRequestUpload: "XMLHttpRequestUpload",
// Identifiers added to worker scope by Appsmith
evaluationVersion: "evaluationVersion",
ALLOW_ASYNC: "ALLOW_ASYNC",
IS_ASYNC: "IS_ASYNC",
TRIGGER_COLLECTOR: "TRIGGER_COLLECTOR",
};

30
app/client/src/constants/ast.ts
View File

@ -1,30 +0,0 @@
export const ECMA_VERSION = 11;
/* Indicates the mode the code should be parsed in.
This influences global strict mode and parsing of import and export declarations.
*/
export enum SourceType {
script = "script",
module = "module",
}
// Each node has an attached type property which further defines
// what all properties can the node have.
// We will just define the ones we are working with
export enum NodeTypes {
Identifier = "Identifier",
AssignmentPattern = "AssignmentPattern",
Literal = "Literal",
Property = "Property",
// Declaration - https://github.com/estree/estree/blob/master/es5.md#declarations
FunctionDeclaration = "FunctionDeclaration",
ExportDefaultDeclaration = "ExportDefaultDeclaration",
VariableDeclarator = "VariableDeclarator",
// Expression - https://github.com/estree/estree/blob/master/es5.md#expressions
MemberExpression = "MemberExpression",
FunctionExpression = "FunctionExpression",
ArrowFunctionExpression = "ArrowFunctionExpression",
ObjectExpression = "ObjectExpression",
ArrayExpression = "ArrayExpression",
ThisExpression = "ThisExpression",
}

5
app/client/src/utils/DynamicBindingUtils.ts
View File

@ -4,6 +4,7 @@ import { Action } from "entities/Action";
import moment from "moment-timezone";
import { WidgetProps } from "widgets/BaseWidget";
import parser from "fast-xml-parser";
import { Severity } from "entities/AppsmithConsole";
import {
getEntityNameAndPropertyPath,
@ -186,20 +187,18 @@ export const extraLibraries: ExtraLibrary[] = [
displayName: "forge",
},
];
/**
* creates dynamic list of constants based on
* current list of extra libraries i.e lodash("_"), moment etc
* to be used in widget and entity name validations
*/
export const extraLibrariesNames = extraLibraries.reduce(
(prev: any, curr: any) => {
(prev: Record<string, string>, curr) => {
prev[curr.accessor] = curr.accessor;
return prev;
},
{},
);
export interface DynamicPath {
key: string;
value?: string;

30
app/client/src/utils/helpers.test.ts
View File

@ -9,6 +9,7 @@ import {
captureInvalidDynamicBindingPath,
mergeWidgetConfig,
extractColorsFromString,
isNameValid,
} from "./helpers";
import WidgetFactory from "./WidgetFactory";
import * as Sentry from "@sentry/react";
@ -567,3 +568,32 @@ describe("#extractColorsFromString", () => {
expect(extractColorsFromString(borderWithRgb)[0]).toEqual("rgb(0,0,0)");
});
});
describe("isNameValid()", () => {
it("works properly", () => {
const invalidEntityNames = [
"console",
"moment",
"Promise",
"appsmith",
"Math",
"_",
"forge",
"yield",
"Boolean",
"ReferenceError",
"clearTimeout",
"parseInt",
"eval",
];
// Some window object methods and properties names should be valid entity names since evaluation is done
// in the worker thread, and some of the window methods and properties are not available there.
const validEntityNames = ["history", "parent", "screen"];
for (const invalidName of invalidEntityNames) {
expect(isNameValid(invalidName, {})).toBe(false);
}
for (const validName of validEntityNames) {
expect(isNameValid(validName, {})).toBe(true);
}
});
});

20
app/client/src/utils/helpers.tsx
View File

@ -6,12 +6,10 @@ import welcomeConfetti from "assets/lottie/welcome-confetti.json";
import successAnimation from "assets/lottie/success-animation.json";
import {
DATA_TREE_KEYWORDS,
DEDICATED_WORKER_GLOBAL_SCOPE_IDENTIFIERS,
JAVASCRIPT_KEYWORDS,
WINDOW_OBJECT_METHODS,
WINDOW_OBJECT_PROPERTIES,
} from "constants/WidgetValidation";
import { GLOBAL_FUNCTIONS } from "./autocomplete/EntityDefinitions";
import { get, set, isNil } from "lodash";
import { get, set, isNil, has } from "lodash";
import { Workspace } from "constants/workspaceConstants";
import {
isPermitted,
@ -32,6 +30,7 @@ import {
VIEWER_PATH_DEPRECATED,
} from "constants/routes";
import history from "./history";
import { APPSMITH_GLOBAL_FUNCTIONS } from "components/editorComponents/ActionCreator/constants";
export const snapToGrid = (
columnWidth: number,
@ -377,13 +376,12 @@ export const isNameValid = (
invalidNames: Record<string, any>,
) => {
return !(
name in JAVASCRIPT_KEYWORDS ||
name in DATA_TREE_KEYWORDS ||
name in GLOBAL_FUNCTIONS ||
name in WINDOW_OBJECT_PROPERTIES ||
name in WINDOW_OBJECT_METHODS ||
name in extraLibrariesNames ||
name in invalidNames
has(JAVASCRIPT_KEYWORDS, name) ||
has(DATA_TREE_KEYWORDS, name) ||
has(DEDICATED_WORKER_GLOBAL_SCOPE_IDENTIFIERS, name) ||
has(APPSMITH_GLOBAL_FUNCTIONS, name) ||
has(extraLibrariesNames, name) ||
has(invalidNames, name)
);
};

5
app/client/src/workers/Actions.ts
View File

@ -11,6 +11,11 @@ import { promisifyAction } from "workers/PromisifyAction";
import { klona } from "klona/full";
import uniqueId from "lodash/uniqueId";
declare global {
/** All identifiers added to the worker global scope should also
* be included in the DEDICATED_WORKER_GLOBAL_SCOPE_IDENTIFIERS in
* app/client/src/constants/WidgetValidation.ts
* */
interface Window {
ALLOW_ASYNC?: boolean;
IS_ASYNC?: boolean;

153
app/client/src/workers/DataTreeEvaluator/index.ts
View File

@ -19,7 +19,6 @@ import {
DataTreeEntity,
DataTreeJSAction,
DataTreeWidget,
ENTITY_TYPE,
EvaluationSubstitutionType,
PrivateWidgets,
} from "entities/DataTree/dataTreeFactory";
@ -43,7 +42,6 @@ import {
validateActionProperty,
addWidgetPropertyDependencies,
overrideWidgetProperties,
isValidEntity,
getAllPaths,
} from "workers/evaluationUtils";
import _ from "lodash";
@ -75,10 +73,6 @@ import {
} from "constants/PropertyControlConstants";
import { klona } from "klona/full";
import { EvalMetaUpdates } from "./types";
import {
extractReferencesFromBinding,
getEntityReferencesFromPropertyBindings,
} from "workers/DependencyMap/utils";
import {
updateDependencyMap,
createDependencyMap,
@ -109,7 +103,14 @@ export default class DataTreeEvaluator {
[actionId: string]: ActionValidationConfigMap;
};
triggerFieldDependencyMap: DependencyMap = {};
triggerFieldInverseDependencyMap: DependencyMap = {};
/** Keeps track of all invalid references in bindings throughout the Application
* Eg. For binding {{unknownEntity.name + Api1.name}} in Button1.text, where Api1 is present in dataTree but unknownEntity is not,
* the map has a key-value pair of
* {
* "Button1.text": [unknownEntity.name]
* }
*/
invalidReferencesMap: DependencyMap = {};
public hasCyclicalDependency = false;
constructor(
widgetConfigMap: WidgetTypeConfigMap,
@ -151,12 +152,14 @@ export default class DataTreeEvaluator {
this.allKeys = getAllPaths(localUnEvalTree);
// Create dependency map
const createDependencyStart = performance.now();
const { dependencyMap, triggerFieldDependencyMap } = createDependencyMap(
this,
localUnEvalTree,
);
const {
dependencyMap,
invalidReferencesMap,
triggerFieldDependencyMap,
} = createDependencyMap(this, localUnEvalTree);
this.dependencyMap = dependencyMap;
this.triggerFieldDependencyMap = triggerFieldDependencyMap;
this.invalidReferencesMap = invalidReferencesMap;
const createDependencyEnd = performance.now();
// Sort
const sortDependenciesStart = performance.now();
@ -164,7 +167,7 @@ export default class DataTreeEvaluator {
const sortDependenciesEnd = performance.now();
// Inverse
this.inverseDependencyMap = this.getInverseDependencyTree();
this.triggerFieldInverseDependencyMap = this.getInverseTriggerDependencyMap();
// Evaluate
const evaluateStart = performance.now();
const { evalMetaUpdates, evaluatedTree } = this.evaluateTree(
@ -185,7 +188,7 @@ export default class DataTreeEvaluator {
unEvalTree: localUnEvalTree,
evalTree: this.evalTree,
sortedDependencies: this.sortedDependencies,
triggerPathsToLint: [],
extraPathsToLint: [],
});
const lintStop = performance.now();
const totalEnd = performance.now();
@ -206,9 +209,6 @@ export default class DataTreeEvaluator {
triggerFieldMap: JSON.parse(
JSON.stringify(this.triggerFieldDependencyMap),
),
triggerFieldInverseMap: JSON.parse(
JSON.stringify(this.triggerFieldInverseDependencyMap),
),
},
lint: (lintStop - lintStart).toFixed(2),
};
@ -318,8 +318,8 @@ export default class DataTreeEvaluator {
// global dependency map if an existing dynamic binding has now become legal
const {
dependenciesOfRemovedPaths,
extraPathsToLint,
removedPaths,
triggerPathsToLint,
} = updateDependencyMap({
dataTreeEvalRef: this,
translatedDiffs,
@ -387,7 +387,7 @@ export default class DataTreeEvaluator {
unEvalTree: localUnEvalTree,
evalTree: newEvalTree,
sortedDependencies: evaluationOrder,
triggerPathsToLint,
extraPathsToLint,
});
const lintStop = performance.now();
@ -596,30 +596,6 @@ export default class DataTreeEvaluator {
return dependencies;
}
listTriggerFieldDependencies(
entity: DataTreeWidget,
entityName: string,
): DependencyMap {
const triggerFieldDependency: DependencyMap = {};
if (isWidget(entity)) {
const dynamicTriggerPathlist = entity.dynamicTriggerPathList;
if (dynamicTriggerPathlist && dynamicTriggerPathlist.length) {
dynamicTriggerPathlist.forEach((dynamicPath) => {
const propertyPath = dynamicPath.key;
const unevalPropValue = _.get(entity, propertyPath);
const { jsSnippets } = getDynamicBindings(unevalPropValue);
const existingDeps =
triggerFieldDependency[`${entityName}.${propertyPath}`] || [];
triggerFieldDependency[
`${entityName}.${propertyPath}`
] = existingDeps.concat(
jsSnippets.filter((jsSnippet) => !!jsSnippet),
);
});
}
}
return triggerFieldDependency;
}
evaluateTree(
oldUnevalTree: DataTree,
resolvedFunctions: Record<string, any>,
@ -1045,7 +1021,6 @@ export default class DataTreeEvaluator {
widget,
propertyPath,
);
const evaluatedValue = isValid
? parsed
: _.isUndefined(transformed)
@ -1198,19 +1173,6 @@ export default class DataTreeEvaluator {
// Remove any paths that do not exist in the data tree anymore
return _.difference(completeSortOrder, removedPaths);
}
getInverseTriggerDependencyMap(): DependencyMap {
const inverseTree: DependencyMap = {};
Object.keys(this.triggerFieldDependencyMap).forEach((triggerField) => {
this.triggerFieldDependencyMap[triggerField].forEach((field) => {
if (inverseTree[field]) {
inverseTree[field].push(triggerField);
} else {
inverseTree[field] = [triggerField];
}
});
});
return inverseTree;
}
getInverseDependencyTree(): DependencyMap {
const inverseDag: DependencyMap = {};
@ -1230,85 +1192,6 @@ export default class DataTreeEvaluator {
return inverseDag;
}
// TODO: create the lookup dictionary once
// Response from listEntityDependencies only needs to change if the entity itself changed.
// Check if it is possible to make a flat structure with O(1) or at least O(m) lookup instead of O(n*m)
getPropertyPathReferencesInExistingBindings(
dataTree: DataTree,
propertyPath: string,
) {
const possibleRefs: DependencyMap = {};
Object.keys(dataTree).forEach((entityName) => {
const entity = dataTree[entityName];
if (
isValidEntity(entity) &&
(entity.ENTITY_TYPE === ENTITY_TYPE.ACTION ||
entity.ENTITY_TYPE === ENTITY_TYPE.JSACTION ||
entity.ENTITY_TYPE === ENTITY_TYPE.WIDGET)
) {
const entityPropertyBindings = this.listEntityDependencies(
entity,
entityName,
);
Object.keys(entityPropertyBindings).forEach((path) => {
const propertyBindings = entityPropertyBindings[path];
const references = _.flatten(
propertyBindings.map((binding) => {
{
try {
return extractReferencesFromBinding(binding, this.allKeys);
} catch (error) {
this.errors.push({
type: EvalErrorTypes.EXTRACT_DEPENDENCY_ERROR,
message: (error as Error).message,
context: {
script: binding,
},
});
return [];
}
}
}),
);
references.forEach((value) => {
if (isChildPropertyPath(propertyPath, value)) {
possibleRefs[path] = propertyBindings;
}
});
});
}
});
return possibleRefs;
}
getTriggerFieldReferencesInExistingBindings(
dataTree: DataTree,
entityNamePath: string,
) {
const possibleRefs: DependencyMap = {};
Object.keys(dataTree).forEach((entityName) => {
const entity = dataTree[entityName];
if (isWidget(entity)) {
let entityPropertyBindings: DependencyMap = {};
entityPropertyBindings = {
...entityPropertyBindings,
...this.listTriggerFieldDependencies(entity, entityName),
};
Object.keys(entityPropertyBindings).forEach((path) => {
const propertyBindings = entityPropertyBindings[path];
const references = getEntityReferencesFromPropertyBindings(
propertyBindings,
this,
);
if (references.includes(entityNamePath)) {
possibleRefs[path] = references;
}
});
}
});
return possibleRefs;
}
evaluateActionBindings(
bindings: string[],
executionParams?: Record<string, unknown> | string,

37
app/client/src/workers/DataTreeEvaluator/test.ts
View File

@ -497,44 +497,28 @@ describe("DataTreeEvaluator", () => {
});
it("Creates correct triggerFieldDependencyMap", () => {
expect(dataTreeEvaluator.triggerFieldDependencyMap).toEqual({
"Button3.onClick": ["Api1", "Button2", "Api2"],
"Button2.onClick": ["Api2"],
"Button3.onClick": ["Api1.run", "Button2.text", "Api2.run"],
"Button2.onClick": ["Api2.run"],
});
});
it("Creates correct triggerFieldInverseDependencyMap", () => {
expect(dataTreeEvaluator.triggerFieldInverseDependencyMap).toEqual({
Api1: ["Button3.onClick"],
Api2: ["Button3.onClick", "Button2.onClick"],
Button2: ["Button3.onClick"],
});
});
it("Correctly updates triggerFieldDependencyMap and triggerFieldInverseDependencyMap", () => {
it("Correctly updates triggerFieldDependencyMap", () => {
const newUnEvalTree = ({ ...lintingUnEvalTree } as unknown) as DataTree;
// delete Api2
delete newUnEvalTree["Api2"];
dataTreeEvaluator.updateDataTree(newUnEvalTree);
expect(dataTreeEvaluator.triggerFieldDependencyMap).toEqual({
"Button3.onClick": ["Api1", "Button2"],
"Button3.onClick": ["Api1.run", "Button2.text"],
"Button2.onClick": [],
});
expect(dataTreeEvaluator.triggerFieldInverseDependencyMap).toEqual({
Api1: ["Button3.onClick"],
Button2: ["Button3.onClick"],
});
// Add Api2
// @ts-expect-error: Types are not available
newUnEvalTree["Api2"] = { ...lintingUnEvalTree }["Api2"];
dataTreeEvaluator.updateDataTree(newUnEvalTree);
expect(dataTreeEvaluator.triggerFieldDependencyMap).toEqual({
"Button3.onClick": ["Api1", "Button2", "Api2"],
"Button2.onClick": ["Api2"],
});
expect(dataTreeEvaluator.triggerFieldInverseDependencyMap).toEqual({
Api1: ["Button3.onClick"],
Api2: ["Button3.onClick", "Button2.onClick"],
Button2: ["Button3.onClick"],
"Button3.onClick": ["Api1.run", "Button2.text", "Api2.run"],
"Button2.onClick": ["Api2.run"],
});
// self-reference Button2
@ -549,12 +533,7 @@ describe("DataTreeEvaluator", () => {
dataTreeEvaluator.updateDataTree(newUnEvalTree);
expect(dataTreeEvaluator.triggerFieldDependencyMap).toEqual({
"Button3.onClick": ["Api1", "Api2"],
});
expect(dataTreeEvaluator.triggerFieldInverseDependencyMap).toEqual({
Api1: ["Button3.onClick"],
Api2: ["Button3.onClick"],
"Button3.onClick": ["Api1.run", "Api2.run"],
});
});
});

552
app/client/src/workers/DependencyMap/index.ts
View File

@ -8,6 +8,7 @@ import {
makeParentsDependOnChildren,
isDynamicLeaf,
isValidEntity,
getEntityNameAndPropertyPath,
} from "workers/evaluationUtils";
import {
DataTree,
@ -21,22 +22,35 @@ import {
getPropertyPath,
isPathADynamicBinding,
getDynamicBindings,
EvalErrorTypes,
isPathADynamicTrigger,
} from "utils/DynamicBindingUtils";
import {
extractReferencesFromBinding,
getEntityReferencesFromPropertyBindings,
extractInfoFromBindings,
extractInfoFromReferences,
listTriggerFieldDependencies,
mergeArrays,
} from "./utils";
import DataTreeEvaluator from "workers/DataTreeEvaluator";
import { flatten, difference, uniq } from "lodash";
import { difference } from "lodash";
interface CreateDependencyMap {
dependencyMap: DependencyMap;
triggerFieldDependencyMap: DependencyMap;
/** Keeps track of all invalid references present in bindings throughout the page.
* We keep this list so that we don't have to traverse the entire dataTree when
* a new entity or path is added to the datatree in order to determine if an old invalid reference has become valid
* because an entity or path is newly added.
* */
invalidReferencesMap: DependencyMap;
}
export function createDependencyMap(
dataTreeEvalRef: DataTreeEvaluator,
unEvalTree: DataTree,
): { dependencyMap: DependencyMap; triggerFieldDependencyMap: DependencyMap } {
): CreateDependencyMap {
let dependencyMap: DependencyMap = {};
let triggerFieldDependencyMap: DependencyMap = {};
const invalidReferencesMap: DependencyMap = {};
Object.keys(unEvalTree).forEach((entityName) => {
const entity = unEvalTree[entityName];
if (isAction(entity) || isWidget(entity) || isJSAction(entity)) {
@ -50,43 +64,66 @@ export function createDependencyMap(
// only widgets have trigger paths
triggerFieldDependencyMap = {
...triggerFieldDependencyMap,
...dataTreeEvalRef.listTriggerFieldDependencies(entity, entityName),
...listTriggerFieldDependencies(entity, entityName),
};
}
});
Object.keys(dependencyMap).forEach((key) => {
const newDep = dependencyMap[key].map((path) => {
try {
return extractReferencesFromBinding(path, dataTreeEvalRef.allKeys);
} catch (error) {
dataTreeEvalRef.errors.push({
type: EvalErrorTypes.EXTRACT_DEPENDENCY_ERROR,
message: (error as Error).message,
context: {
script: path,
},
});
return [];
}
});
dependencyMap[key] = flatten(newDep);
Object.keys(dependencyMap).forEach((key) => {
const {
errors,
invalidReferences,
validReferences,
} = extractInfoFromBindings(dependencyMap[key], dataTreeEvalRef.allKeys);
dependencyMap[key] = validReferences;
// To keep invalidReferencesMap as minimal as possible, only paths with invalid references
// are stored.
if (invalidReferences.length) {
invalidReferencesMap[key] = invalidReferences;
}
errors.forEach((error) => {
dataTreeEvalRef.errors.push(error);
});
});
// extract references from bindings
// extract references from bindings in trigger fields
Object.keys(triggerFieldDependencyMap).forEach((key) => {
triggerFieldDependencyMap[key] = getEntityReferencesFromPropertyBindings(
const {
errors,
invalidReferences,
validReferences,
} = extractInfoFromBindings(
triggerFieldDependencyMap[key],
dataTreeEvalRef,
dataTreeEvalRef.allKeys,
);
triggerFieldDependencyMap[key] = validReferences;
// To keep invalidReferencesMap as minimal as possible, only paths with invalid references
// are stored.
if (invalidReferences.length) {
invalidReferencesMap[key] = invalidReferences;
}
errors.forEach((error) => {
dataTreeEvalRef.errors.push(error);
});
});
dependencyMap = makeParentsDependOnChildren(
dependencyMap,
dataTreeEvalRef.allKeys,
);
return { dependencyMap, triggerFieldDependencyMap };
return { dependencyMap, triggerFieldDependencyMap, invalidReferencesMap };
}
interface UpdateDependencyMap {
dependenciesOfRemovedPaths: string[];
removedPaths: string[];
/** Some paths do not need to go through evaluation, but require linting
* For example:
* 1. For changes in paths that trigger fields depend on, the triggerFields need to be "linted" but not evaluated.
* 2. Paths containing invalid references - Eg. for binding {{Api1.unknown}} in button.text, although Api1.unknown
* is not a valid reference, when Api1 is deleted button.text needs to be linted
*/
extraPathsToLint: string[];
}
export const updateDependencyMap = ({
dataTreeEvalRef,
translatedDiffs,
@ -95,13 +132,12 @@ export const updateDependencyMap = ({
dataTreeEvalRef: DataTreeEvaluator;
translatedDiffs: Array<DataTreeDiff>;
unEvalDataTree: DataTree;
}) => {
}): UpdateDependencyMap => {
const diffCalcStart = performance.now();
let didUpdateDependencyMap = false;
let triggerPathsToLint: string[] = [];
let didUpdateTriggerDependencyMap = false;
const dependenciesOfRemovedPaths: Array<string> = [];
const removedPaths: Array<string> = [];
const extraPathsToLint = new Set<string>();
// This is needed for NEW and DELETE events below.
// In worst case, it tends to take ~12.5% of entire diffCalc (8 ms out of 67ms for 132 array of NEW)
@ -109,7 +145,9 @@ export const updateDependencyMap = ({
dataTreeEvalRef.allKeys = getAllPaths(unEvalDataTree);
// Transform the diff library events to Appsmith evaluator events
translatedDiffs.forEach((dataTreeDiff) => {
const entityName = dataTreeDiff.payload.propertyPath.split(".")[0];
const { entityName } = getEntityNameAndPropertyPath(
dataTreeDiff.payload.propertyPath,
);
let entity = unEvalDataTree[entityName];
if (dataTreeDiff.event === DataTreeDiffEvent.DELETE) {
entity = dataTreeEvalRef.oldUnEvalTree[entityName];
@ -119,7 +157,8 @@ export const updateDependencyMap = ({
if (entityType !== "noop") {
switch (dataTreeDiff.event) {
case DataTreeDiffEvent.NEW: {
// If a new entity/property was added, add all the internal bindings for this entity to the global dependency map
// If a new entity/property was added,
// add all the internal bindings for this entity to the global dependency map
if (
(isWidget(entity) || isAction(entity) || isJSAction(entity)) &&
!isDynamicLeaf(unEvalDataTree, dataTreeDiff.payload.propertyPath)
@ -130,66 +169,188 @@ export const updateDependencyMap = ({
);
if (Object.keys(entityDependencyMap).length) {
didUpdateDependencyMap = true;
// The entity might already have some dependencies,
// so we just want to update those
Object.entries(entityDependencyMap).forEach(
([entityDependent, entityDependencies]) => {
if (dataTreeEvalRef.dependencyMap[entityDependent]) {
dataTreeEvalRef.dependencyMap[
const {
errors,
invalidReferences,
validReferences,
} = extractInfoFromBindings(
entityDependencies,
dataTreeEvalRef.allKeys,
);
// Update dependencyMap
dataTreeEvalRef.dependencyMap[entityDependent] = mergeArrays(
dataTreeEvalRef.dependencyMap[entityDependent],
validReferences,
);
// Update invalidReferencesMap
if (invalidReferences.length) {
dataTreeEvalRef.invalidReferencesMap[
entityDependent
] = dataTreeEvalRef.dependencyMap[entityDependent].concat(
entityDependencies,
] = invalidReferences;
} else {
delete dataTreeEvalRef.invalidReferencesMap[
entityDependent
];
}
errors.forEach((error) => {
dataTreeEvalRef.errors.push(error);
});
},
);
}
// For widgets, we need to update the triggerfield dependencyMap
if (isWidget(entity)) {
const triggerFieldDependencies = listTriggerFieldDependencies(
entity,
entityName,
);
Object.entries(triggerFieldDependencies).forEach(
([triggerFieldDependent, triggerFieldDependencies]) => {
const {
errors,
invalidReferences,
validReferences,
} = extractInfoFromBindings(
triggerFieldDependencies,
dataTreeEvalRef.allKeys,
);
// Update triggerfield dependencyMap
dataTreeEvalRef.triggerFieldDependencyMap[
triggerFieldDependent
] = mergeArrays(
dataTreeEvalRef.triggerFieldDependencyMap[
triggerFieldDependent
],
validReferences,
);
// Update invalidReferencesMap
if (invalidReferences.length) {
dataTreeEvalRef.invalidReferencesMap[
triggerFieldDependent
] = invalidReferences;
} else {
delete dataTreeEvalRef.invalidReferencesMap[
triggerFieldDependent
];
}
errors.forEach((error) => {
dataTreeEvalRef.errors.push(error);
});
},
);
}
}
// Either a new entity or a new property path has been added. Go through the list of invalid references and
// find out if a new dependency has to be created because the property path used in the binding just became
// eligible (a previously invalid reference has become valid because a new entity/path got added).
const newlyValidReferencesMap: DependencyMap = {};
Object.keys(dataTreeEvalRef.invalidReferencesMap).forEach((path) => {
dataTreeEvalRef.invalidReferencesMap[path].forEach(
(invalidReference) => {
if (
isChildPropertyPath(
dataTreeDiff.payload.propertyPath,
invalidReference,
)
) {
newlyValidReferencesMap[
invalidReference
] = mergeArrays(newlyValidReferencesMap[invalidReference], [
path,
]);
if (!dataTreeEvalRef.dependencyMap[invalidReference]) {
extraPathsToLint.add(path);
}
}
},
);
});
// We have found some bindings which are related to the new property path and hence should be added to the
// global dependency map
if (Object.keys(newlyValidReferencesMap).length) {
didUpdateDependencyMap = true;
Object.keys(newlyValidReferencesMap).forEach((reference) => {
const { validReferences } = extractInfoFromReferences(
[reference],
dataTreeEvalRef.allKeys,
);
newlyValidReferencesMap[reference].forEach((path) => {
const {
entityName,
propertyPath,
} = getEntityNameAndPropertyPath(path);
const entity = unEvalDataTree[entityName];
if (validReferences.length) {
// For trigger paths, update the triggerfield dependency map
// For other paths, update the dependency map
if (
isWidget(entity) &&
isPathADynamicTrigger(entity, propertyPath)
) {
dataTreeEvalRef.triggerFieldDependencyMap[
path
] = mergeArrays(
dataTreeEvalRef.triggerFieldDependencyMap[path],
validReferences,
);
} else {
dataTreeEvalRef.dependencyMap[
entityDependent
] = entityDependencies;
dataTreeEvalRef.dependencyMap[path] = mergeArrays(
dataTreeEvalRef.dependencyMap[path],
validReferences,
);
}
// Since the previously invalid reference has become valid,
// remove it from the invalidReferencesMap
if (dataTreeEvalRef.invalidReferencesMap[path]) {
const newInvalidReferences = dataTreeEvalRef.invalidReferencesMap[
path
].filter(
(invalidReference) =>
invalidReference !== invalidReference,
);
if (newInvalidReferences.length) {
dataTreeEvalRef.invalidReferencesMap[
path
] = newInvalidReferences;
} else {
delete dataTreeEvalRef.invalidReferencesMap[path];
}
}
}
});
});
}
// Add trigger paths that depend on the added path/entity to "extrapathstolint"
Object.keys(dataTreeEvalRef.triggerFieldDependencyMap).forEach(
(triggerPath) => {
dataTreeEvalRef.triggerFieldDependencyMap[triggerPath].forEach(
(triggerPathDependency) => {
if (
isChildPropertyPath(
dataTreeDiff.payload.propertyPath,
triggerPathDependency,
)
) {
extraPathsToLint.add(triggerPath);
}
},
);
}
}
// Either a new entity or a new property path has been added. Go through existing dynamic bindings and
// find out if a new dependency has to be created because the property path used in the binding just became
// eligible
const possibleReferencesInOldBindings: DependencyMap = dataTreeEvalRef.getPropertyPathReferencesInExistingBindings(
unEvalDataTree,
dataTreeDiff.payload.propertyPath,
},
);
// We have found some bindings which are related to the new property path and hence should be added to the
// global dependency map
if (Object.keys(possibleReferencesInOldBindings).length) {
didUpdateDependencyMap = true;
Object.assign(
dataTreeEvalRef.dependencyMap,
possibleReferencesInOldBindings,
);
}
// When a new Entity is added, check if a new dependency has been created because the property path used in the binding just became valid
if (entityName === dataTreeDiff.payload.propertyPath) {
const possibleTriggerFieldReferences = dataTreeEvalRef.getTriggerFieldReferencesInExistingBindings(
unEvalDataTree,
entityName,
);
if (Object.keys(possibleTriggerFieldReferences).length) {
didUpdateTriggerDependencyMap = true;
Object.assign(
dataTreeEvalRef.triggerFieldDependencyMap,
possibleTriggerFieldReferences,
);
Object.keys(possibleTriggerFieldReferences).forEach(
(triggerPath) => {
triggerPathsToLint.push(triggerPath);
},
);
}
}
break;
}
case DataTreeDiffEvent.DELETE: {
// Add to removedPaths as they have been deleted from the evalTree
removedPaths.push(dataTreeDiff.payload.propertyPath);
// If an existing widget was deleted, remove all the bindings from the global dependency map
// If an existing entity was deleted, remove all the bindings from the global dependency map
if (
(isWidget(entity) || isAction(entity) || isJSAction(entity)) &&
dataTreeDiff.payload.propertyPath === entityName
@ -201,7 +362,19 @@ export const updateDependencyMap = ({
Object.keys(entityDependencies).forEach((widgetDep) => {
didUpdateDependencyMap = true;
delete dataTreeEvalRef.dependencyMap[widgetDep];
delete dataTreeEvalRef.invalidReferencesMap[widgetDep];
});
if (isWidget(entity)) {
const triggerFieldDependencies = listTriggerFieldDependencies(
entity,
entityName,
);
Object.keys(triggerFieldDependencies).forEach((triggerDep) => {
delete dataTreeEvalRef.triggerFieldDependencyMap[triggerDep];
delete dataTreeEvalRef.invalidReferencesMap[triggerDep];
});
}
}
// Either an existing entity or an existing property path has been deleted. Update the global dependency map
// by removing the bindings from the same.
@ -215,6 +388,7 @@ export const updateDependencyMap = ({
)
) {
delete dataTreeEvalRef.dependencyMap[dependencyPath];
delete dataTreeEvalRef.invalidReferencesMap[dependencyPath];
} else {
const toRemove: Array<string> = [];
dataTreeEvalRef.dependencyMap[dependencyPath].forEach(
@ -234,41 +408,96 @@ export const updateDependencyMap = ({
dataTreeEvalRef.dependencyMap[dependencyPath],
toRemove,
);
// If we find any invalid reference (untracked in the dependency map) for this path,
// which is a child of the deleted path, add it to the of paths to lint.
// Example scenario => For {{Api1.unknown}} in button.text, if Api1 is deleted, we need to lint button.text
// Although, "Api1.unknown" is not a valid reference
if (dataTreeEvalRef.invalidReferencesMap[dependencyPath]) {
dataTreeEvalRef.invalidReferencesMap[dependencyPath].forEach(
(invalidReference) => {
if (
isChildPropertyPath(
dataTreeDiff.payload.propertyPath,
invalidReference,
)
) {
extraPathsToLint.add(dependencyPath);
}
},
);
}
// Since we are removing previously valid references,
// We also update the invalidReferenceMap for this path
if (toRemove.length) {
dataTreeEvalRef.invalidReferencesMap[
dependencyPath
] = mergeArrays(
dataTreeEvalRef.invalidReferencesMap[dependencyPath],
toRemove,
);
}
}
},
);
if (entityName === dataTreeDiff.payload.propertyPath) {
// When deleted entity is referenced in a trigger field, remove deleted entity from it's triggerfieldDependencyMap
if (
entityName in dataTreeEvalRef.triggerFieldInverseDependencyMap
) {
triggerPathsToLint = triggerPathsToLint.concat(
dataTreeEvalRef.triggerFieldInverseDependencyMap[entityName],
);
didUpdateTriggerDependencyMap = true;
dataTreeEvalRef.triggerFieldInverseDependencyMap[
entityName
].forEach((triggerField) => {
if (!dataTreeEvalRef.triggerFieldDependencyMap[triggerField])
return;
dataTreeEvalRef.triggerFieldDependencyMap[
triggerField
] = dataTreeEvalRef.triggerFieldDependencyMap[
triggerField
].filter((field) => field !== entityName);
});
}
// Remove deleted trigger fields from triggerFieldDependencyMap
if (isWidget(entity)) {
entity.dynamicTriggerPathList?.forEach((triggerFieldName) => {
Object.keys(dataTreeEvalRef.triggerFieldDependencyMap).forEach(
(dependencyPath) => {
if (
isChildPropertyPath(
dataTreeDiff.payload.propertyPath,
dependencyPath,
)
) {
delete dataTreeEvalRef.triggerFieldDependencyMap[
`${entityName}.${triggerFieldName.key}`
dependencyPath
];
didUpdateTriggerDependencyMap = true;
});
}
}
delete dataTreeEvalRef.invalidReferencesMap[dependencyPath];
} else {
const toRemove: Array<string> = [];
dataTreeEvalRef.triggerFieldDependencyMap[
dependencyPath
].forEach((dependantPath) => {
if (
isChildPropertyPath(
dataTreeDiff.payload.propertyPath,
dependantPath,
)
) {
toRemove.push(dependantPath);
}
});
dataTreeEvalRef.triggerFieldDependencyMap[
dependencyPath
] = difference(
dataTreeEvalRef.triggerFieldDependencyMap[dependencyPath],
toRemove,
);
if (toRemove.length) {
dataTreeEvalRef.invalidReferencesMap[
dependencyPath
] = mergeArrays(
dataTreeEvalRef.invalidReferencesMap[dependencyPath],
toRemove,
);
}
if (dataTreeEvalRef.invalidReferencesMap[dependencyPath]) {
dataTreeEvalRef.invalidReferencesMap[dependencyPath].forEach(
(invalidReference) => {
if (
isChildPropertyPath(
dataTreeDiff.payload.propertyPath,
invalidReference,
)
) {
extraPathsToLint.add(dependencyPath);
}
},
);
}
}
},
);
break;
}
@ -302,12 +531,33 @@ export const updateDependencyMap = ({
const correctSnippets = jsSnippets.filter(
(jsSnippet) => !!jsSnippet,
);
const {
errors,
invalidReferences,
validReferences,
} = extractInfoFromBindings(
correctSnippets,
dataTreeEvalRef.allKeys,
);
if (invalidReferences.length) {
dataTreeEvalRef.invalidReferencesMap[
fullPropertyPath
] = invalidReferences;
} else {
delete dataTreeEvalRef.invalidReferencesMap[fullPropertyPath];
}
errors.forEach((error) => {
dataTreeEvalRef.errors.push(error);
});
// We found a new dynamic binding for this property path. We update the dependency map by overwriting the
// dependencies for this property path with the newly found dependencies
if (correctSnippets.length) {
dataTreeEvalRef.dependencyMap[
fullPropertyPath
] = correctSnippets;
] = validReferences;
} else {
// The dependency on this property path has been removed. Delete this property path from the global
// dependency map
@ -320,22 +570,40 @@ export const updateDependencyMap = ({
entityPropertyPath
].map((dep) => `${entityName}.${dep}`);
// Filter only the paths which exist in the appsmith world to avoid cyclical dependencies
const filteredEntityDependencies = entityDependenciesName.filter(
(path) => dataTreeEvalRef.allKeys.hasOwnProperty(path),
const {
errors,
invalidReferences,
validReferences,
} = extractInfoFromBindings(
entityDependenciesName,
dataTreeEvalRef.allKeys,
);
if (invalidReferences.length) {
dataTreeEvalRef.invalidReferencesMap[
dataTreeDiff.payload.propertyPath
] = invalidReferences;
} else {
delete dataTreeEvalRef.invalidReferencesMap[
dataTreeDiff.payload.propertyPath
];
}
errors.forEach((error) => {
dataTreeEvalRef.errors.push(error);
});
// Now assign these existing dependent paths to the property path in dependencyMap
if (fullPropertyPath in dataTreeEvalRef.dependencyMap) {
dataTreeEvalRef.dependencyMap[
fullPropertyPath
] = dataTreeEvalRef.dependencyMap[fullPropertyPath].concat(
filteredEntityDependencies,
validReferences,
);
} else {
dataTreeEvalRef.dependencyMap[
fullPropertyPath
] = filteredEntityDependencies;
] = validReferences;
}
}
}
@ -348,6 +616,7 @@ export const updateDependencyMap = ({
) {
didUpdateDependencyMap = true;
delete dataTreeEvalRef.dependencyMap[fullPropertyPath];
delete dataTreeEvalRef.invalidReferencesMap[fullPropertyPath];
}
}
if (
@ -364,16 +633,33 @@ export const updateDependencyMap = ({
const entityDependencies = jsSnippets.filter(
(jsSnippet) => !!jsSnippet,
);
const extractedEntityDependencies = getEntityReferencesFromPropertyBindings(
const {
errors,
invalidReferences,
validReferences,
} = extractInfoFromBindings(
entityDependencies,
dataTreeEvalRef,
dataTreeEvalRef.allKeys,
);
errors.forEach((error) => {
dataTreeEvalRef.errors.push(error);
});
if (invalidReferences.length) {
dataTreeEvalRef.invalidReferencesMap[
dataTreeDiff.payload.propertyPath
] = invalidReferences;
} else {
delete dataTreeEvalRef.invalidReferencesMap[
dataTreeDiff.payload.propertyPath
];
}
dataTreeEvalRef.triggerFieldDependencyMap[
dataTreeDiff.payload.propertyPath
] = extractedEntityDependencies;
didUpdateTriggerDependencyMap = true;
] = validReferences;
}
break;
}
@ -386,30 +672,6 @@ export const updateDependencyMap = ({
const diffCalcEnd = performance.now();
const subDepCalcStart = performance.now();
if (didUpdateDependencyMap) {
// TODO Optimise
Object.keys(dataTreeEvalRef.dependencyMap).forEach((key) => {
dataTreeEvalRef.dependencyMap[key] = uniq(
flatten(
dataTreeEvalRef.dependencyMap[key].map((path) => {
try {
return extractReferencesFromBinding(
path,
dataTreeEvalRef.allKeys,
);
} catch (error) {
dataTreeEvalRef.errors.push({
type: EvalErrorTypes.EXTRACT_DEPENDENCY_ERROR,
message: (error as Error).message,
context: {
script: path,
},
});
return [];
}
}),
),
);
});
dataTreeEvalRef.dependencyMap = makeParentsDependOnChildren(
dataTreeEvalRef.dependencyMap,
dataTreeEvalRef.allKeys,
@ -427,10 +689,6 @@ export const updateDependencyMap = ({
);
dataTreeEvalRef.inverseDependencyMap = dataTreeEvalRef.getInverseDependencyTree();
}
if (didUpdateTriggerDependencyMap) {
dataTreeEvalRef.triggerFieldInverseDependencyMap = dataTreeEvalRef.getInverseTriggerDependencyMap();
}
const updateChangedDependenciesStop = performance.now();
dataTreeEvalRef.logs.push({
diffCalcDeps: (diffCalcEnd - diffCalcStart).toFixed(2),
@ -440,5 +698,9 @@ export const updateDependencyMap = ({
).toFixed(2),
});
return { dependenciesOfRemovedPaths, removedPaths, triggerPathsToLint };
return {
dependenciesOfRemovedPaths,
removedPaths,
extraPathsToLint: Array.from(extraPathsToLint),
};
};

196
app/client/src/workers/DependencyMap/utils.ts
View File

@ -1,27 +1,71 @@
import { flatten } from "lodash";
import { get, union } from "lodash";
import toPath from "lodash/toPath";
import { EvalErrorTypes } from "utils/DynamicBindingUtils";
import { extractIdentifiersFromCode } from "@shared/ast";
import DataTreeEvaluator from "workers/DataTreeEvaluator";
import { convertPathToString } from "../evaluationUtils";
import {
EvalErrorTypes,
EvalError,
DependencyMap,
getDynamicBindings,
extraLibrariesNames,
} from "utils/DynamicBindingUtils";
import { extractInfoFromCode } from "@shared/ast";
import { convertPathToString, isWidget } from "../evaluationUtils";
import { DataTreeWidget } from "entities/DataTree/dataTreeFactory";
import {
DEDICATED_WORKER_GLOBAL_SCOPE_IDENTIFIERS,
JAVASCRIPT_KEYWORDS,
} from "constants/WidgetValidation";
import { APPSMITH_GLOBAL_FUNCTIONS } from "components/editorComponents/ActionCreator/constants";
export const extractReferencesFromBinding = (
/** This function extracts validReferences and invalidReferences from a binding {{}}
* @param script
* @param allPaths
* @returns validReferences - Valid references from bindings
* invalidReferences- References which are currently invalid
* @example - For binding {{unknownEntity.name + Api1.name}}, it returns
* {
* validReferences:[Api1.name],
* invalidReferences: [unknownEntity.name]
* }
*/
export const extractInfoFromBinding = (
script: string,
allPaths: Record<string, true>,
): string[] => {
const references: Set<string> = new Set<string>();
const identifiers = extractIdentifiersFromCode(
): { validReferences: string[]; invalidReferences: string[] } => {
const { references } = extractInfoFromCode(
script,
self?.evaluationVersion,
self.evaluationVersion,
invalidEntityIdentifiers,
);
return extractInfoFromReferences(references, allPaths);
};
identifiers.forEach((identifier: string) => {
/** This function extracts validReferences and invalidReferences from an Array of Identifiers
* @param references
* @param allPaths
* @returns validReferences - Valid references from bindings
* invalidReferences- References which are currently invalid
* @example - For identifiers [unknownEntity.name , Api1.name], it returns
* {
* validReferences:[Api1.name],
* invalidReferences: [unknownEntity.name]
* }
*/
export const extractInfoFromReferences = (
references: string[],
allPaths: Record<string, true>,
): {
validReferences: string[];
invalidReferences: string[];
} => {
const validReferences: Set<string> = new Set<string>();
const invalidReferences: string[] = [];
references.forEach((reference: string) => {
// If the identifier exists directly, add it and return
if (allPaths.hasOwnProperty(identifier)) {
references.add(identifier);
if (allPaths.hasOwnProperty(reference)) {
validReferences.add(reference);
return;
}
const subpaths = toPath(identifier);
const subpaths = toPath(reference);
let current = "";
// We want to keep going till we reach top level, but not add top level
// Eg: Input1.text should not depend on entire Table1 unless it explicitly asked for that.
@ -31,48 +75,102 @@ export const extractReferencesFromBinding = (
current = convertPathToString(subpaths);
// We've found the dep, add it and return
if (allPaths.hasOwnProperty(current)) {
references.add(current);
validReferences.add(current);
return;
}
subpaths.pop();
}
// If no valid reference is derived, add it to the list of invalidReferences
invalidReferences.push(reference);
});
return Array.from(references);
return { validReferences: Array.from(validReferences), invalidReferences };
};
interface BindingsInfo {
validReferences: string[];
invalidReferences: string[];
errors: EvalError[];
}
export const extractInfoFromBindings = (
bindings: string[],
allPaths: Record<string, true>,
) => {
return bindings.reduce(
(bindingsInfo: BindingsInfo, binding) => {
try {
const { invalidReferences, validReferences } = extractInfoFromBinding(
binding,
allPaths,
);
return {
...bindingsInfo,
validReferences: union(bindingsInfo.validReferences, validReferences),
invalidReferences: union(
bindingsInfo.invalidReferences,
invalidReferences,
),
};
} catch (error) {
const newEvalError: EvalError = {
type: EvalErrorTypes.EXTRACT_DEPENDENCY_ERROR,
message: (error as Error).message,
context: {
script: binding,
},
};
return {
...bindingsInfo,
errors: union(bindingsInfo.errors, [newEvalError]),
};
}
},
{ validReferences: [], invalidReferences: [], errors: [] },
);
};
export function listTriggerFieldDependencies(
entity: DataTreeWidget,
entityName: string,
): DependencyMap {
const triggerFieldDependency: DependencyMap = {};
if (isWidget(entity)) {
const dynamicTriggerPathlist = entity.dynamicTriggerPathList;
if (dynamicTriggerPathlist && dynamicTriggerPathlist.length) {
dynamicTriggerPathlist.forEach((dynamicPath) => {
const propertyPath = dynamicPath.key;
const unevalPropValue = get(entity, propertyPath);
const { jsSnippets } = getDynamicBindings(unevalPropValue);
const existingDeps =
triggerFieldDependency[`${entityName}.${propertyPath}`] || [];
triggerFieldDependency[
`${entityName}.${propertyPath}`
] = existingDeps.concat(jsSnippets.filter((jsSnippet) => !!jsSnippet));
});
}
}
return triggerFieldDependency;
}
/**This function returns a unique array containing a merge of both arrays
* @param currentArr
* @param updateArr
* @returns A unique array containing a merge of both arrays
*/
export const mergeArrays = <T>(currentArr: T[], updateArr: T[]): T[] => {
if (!currentArr) return updateArr;
return union(currentArr, updateArr);
};
/**
*
* @param propertyBindings
* @returns list of entities referenced in propertyBindings
* Eg. [Api1.run(), Api2.data, Api1.data] => [Api1, Api2]
* Identifiers which can not be valid names of entities and are not dynamic in nature.
* therefore should be removed from the list of references extracted from code.
* NB: DATA_TREE_KEYWORDS in app/client/src/constants/WidgetValidation.ts isn't included, although they are not valid entity names,
* they can refer to potentially dynamic entities.
* Eg. "appsmith"
*/
export const getEntityReferencesFromPropertyBindings = (
propertyBindings: string[],
dataTreeEvalRef: DataTreeEvaluator,
): string[] => {
return flatten(
propertyBindings.map((binding) => {
{
try {
return [
...new Set(
extractReferencesFromBinding(
binding,
dataTreeEvalRef.allKeys,
).map((reference) => reference.split(".")[0]),
),
];
} catch (error) {
dataTreeEvalRef.errors.push({
type: EvalErrorTypes.EXTRACT_DEPENDENCY_ERROR,
message: (error as Error).message,
context: {
script: binding,
},
});
return [];
}
}
}),
);
const invalidEntityIdentifiers: Record<string, unknown> = {
...JAVASCRIPT_KEYWORDS,
...APPSMITH_GLOBAL_FUNCTIONS,
...DEDICATED_WORKER_GLOBAL_SCOPE_IDENTIFIERS,
...extraLibrariesNames,
};

30
app/client/src/workers/Lint/index.ts
View File

@ -1,5 +1,5 @@
import { DataTree, DataTreeEntity } from "entities/DataTree/dataTreeFactory";
import { get } from "lodash";
import { get, union } from "lodash";
import { EvaluationError, getDynamicBindings } from "utils/DynamicBindingUtils";
import {
createGlobalData,
@ -18,25 +18,25 @@ import {
import { getJSToLint, getLintingErrors, pathRequiresLinting } from "./utils";
interface LintTreeArgs {
extraPathsToLint: string[];
unEvalTree: DataTree;
evalTree: DataTree;
sortedDependencies: string[];
triggerPathsToLint: string[];
}
export const lintTree = (args: LintTreeArgs) => {
const { evalTree, sortedDependencies, triggerPathsToLint, unEvalTree } = args;
const { evalTree, extraPathsToLint, sortedDependencies, unEvalTree } = args;
const GLOBAL_DATA_WITHOUT_FUNCTIONS = createGlobalData({
dataTree: unEvalTree,
resolvedFunctions: {},
isTriggerBased: false,
});
// trigger paths
const triggerPaths = [...triggerPathsToLint];
const triggerPaths = new Set<string>();
// Certain paths, like JS Object's body are binding paths where appsmith functions are needed in the global data
const bindingPathsRequiringFunctions: string[] = [];
const bindingPathsRequiringFunctions = new Set<string>();
sortedDependencies.forEach((fullPropertyPath) => {
union(sortedDependencies, extraPathsToLint).forEach((fullPropertyPath) => {
const { entityName, propertyPath } = getEntityNameAndPropertyPath(
fullPropertyPath,
);
@ -50,10 +50,9 @@ export const lintTree = (args: LintTreeArgs) => {
// We are only interested in paths that require linting
if (!pathRequiresLinting(unEvalTree, entity, fullPropertyPath)) return;
if (isATriggerPath(entity, propertyPath))
return triggerPaths.push(fullPropertyPath);
return triggerPaths.add(fullPropertyPath);
if (isJSAction(entity))
return bindingPathsRequiringFunctions.push(fullPropertyPath);
return bindingPathsRequiringFunctions.add(`${entityName}.body`);
const lintErrors = lintBindingPath(
unEvalPropertyValue,
entity,
@ -64,7 +63,7 @@ export const lintTree = (args: LintTreeArgs) => {
addErrorToEntityProperty(lintErrors, evalTree, fullPropertyPath);
});
if (triggerPaths.length || bindingPathsRequiringFunctions.length) {
if (triggerPaths.size || bindingPathsRequiringFunctions.size) {
// we only create GLOBAL_DATA_WITH_FUNCTIONS if there are paths requiring it
// In trigger based fields, functions such as showAlert, storeValue, etc need to be added to the global data
const GLOBAL_DATA_WITH_FUNCTIONS = createGlobalData({
@ -75,7 +74,7 @@ export const lintTree = (args: LintTreeArgs) => {
});
// lint binding paths that need GLOBAL_DATA_WITH_FUNCTIONS
if (bindingPathsRequiringFunctions.length) {
if (bindingPathsRequiringFunctions.size) {
bindingPathsRequiringFunctions.forEach((fullPropertyPath) => {
const { entityName } = getEntityNameAndPropertyPath(fullPropertyPath);
const entity = unEvalTree[entityName];
@ -83,6 +82,8 @@ export const lintTree = (args: LintTreeArgs) => {
unEvalTree,
fullPropertyPath,
) as unknown) as string;
// remove all lint errors from path
removeLintErrorsFromEntityProperty(evalTree, fullPropertyPath);
const lintErrors = lintBindingPath(
unEvalPropertyValue,
entity,
@ -95,7 +96,7 @@ export const lintTree = (args: LintTreeArgs) => {
}
// Lint triggerPaths
if (triggerPaths.length) {
if (triggerPaths.size) {
triggerPaths.forEach((triggerPath) => {
const { entityName } = getEntityNameAndPropertyPath(triggerPath);
const entity = unEvalTree[entityName];
@ -132,7 +133,7 @@ const lintBindingPath = (
);
if (stringSegments) {
jsSnippets.map((jsSnippet, index) => {
jsSnippets.forEach((jsSnippet, index) => {
if (jsSnippet) {
const jsSnippetToLint = getJSToLint(entity, jsSnippet, propertyPath);
// {{user's code}}
@ -143,12 +144,13 @@ const lintBindingPath = (
);
const scriptType = getScriptType(false, false);
const scriptToLint = getScriptToEval(jsSnippetToLint, scriptType);
lintErrors = getLintingErrors(
const lintErrorsFromSnippet = getLintingErrors(
scriptToLint,
globalData,
originalBinding,
scriptType,
);
lintErrors = lintErrors.concat(lintErrorsFromSnippet);
}
});
}

102
app/client/src/workers/Lint/utils.ts
View File

@ -25,11 +25,18 @@ import {
getLintErrorMessage,
getLintSeverity,
} from "components/editorComponents/CodeEditor/lintHelpers";
import { ECMA_VERSION } from "@shared/ast";
import {
CustomLintErrorCode,
CUSTOM_LINT_ERRORS,
IGNORED_LINT_ERRORS,
SUPPORTED_WEB_APIS,
} from "components/editorComponents/CodeEditor/constants";
import {
extractInvalidTopLevelMemberExpressionsFromCode,
isLiteralNode,
ECMA_VERSION,
MemberExpressionData,
} from "@shared/ast";
export const pathRequiresLinting = (
dataTree: DataTree,
@ -49,9 +56,8 @@ export const pathRequiresLinting = (
(isAction(entity) || isWidget(entity) || isJSAction(entity)) &&
isPathADynamicBinding(entity, propertyPath);
const requiresLinting =
isADynamicBindingPath &&
(isDynamicValue(unEvalPropertyValue) ||
(isJSAction(entity) && propertyPath === "body"));
(isADynamicBindingPath && isDynamicValue(unEvalPropertyValue)) ||
isJSAction(entity);
return requiresLinting;
};
@ -142,22 +148,31 @@ export const getLintingErrors = (
jshint(script, options);
return getValidLintErrors(jshint.errors, scriptPos).map((lintError) => {
const ch = lintError.character;
return {
errorType: PropertyEvaluationErrorType.LINT,
raw: script,
severity: getLintSeverity(lintError.code),
errorMessage: getLintErrorMessage(lintError.reason),
errorSegment: lintError.evidence,
originalBinding,
// By keeping track of these variables we can highlight the exact text that caused the error.
variables: [lintError.a, lintError.b, lintError.c, lintError.d],
code: lintError.code,
line: lintError.line - scriptPos.line,
ch: lintError.line === scriptPos.line ? ch - scriptPos.ch : ch,
};
});
const jshintErrors = getValidLintErrors(jshint.errors, scriptPos).map(
(lintError) => {
const ch = lintError.character;
return {
errorType: PropertyEvaluationErrorType.LINT,
raw: script,
severity: getLintSeverity(lintError.code),
errorMessage: getLintErrorMessage(lintError.reason),
errorSegment: lintError.evidence,
originalBinding,
// By keeping track of these variables we can highlight the exact text that caused the error.
variables: [lintError.a, lintError.b, lintError.c, lintError.d],
code: lintError.code,
line: lintError.line - scriptPos.line,
ch: lintError.line === scriptPos.line ? ch - scriptPos.ch : ch,
};
},
);
const invalidPropertyErrors = getInvalidPropertyErrorsFromScript(
script,
data,
scriptPos,
originalBinding,
);
return jshintErrors.concat(invalidPropertyErrors);
};
const getValidLintErrors = (lintErrors: LintError[], scriptPos: Position) => {
@ -203,3 +218,50 @@ const getValidLintErrors = (lintErrors: LintError[], scriptPos: Position) => {
return result;
}, []);
};
const getInvalidPropertyErrorsFromScript = (
script: string,
data: Record<string, unknown>,
scriptPos: Position,
originalBinding: string,
) => {
let invalidTopLevelMemberExpressions: MemberExpressionData[] = [];
try {
invalidTopLevelMemberExpressions = extractInvalidTopLevelMemberExpressionsFromCode(
script,
data,
self.evaluationVersion,
);
} catch (e) {}
const invalidPropertyErrors = invalidTopLevelMemberExpressions.map(
({ object, property }) => {
const propertyName = isLiteralNode(property)
? property.value
: property.name;
const objectStartLine = object.loc.start.line - 1;
// For computed member expressions (entity["property"]), add an extra 1 to the start column to account for "[".
const propertyStartColumn = !isLiteralNode(property)
? property.loc.start.column + 1
: property.loc.start.column + 2;
return {
errorType: PropertyEvaluationErrorType.LINT,
raw: script,
severity: getLintSeverity(CustomLintErrorCode.INVALID_ENTITY_PROPERTY),
errorMessage: CUSTOM_LINT_ERRORS[
CustomLintErrorCode.INVALID_ENTITY_PROPERTY
](object.name, propertyName),
errorSegment: `${object.name}.${propertyName}`,
originalBinding,
variables: [propertyName, null, null, null],
code: CustomLintErrorCode.INVALID_ENTITY_PROPERTY,
line: objectStartLine - scriptPos.line,
ch:
objectStartLine === scriptPos.line
? propertyStartColumn - scriptPos.ch
: propertyStartColumn,
};
},
);
return invalidPropertyErrors;
};

412
app/client/src/workers/ast.test.ts
View File

@ -1,412 +0,0 @@
import { extractIdentifiersFromCode, parseJSObjectWithAST } from "workers/ast";
describe("getAllIdentifiers", () => {
it("works properly", () => {
const cases: { script: string; expectedResults: string[] }[] = [
{
// Entity reference
script: "DirectTableReference",
expectedResults: ["DirectTableReference"],
},
{
// One level nesting
script: "TableDataReference.data",
expectedResults: ["TableDataReference.data"],
},
{
// Deep nesting
script: "TableDataDetailsReference.data.details",
expectedResults: ["TableDataDetailsReference.data.details"],
},
{
// Deep nesting
script: "TableDataDetailsMoreReference.data.details.more",
expectedResults: ["TableDataDetailsMoreReference.data.details.more"],
},
{
// Deep optional chaining
script: "TableDataOptionalReference.data?.details.more",
expectedResults: ["TableDataOptionalReference.data"],
},
{
// Deep optional chaining with logical operator
script:
"TableDataOptionalWithLogical.data?.details.more || FallbackTableData.data",
expectedResults: [
"TableDataOptionalWithLogical.data",
"FallbackTableData.data",
],
},
{
// null coalescing
script: "TableDataOptionalWithLogical.data ?? FallbackTableData.data",
expectedResults: [
"TableDataOptionalWithLogical.data",
"FallbackTableData.data",
],
},
{
// Basic map function
script: "Table5.data.map(c => ({ name: c.name }))",
expectedResults: ["Table5.data.map", "c.name"],
},
{
// Literal property search
script: "Table6['data']",
expectedResults: ["Table6"],
},
{
// Deep literal property search
script: "TableDataOptionalReference['data'].details",
expectedResults: ["TableDataOptionalReference"],
},
{
// Array index search
script: "array[8]",
expectedResults: ["array[8]"],
},
{
// Deep array index search
script: "Table7.data[4]",
expectedResults: ["Table7.data[4]"],
},
{
// Deep array index search
script: "Table7.data[4].value",
expectedResults: ["Table7.data[4].value"],
},
{
// string literal and array index search
script: "Table['data'][9]",
expectedResults: ["Table"],
},
{
// array index and string literal search
script: "Array[9]['data']",
expectedResults: ["Array[9]"],
},
{
// Index identifier search
script: "Table8.data[row][name]",
expectedResults: ["Table8.data", "row", "name"],
},
{
// Index identifier search with global
script: "Table9.data[appsmith.store.row]",
expectedResults: ["Table9.data", "appsmith.store.row"],
},
{
// Index literal with further nested lookups
script: "Table10.data[row].name",
expectedResults: ["Table10.data", "row"],
},
{
// IIFE and if conditions
script:
"(function(){ if(Table11.isVisible) { return Api1.data } else { return Api2.data } })()",
expectedResults: ["Table11.isVisible", "Api1.data", "Api2.data"],
},
{
// Functions and arguments
script: "JSObject1.run(Api1.data, Api2.data)",
expectedResults: ["JSObject1.run", "Api1.data", "Api2.data"],
},
{
// IIFE - without braces
script: `function() {
const index = Input1.text
const obj = {
"a": 123
}
return obj[index]
}()`,
expectedResults: ["Input1.text"],
},
{
// IIFE
script: `(function() {
const index = Input2.text
const obj = {
"a": 123
}
return obj[index]
})()`,
expectedResults: ["Input2.text"],
},
{
// arrow IIFE - without braces - will fail
script: `() => {
const index = Input3.text
const obj = {
"a": 123
}
return obj[index]
}()`,
expectedResults: [],
},
{
// arrow IIFE
script: `(() => {
const index = Input4.text
const obj = {
"a": 123
}
return obj[index]
})()`,
expectedResults: ["Input4.text"],
},
{
// Direct object access
script: `{ "a": 123 }[Input5.text]`,
expectedResults: ["Input5.text"],
},
{
// Function declaration and default arguments
script: `function run(apiData = Api1.data) {
return apiData;
}`,
expectedResults: ["Api1.data"],
},
{
// Function declaration with arguments
script: `function run(data) {
return data;
}`,
expectedResults: [],
},
{
// anonymous function with variables
script: `() => {
let row = 0;
const data = {};
while(row < 10) {
data["test__" + row] = Table12.data[row];
row = row += 1;
}
}`,
expectedResults: ["Table12.data"],
},
{
// function with variables
script: `function myFunction() {
let row = 0;
const data = {};
while(row < 10) {
data["test__" + row] = Table13.data[row];
row = row += 1;
}
}`,
expectedResults: ["Table13.data"],
},
{
// expression with arithmetic operations
script: `Table14.data + 15`,
expectedResults: ["Table14.data"],
},
{
// expression with logical operations
script: `Table15.data || [{}]`,
expectedResults: ["Table15.data"],
},
];
cases.forEach((perCase) => {
const references = extractIdentifiersFromCode(perCase.script);
expect(references).toStrictEqual(perCase.expectedResults);
});
});
});
describe("parseJSObjectWithAST", () => {
it("parse js object", () => {
const body = `{
myVar1: [],
myVar2: {},
myFun1: () => {
//write code here
},
myFun2: async () => {
//use async-await or promises
}
}`;
const parsedObject = [
{
key: "myVar1",
value: "[]",
type: "ArrayExpression",
},
{
key: "myVar2",
value: "{}",
type: "ObjectExpression",
},
{
key: "myFun1",
value: "() => {}",
type: "ArrowFunctionExpression",
arguments: [],
},
{
key: "myFun2",
value: "async () => {}",
type: "ArrowFunctionExpression",
arguments: [],
},
];
const resultParsedObject = parseJSObjectWithAST(body);
expect(resultParsedObject).toStrictEqual(parsedObject);
});
it("parse js object with literal", () => {
const body = `{
myVar1: [],
myVar2: {
"a": "app",
},
myFun1: () => {
//write code here
},
myFun2: async () => {
//use async-await or promises
}
}`;
const parsedObject = [
{
key: "myVar1",
value: "[]",
type: "ArrayExpression",
},
{
key: "myVar2",
value: '{\n "a": "app"\n}',
type: "ObjectExpression",
},
{
key: "myFun1",
value: "() => {}",
type: "ArrowFunctionExpression",
arguments: [],
},
{
key: "myFun2",
value: "async () => {}",
type: "ArrowFunctionExpression",
arguments: [],
},
];
const resultParsedObject = parseJSObjectWithAST(body);
expect(resultParsedObject).toStrictEqual(parsedObject);
});
it("parse js object with variable declaration inside function", () => {
const body = `{
myFun1: () => {
const a = {
conditions: [],
requires: 1,
testFunc: () => {},
testFunc2: function(){}
};
},
myFun2: async () => {
//use async-await or promises
}
}`;
const parsedObject = [
{
key: "myFun1",
value: `() => {
const a = {
conditions: [],
requires: 1,
testFunc: () => {},
testFunc2: function () {}
};
}`,
type: "ArrowFunctionExpression",
arguments: [],
},
{
key: "myFun2",
value: "async () => {}",
type: "ArrowFunctionExpression",
arguments: [],
},
];
const resultParsedObject = parseJSObjectWithAST(body);
expect(resultParsedObject).toStrictEqual(parsedObject);
});
it("parse js object with params of all types", () => {
const body = `{
myFun2: async (a,b = Array(1,2,3),c = "", d = [], e = this.myVar1, f = {}, g = function(){}, h = Object.assign({}), i = String(), j = storeValue()) => {
//use async-await or promises
},
}`;
const parsedObject = [
{
key: "myFun2",
value:
'async (a, b = Array(1, 2, 3), c = "", d = [], e = this.myVar1, f = {}, g = function () {}, h = Object.assign({}), i = String(), j = storeValue()) => {}',
type: "ArrowFunctionExpression",
arguments: [
{
paramName: "a",
defaultValue: undefined,
},
{
paramName: "b",
defaultValue: undefined,
},
{
paramName: "c",
defaultValue: undefined,
},
{
paramName: "d",
defaultValue: undefined,
},
{
paramName: "e",
defaultValue: undefined,
},
{
paramName: "f",
defaultValue: undefined,
},
{
paramName: "g",
defaultValue: undefined,
},
{
paramName: "h",
defaultValue: undefined,
},
{
paramName: "i",
defaultValue: undefined,
},
{
paramName: "j",
defaultValue: undefined,
},
],
},
];
const resultParsedObject = parseJSObjectWithAST(body);
expect(resultParsedObject).toEqual(parsedObject);
});
});

411
app/client/src/workers/ast.ts
View File

@ -1,411 +0,0 @@
import { parse, Node } from "acorn";
import { ancestor, simple } from "acorn-walk";
import { ECMA_VERSION, NodeTypes } from "constants/ast";
import { isFinite, isString } from "lodash";
import { sanitizeScript } from "./evaluate";
import { generate } from "astring";
/*
* Valuable links:
*
* * ESTree spec: Javascript AST is called ESTree.
* Each es version has its md file in the repo to find features
* implemented and their node type
* https://github.com/estree/estree
*
* * Acorn: The parser we use to get the AST
* https://github.com/acornjs/acorn
*
* * Acorn walk: The walker we use to traverse the AST
* https://github.com/acornjs/acorn/tree/master/acorn-walk
*
* * AST Explorer: Helpful web tool to see ASTs and its parts
* https://astexplorer.net/
*
*/
type Pattern = IdentifierNode | AssignmentPatternNode;
type Expression = Node;
// doc: https://github.com/estree/estree/blob/master/es5.md#memberexpression
interface MemberExpressionNode extends Node {
type: NodeTypes.MemberExpression;
object: MemberExpressionNode | IdentifierNode;
property: IdentifierNode | LiteralNode;
computed: boolean;
// doc: https://github.com/estree/estree/blob/master/es2020.md#chainexpression
optional?: boolean;
}
// doc: https://github.com/estree/estree/blob/master/es5.md#identifier
interface IdentifierNode extends Node {
type: NodeTypes.Identifier;
name: string;
}
// doc: https://github.com/estree/estree/blob/master/es5.md#variabledeclarator
interface VariableDeclaratorNode extends Node {
type: NodeTypes.VariableDeclarator;
id: IdentifierNode;
init: Expression | null;
}
// doc: https://github.com/estree/estree/blob/master/es5.md#functions
interface Function extends Node {
id: IdentifierNode | null;
params: Pattern[];
}
// doc: https://github.com/estree/estree/blob/master/es5.md#functiondeclaration
interface FunctionDeclarationNode extends Node, Function {
type: NodeTypes.FunctionDeclaration;
}
// doc: https://github.com/estree/estree/blob/master/es5.md#functionexpression
interface FunctionExpressionNode extends Expression, Function {
type: NodeTypes.FunctionExpression;
}
interface ArrowFunctionExpressionNode extends Expression, Function {
type: NodeTypes.ArrowFunctionExpression;
}
export interface ObjectExpression extends Expression {
type: NodeTypes.ObjectExpression;
properties: Array<PropertyNode>;
}
// doc: https://github.com/estree/estree/blob/master/es2015.md#assignmentpattern
interface AssignmentPatternNode extends Node {
type: NodeTypes.AssignmentPattern;
left: Pattern;
}
// doc: https://github.com/estree/estree/blob/master/es5.md#literal
interface LiteralNode extends Node {
type: NodeTypes.Literal;
value: string | boolean | null | number | RegExp;
}
// https://github.com/estree/estree/blob/master/es5.md#property
export interface PropertyNode extends Node {
type: NodeTypes.Property;
key: LiteralNode | IdentifierNode;
value: Node;
kind: "init" | "get" | "set";
}
/* We need these functions to typescript casts the nodes with the correct types */
export const isIdentifierNode = (node: Node): node is IdentifierNode => {
return node.type === NodeTypes.Identifier;
};
const isMemberExpressionNode = (node: Node): node is MemberExpressionNode => {
return node.type === NodeTypes.MemberExpression;
};
const isVariableDeclarator = (node: Node): node is VariableDeclaratorNode => {
return node.type === NodeTypes.VariableDeclarator;
};
const isFunctionDeclaration = (node: Node): node is FunctionDeclarationNode => {
return node.type === NodeTypes.FunctionDeclaration;
};
const isFunctionExpression = (node: Node): node is FunctionExpressionNode => {
return node.type === NodeTypes.FunctionExpression;
};
const isObjectExpression = (node: Node): node is ObjectExpression => {
return node.type === NodeTypes.ObjectExpression;
};
const isAssignmentPatternNode = (node: Node): node is AssignmentPatternNode => {
return node.type === NodeTypes.AssignmentPattern;
};
export const isLiteralNode = (node: Node): node is LiteralNode => {
return node.type === NodeTypes.Literal;
};
export const isPropertyNode = (node: Node): node is PropertyNode => {
return node.type === NodeTypes.Property;
};
export const isPropertyAFunctionNode = (
node: Node,
): node is ArrowFunctionExpressionNode | FunctionExpressionNode => {
return (
node.type === NodeTypes.ArrowFunctionExpression ||
node.type === NodeTypes.FunctionExpression
);
};
const isArrayAccessorNode = (node: Node): node is MemberExpressionNode => {
return (
isMemberExpressionNode(node) &&
node.computed &&
isLiteralNode(node.property) &&
isFinite(node.property.value)
);
};
const wrapCode = (code: string) => {
return `
(function() {
return ${code}
})
`;
};
export const getAST = (code: string) =>
parse(code, { ecmaVersion: ECMA_VERSION });
/**
* An AST based extractor that fetches all possible identifiers in a given
* piece of code. We use this to get any references to the global entities in Appsmith
* and create dependencies on them. If the reference was updated, the given piece of code
* should run again.
* @param code: The piece of script where identifiers need to be extracted from
*/
export const extractIdentifiersFromCode = (code: string): string[] => {
// List of all identifiers found
const identifiers = new Set<string>();
// List of variables declared within the script. This will be removed from identifier list
const variableDeclarations = new Set<string>();
// List of functionalParams found. This will be removed from the identifier list
let functionalParams = new Set<functionParams>();
let ast: Node = { end: 0, start: 0, type: "" };
try {
const sanitizedScript = sanitizeScript(code);
/* wrapCode - Wrapping code in a function, since all code/script get wrapped with a function during evaluation.
Some syntax won't be valid unless they're at the RHS of a statement.
Since we're assigning all code/script to RHS during evaluation, we do the same here.
So that during ast parse, those errors are neglected.
*/
/* e.g. IIFE without braces
function() { return 123; }() -> is invalid
let result = function() { return 123; }() -> is valid
*/
const wrappedCode = wrapCode(sanitizedScript);
ast = getAST(wrappedCode);
} catch (e) {
if (e instanceof SyntaxError) {
// Syntax error. Ignore and return 0 identifiers
return [];
}
throw e;
}
/*
* We do an ancestor walk on the AST to get all identifiers. Since we need to know
* what surrounds the identifier, ancestor walk will give that information in the callback
* doc: https://github.com/acornjs/acorn/tree/master/acorn-walk
*/
ancestor(ast, {
Identifier(node: Node, ancestors: Node[]) {
/*
* We are interested in identifiers. Due to the nature of AST, Identifier nodes can
* also be nested inside MemberExpressions. For deeply nested object references, there
* could be nesting of many MemberExpressions. To find the final reference, we will
* try to find the top level MemberExpression that does not have a MemberExpression parent.
* */
let candidateTopLevelNode:
| IdentifierNode
| MemberExpressionNode = node as IdentifierNode;
let depth = ancestors.length - 2; // start "depth" with first parent
while (depth > 0) {
const parent = ancestors[depth];
if (
isMemberExpressionNode(parent) &&
/* Member expressions that are "computed" (with [ ] search)
and the ones that have optional chaining ( a.b?.c )
will be considered top level node.
We will stop looking for further parents */
/* "computed" exception - isArrayAccessorNode
Member expressions that are array accessors with static index - [9]
will not be considered top level.
We will continue looking further. */
(!parent.computed || isArrayAccessorNode(parent)) &&
!parent.optional
) {
candidateTopLevelNode = parent;
depth = depth - 1;
} else {
// Top level found
break;
}
}
if (isIdentifierNode(candidateTopLevelNode)) {
// If the node is an Identifier, just save that
identifiers.add(candidateTopLevelNode.name);
} else {
// For MemberExpression Nodes, we will construct a final reference string and then add
// it to the identifier list
const memberExpIdentifier = constructFinalMemberExpIdentifier(
candidateTopLevelNode,
);
identifiers.add(memberExpIdentifier);
}
},
VariableDeclarator(node: Node) {
// keep a track of declared variables so they can be
// subtracted from the final list of identifiers
if (isVariableDeclarator(node)) {
variableDeclarations.add(node.id.name);
}
},
FunctionDeclaration(node: Node) {
// params in function declarations are also counted as identifiers so we keep
// track of them and remove them from the final list of identifiers
if (!isFunctionDeclaration(node)) return;
functionalParams = new Set([
...functionalParams,
...getFunctionalParamsFromNode(node),
]);
},
FunctionExpression(node: Node) {
// params in function experssions are also counted as identifiers so we keep
// track of them and remove them from the final list of identifiers
if (!isFunctionExpression(node)) return;
functionalParams = new Set([
...functionalParams,
...getFunctionalParamsFromNode(node),
]);
},
});
// Remove declared variables and function params
variableDeclarations.forEach((variable) => identifiers.delete(variable));
functionalParams.forEach((param) => identifiers.delete(param.paramName));
return Array.from(identifiers);
};
type functionParams = { paramName: string; defaultValue: unknown };
const getFunctionalParamsFromNode = (
node:
| FunctionDeclarationNode
| FunctionExpressionNode
| ArrowFunctionExpressionNode,
needValue = false,
): Set<functionParams> => {
const functionalParams = new Set<functionParams>();
node.params.forEach((paramNode) => {
if (isIdentifierNode(paramNode)) {
functionalParams.add({
paramName: paramNode.name,
defaultValue: undefined,
});
} else if (isAssignmentPatternNode(paramNode)) {
if (isIdentifierNode(paramNode.left)) {
const paramName = paramNode.left.name;
if (!needValue) {
functionalParams.add({ paramName, defaultValue: undefined });
} else {
// figure out how to get value of paramNode.right for each node type
// currently we don't use params value, hence skipping it
// functionalParams.add({
// defaultValue: paramNode.right.value,
// });
}
}
}
});
return functionalParams;
};
const constructFinalMemberExpIdentifier = (
node: MemberExpressionNode,
child = "",
): string => {
const propertyAccessor = getPropertyAccessor(node.property);
if (isIdentifierNode(node.object)) {
return `${node.object.name}${propertyAccessor}${child}`;
} else {
const propertyAccessor = getPropertyAccessor(node.property);
const nestedChild = `${propertyAccessor}${child}`;
return constructFinalMemberExpIdentifier(node.object, nestedChild);
}
};
const getPropertyAccessor = (propertyNode: IdentifierNode | LiteralNode) => {
if (isIdentifierNode(propertyNode)) {
return `.${propertyNode.name}`;
} else if (isLiteralNode(propertyNode) && isString(propertyNode.value)) {
// is string literal search a['b']
return `.${propertyNode.value}`;
} else if (isLiteralNode(propertyNode) && isFinite(propertyNode.value)) {
// is array index search - a[9]
return `[${propertyNode.value}]`;
}
};
export const isTypeOfFunction = (type: string) => {
return (
type === NodeTypes.ArrowFunctionExpression ||
type === NodeTypes.FunctionExpression
);
};
type JsObjectProperty = {
key: string;
value: string;
type: string;
arguments?: Array<functionParams>;
};
export const parseJSObjectWithAST = (
jsObjectBody: string,
): Array<JsObjectProperty> => {
/*
jsObjectVariableName value is added such actual js code would never name same variable name.
if the variable name will be same then also we won't have problem here as jsObjectVariableName will be last node in VariableDeclarator hence overriding the previous JSObjectProperties.
Keeping this just for sanity check if any caveat was missed.
*/
const jsObjectVariableName =
"____INTERNAL_JS_OBJECT_NAME_USED_FOR_PARSING_____";
const jsCode = `var ${jsObjectVariableName} = ${jsObjectBody}`;
const ast = parse(jsCode, { ecmaVersion: ECMA_VERSION });
const parsedObjectProperties = new Set<JsObjectProperty>();
let JSObjectProperties: Array<PropertyNode> = [];
simple(ast, {
VariableDeclarator(node: Node) {
if (
isVariableDeclarator(node) &&
node.id.name === jsObjectVariableName &&
node.init &&
isObjectExpression(node.init)
) {
JSObjectProperties = node.init.properties;
}
},
});
JSObjectProperties.forEach((node) => {
let params = new Set<functionParams>();
const propertyNode = node;
let property: JsObjectProperty = {
key: generate(propertyNode.key),
value: generate(propertyNode.value),
type: propertyNode.value.type,
};
if (isPropertyAFunctionNode(propertyNode.value)) {
// if in future we need default values of each param, we could implement that in getFunctionalParamsFromNode
// currently we don't consume it anywhere hence avoiding to calculate that.
params = getFunctionalParamsFromNode(propertyNode.value);
property = {
...property,
arguments: [...params],
};
}
// here we use `generate` function to convert our AST Node to JSCode
parsedObjectProperties.add(property);
});
return [...parsedObjectProperties];
};

2
app/shared/ast/index.d.ts vendored
View File

@ -1 +1 @@
declare module "@shared/ast";
declare module '@shared/ast';

15
app/shared/ast/index.ts
View File

@ -8,19 +8,21 @@ import {
isPropertyNode,
isPropertyAFunctionNode,
getAST,
extractIdentifiersFromCode,
extractInfoFromCode,
extractInvalidTopLevelMemberExpressionsFromCode,
getFunctionalParamsFromNode,
isTypeOfFunction,
} from "./src/index";
MemberExpressionData,
} from './src';
// constants
import { ECMA_VERSION, SourceType, NodeTypes } from "./src/constants";
import { ECMA_VERSION, SourceType, NodeTypes } from './src/constants';
// JSObjects
import { parseJSObjectWithAST } from "./src/jsObject";
import { parseJSObjectWithAST } from './src/jsObject';
// types or intefaces should be exported with type keyword, while enums can be exported like normal functions
export type { ObjectExpression, PropertyNode };
export type { ObjectExpression, PropertyNode, MemberExpressionData };
export {
isIdentifierNode,
@ -30,7 +32,8 @@ export {
isPropertyNode,
isPropertyAFunctionNode,
getAST,
extractIdentifiersFromCode,
extractInfoFromCode,
extractInvalidTopLevelMemberExpressionsFromCode,
getFunctionalParamsFromNode,
isTypeOfFunction,
parseJSObjectWithAST,

30
app/shared/ast/src/constants.ts
View File

@ -1,30 +0,0 @@
export const ECMA_VERSION = 11;
/* Indicates the mode the code should be parsed in.
This influences global strict mode and parsing of import and export declarations.
*/
export enum SourceType {
script = "script",
module = "module",
}
// Each node has an attached type property which further defines
// what all properties can the node have.
// We will just define the ones we are working with
export enum NodeTypes {
Identifier = "Identifier",
AssignmentPattern = "AssignmentPattern",
Literal = "Literal",
Property = "Property",
// Declaration - https://github.com/estree/estree/blob/master/es5.md#declarations
FunctionDeclaration = "FunctionDeclaration",
ExportDefaultDeclaration = "ExportDefaultDeclaration",
VariableDeclarator = "VariableDeclarator",
// Expression - https://github.com/estree/estree/blob/master/es5.md#expressions
MemberExpression = "MemberExpression",
FunctionExpression = "FunctionExpression",
ArrowFunctionExpression = "ArrowFunctionExpression",
ObjectExpression = "ObjectExpression",
ArrayExpression = "ArrayExpression",
ThisExpression = "ThisExpression",
}

30
app/shared/ast/src/constants/ast.ts Normal file
View File

@ -0,0 +1,30 @@
export const ECMA_VERSION = 11;
/* Indicates the mode the code should be parsed in.
This influences global strict mode and parsing of import and export declarations.
*/
export enum SourceType {
script = 'script',
module = 'module',
}
// Each node has an attached type property which further defines
// what all properties can the node have.
// We will just define the ones we are working with
export enum NodeTypes {
Identifier = 'Identifier',
AssignmentPattern = 'AssignmentPattern',
Literal = 'Literal',
Property = 'Property',
// Declaration - https://github.com/estree/estree/blob/master/es5.md#declarations
FunctionDeclaration = 'FunctionDeclaration',
ExportDefaultDeclaration = 'ExportDefaultDeclaration',
VariableDeclarator = 'VariableDeclarator',
// Expression - https://github.com/estree/estree/blob/master/es5.md#expressions
MemberExpression = 'MemberExpression',
FunctionExpression = 'FunctionExpression',
ArrowFunctionExpression = 'ArrowFunctionExpression',
ObjectExpression = 'ObjectExpression',
ArrayExpression = 'ArrayExpression',
ThisExpression = 'ThisExpression',
}

1
app/shared/ast/src/constants/index.ts Normal file
View File

@ -0,0 +1 @@
export * from './ast';

867
app/shared/ast/src/index.test.ts
View File

@ -1,413 +1,468 @@
import { extractIdentifiersFromCode } from "./index";
import { parseJSObjectWithAST } from "./jsObject";
import { extractInfoFromCode } from '../src/index';
import { parseJSObjectWithAST } from '../src/jsObject';
// describe("getAllIdentifiers", () => {
// it("works properly", () => {
// const cases: { script: string; expectedResults: string[] }[] = [
// {
// // Entity reference
// script: "DirectTableReference",
// expectedResults: ["DirectTableReference"],
// },
// {
// // One level nesting
// script: "TableDataReference.data",
// expectedResults: ["TableDataReference.data"],
// },
// {
// // Deep nesting
// script: "TableDataDetailsReference.data.details",
// expectedResults: ["TableDataDetailsReference.data.details"],
// },
// {
// // Deep nesting
// script: "TableDataDetailsMoreReference.data.details.more",
// expectedResults: ["TableDataDetailsMoreReference.data.details.more"],
// },
// {
// // Deep optional chaining
// script: "TableDataOptionalReference.data?.details.more",
// expectedResults: ["TableDataOptionalReference.data"],
// },
// {
// // Deep optional chaining with logical operator
// script:
// "TableDataOptionalWithLogical.data?.details.more || FallbackTableData.data",
// expectedResults: [
// "TableDataOptionalWithLogical.data",
// "FallbackTableData.data",
// ],
// },
// {
// // null coalescing
// script: "TableDataOptionalWithLogical.data ?? FallbackTableData.data",
// expectedResults: [
// "TableDataOptionalWithLogical.data",
// "FallbackTableData.data",
// ],
// },
// {
// // Basic map function
// script: "Table5.data.map(c => ({ name: c.name }))",
// expectedResults: ["Table5.data.map", "c.name"],
// },
// {
// // Literal property search
// script: "Table6['data']",
// expectedResults: ["Table6"],
// },
// {
// // Deep literal property search
// script: "TableDataOptionalReference['data'].details",
// expectedResults: ["TableDataOptionalReference"],
// },
// {
// // Array index search
// script: "array[8]",
// expectedResults: ["array[8]"],
// },
// {
// // Deep array index search
// script: "Table7.data[4]",
// expectedResults: ["Table7.data[4]"],
// },
// {
// // Deep array index search
// script: "Table7.data[4].value",
// expectedResults: ["Table7.data[4].value"],
// },
// {
// // string literal and array index search
// script: "Table['data'][9]",
// expectedResults: ["Table"],
// },
// {
// // array index and string literal search
// script: "Array[9]['data']",
// expectedResults: ["Array[9]"],
// },
// {
// // Index identifier search
// script: "Table8.data[row][name]",
// expectedResults: ["Table8.data", "row", "name"],
// },
// {
// // Index identifier search with global
// script: "Table9.data[appsmith.store.row]",
// expectedResults: ["Table9.data", "appsmith.store.row"],
// },
// {
// // Index literal with further nested lookups
// script: "Table10.data[row].name",
// expectedResults: ["Table10.data", "row"],
// },
// {
// // IIFE and if conditions
// script:
// "(function(){ if(Table11.isVisible) { return Api1.data } else { return Api2.data } })()",
// expectedResults: ["Table11.isVisible", "Api1.data", "Api2.data"],
// },
// {
// // Functions and arguments
// script: "JSObject1.run(Api1.data, Api2.data)",
// expectedResults: ["JSObject1.run", "Api1.data", "Api2.data"],
// },
// {
// // IIFE - without braces
// script: `function() {
// const index = Input1.text
describe('getAllIdentifiers', () => {
it('works properly', () => {
const cases: { script: string; expectedResults: string[] }[] = [
{
// Entity reference
script: 'DirectTableReference',
expectedResults: ['DirectTableReference'],
},
{
// One level nesting
script: 'TableDataReference.data',
expectedResults: ['TableDataReference.data'],
},
{
// Deep nesting
script: 'TableDataDetailsReference.data.details',
expectedResults: ['TableDataDetailsReference.data.details'],
},
{
// Deep nesting
script: 'TableDataDetailsMoreReference.data.details.more',
expectedResults: ['TableDataDetailsMoreReference.data.details.more'],
},
{
// Deep optional chaining
script: 'TableDataOptionalReference.data?.details.more',
expectedResults: ['TableDataOptionalReference.data'],
},
{
// Deep optional chaining with logical operator
script:
'TableDataOptionalWithLogical.data?.details.more || FallbackTableData.data',
expectedResults: [
'TableDataOptionalWithLogical.data',
'FallbackTableData.data',
],
},
{
// null coalescing
script: 'TableDataOptionalWithLogical.data ?? FallbackTableData.data',
expectedResults: [
'TableDataOptionalWithLogical.data',
'FallbackTableData.data',
],
},
{
// Basic map function
script: 'Table5.data.map(c => ({ name: c.name }))',
expectedResults: ['Table5.data.map'],
},
{
// Literal property search
script: "Table6['data']",
expectedResults: ['Table6'],
},
{
// Deep literal property search
script: "TableDataOptionalReference['data'].details",
expectedResults: ['TableDataOptionalReference'],
},
{
// Array index search
script: 'array[8]',
expectedResults: ['array[8]'],
},
{
// Deep array index search
script: 'Table7.data[4]',
expectedResults: ['Table7.data[4]'],
},
{
// Deep array index search
script: 'Table7.data[4].value',
expectedResults: ['Table7.data[4].value'],
},
{
// string literal and array index search
script: "Table['data'][9]",
expectedResults: ['Table'],
},
{
// array index and string literal search
script: "Array[9]['data']",
expectedResults: [],
},
{
// Index identifier search
script: 'Table8.data[row][name]',
expectedResults: ['Table8.data', 'row'],
},
{
// Index identifier search with global
script: 'Table9.data[appsmith.store.row]',
expectedResults: ['Table9.data', 'appsmith.store.row'],
},
{
// Index literal with further nested lookups
script: 'Table10.data[row].name',
expectedResults: ['Table10.data', 'row'],
},
{
// IIFE and if conditions
script:
'(function(){ if(Table11.isVisible) { return Api1.data } else { return Api2.data } })()',
expectedResults: ['Table11.isVisible', 'Api1.data', 'Api2.data'],
},
{
// Functions and arguments
script: 'JSObject1.run(Api1.data, Api2.data)',
expectedResults: ['JSObject1.run', 'Api1.data', 'Api2.data'],
},
{
// IIFE - without braces
script: `function() {
const index = Input1.text
const obj = {
"a": 123
}
return obj[index]
}()`,
expectedResults: ['Input1.text'],
},
{
// IIFE
script: `(function() {
const index = Input2.text
const obj = {
"a": 123
}
return obj[index]
})()`,
expectedResults: ['Input2.text'],
},
{
// arrow IIFE - without braces - will fail
script: `() => {
const index = Input3.text
const obj = {
"a": 123
}
return obj[index]
}()`,
expectedResults: [],
},
{
// arrow IIFE
script: `(() => {
const index = Input4.text
const obj = {
"a": 123
}
return obj[index]
})()`,
expectedResults: ['Input4.text'],
},
{
// Direct object access
script: `{ "a": 123 }[Input5.text]`,
expectedResults: ['Input5.text'],
},
{
// Function declaration and default arguments
script: `function run(apiData = Api1.data) {
return apiData;
}`,
expectedResults: ['Api1.data'],
},
{
// Function declaration with arguments
script: `function run(data) {
return data;
}`,
expectedResults: [],
},
{
// anonymous function with variables
script: `() => {
let row = 0;
const data = {};
while(row < 10) {
data["test__" + row] = Table12.data[row];
row = row += 1;
}
}`,
expectedResults: ['Table12.data'],
},
{
// function with variables
script: `function myFunction() {
let row = 0;
const data = {};
while(row < 10) {
data["test__" + row] = Table13.data[row];
row = row += 1;
}
}`,
expectedResults: ['Table13.data'],
},
{
// expression with arithmetic operations
script: `Table14.data + 15`,
expectedResults: ['Table14.data'],
},
{
// expression with logical operations
script: `Table15.data || [{}]`,
expectedResults: ['Table15.data'],
},
// JavaScript built in classes should not be valid identifiers
{
script: `function(){
const firstApiRun = Api1.run();
const secondApiRun = Api2.run();
const randomNumber = Math.random();
return Promise.all([firstApiRun, secondApiRun])
}()`,
expectedResults: ['Api1.run', 'Api2.run'],
},
// Global dependencies should not be valid identifiers
{
script: `function(){
const names = [["john","doe"],["Jane","dane"]];
const flattenedNames = _.flatten(names);
return {flattenedNames, time: moment()}
}()`,
expectedResults: [],
},
// browser Apis should not be valid identifiers
{
script: `function(){
const names = {
firstName: "John",
lastName:"Doe"
};
const joinedName = Object.values(names).join(" ");
console.log(joinedName)
return Api2.name
}()`,
expectedResults: ['Api2.name'],
},
// identifiers and member expressions derived from params should not be valid identifiers
{
script: `function(a, b){
return a.name + b.name
}()`,
expectedResults: [],
},
// identifiers and member expressions derived from local variables should not be valid identifiers
{
script: `function(){
const a = "variableA";
const b = "variableB";
return a.length + b.length
}()`,
expectedResults: [],
},
// "appsmith" is an internal identifier and should be a valid reference
{
script: `function(){
return appsmith.user
}()`,
expectedResults: ['appsmith.user'],
},
];
// const obj = {
// "a": 123
// }
cases.forEach((perCase) => {
const { references } = extractInfoFromCode(perCase.script, 2);
expect(references).toStrictEqual(perCase.expectedResults);
});
});
});
// return obj[index]
describe('parseJSObjectWithAST', () => {
it('parse js object', () => {
const body = `{
myVar1: [],
myVar2: {},
myFun1: () => {
//write code here
},
myFun2: async () => {
//use async-await or promises
}
}`;
const parsedObject = [
{
key: 'myVar1',
value: '[]',
type: 'ArrayExpression',
},
{
key: 'myVar2',
value: '{}',
type: 'ObjectExpression',
},
{
key: 'myFun1',
value: '() => {}',
type: 'ArrowFunctionExpression',
arguments: [],
},
{
key: 'myFun2',
value: 'async () => {}',
type: 'ArrowFunctionExpression',
arguments: [],
},
];
const resultParsedObject = parseJSObjectWithAST(body);
expect(resultParsedObject).toStrictEqual(parsedObject);
});
// }()`,
// expectedResults: ["Input1.text"],
// },
// {
// // IIFE
// script: `(function() {
// const index = Input2.text
it('parse js object with literal', () => {
const body = `{
myVar1: [],
myVar2: {
"a": "app",
},
myFun1: () => {
//write code here
},
myFun2: async () => {
//use async-await or promises
}
}`;
const parsedObject = [
{
key: 'myVar1',
value: '[]',
type: 'ArrayExpression',
},
{
key: 'myVar2',
value: '{\n "a": "app"\n}',
type: 'ObjectExpression',
},
{
key: 'myFun1',
value: '() => {}',
type: 'ArrowFunctionExpression',
arguments: [],
},
{
key: 'myFun2',
value: 'async () => {}',
type: 'ArrowFunctionExpression',
arguments: [],
},
];
const resultParsedObject = parseJSObjectWithAST(body);
expect(resultParsedObject).toStrictEqual(parsedObject);
});
// const obj = {
// "a": 123
// }
it('parse js object with variable declaration inside function', () => {
const body = `{
myFun1: () => {
const a = {
conditions: [],
requires: 1,
testFunc: () => {},
testFunc2: function(){}
};
},
myFun2: async () => {
//use async-await or promises
}
}`;
const parsedObject = [
{
key: 'myFun1',
value: `() => {
const a = {
conditions: [],
requires: 1,
testFunc: () => {},
testFunc2: function () {}
};
}`,
type: 'ArrowFunctionExpression',
arguments: [],
},
{
key: 'myFun2',
value: 'async () => {}',
type: 'ArrowFunctionExpression',
arguments: [],
},
];
const resultParsedObject = parseJSObjectWithAST(body);
expect(resultParsedObject).toStrictEqual(parsedObject);
});
// return obj[index]
it('parse js object with params of all types', () => {
const body = `{
myFun2: async (a,b = Array(1,2,3),c = "", d = [], e = this.myVar1, f = {}, g = function(){}, h = Object.assign({}), i = String(), j = storeValue()) => {
//use async-await or promises
},
}`;
// })()`,
// expectedResults: ["Input2.text"],
// },
// {
// // arrow IIFE - without braces - will fail
// script: `() => {
// const index = Input3.text
// const obj = {
// "a": 123
// }
// return obj[index]
// }()`,
// expectedResults: [],
// },
// {
// // arrow IIFE
// script: `(() => {
// const index = Input4.text
// const obj = {
// "a": 123
// }
// return obj[index]
// })()`,
// expectedResults: ["Input4.text"],
// },
// {
// // Direct object access
// script: `{ "a": 123 }[Input5.text]`,
// expectedResults: ["Input5.text"],
// },
// {
// // Function declaration and default arguments
// script: `function run(apiData = Api1.data) {
// return apiData;
// }`,
// expectedResults: ["Api1.data"],
// },
// {
// // Function declaration with arguments
// script: `function run(data) {
// return data;
// }`,
// expectedResults: [],
// },
// {
// // anonymous function with variables
// script: `() => {
// let row = 0;
// const data = {};
// while(row < 10) {
// data["test__" + row] = Table12.data[row];
// row = row += 1;
// }
// }`,
// expectedResults: ["Table12.data"],
// },
// {
// // function with variables
// script: `function myFunction() {
// let row = 0;
// const data = {};
// while(row < 10) {
// data["test__" + row] = Table13.data[row];
// row = row += 1;
// }
// }`,
// expectedResults: ["Table13.data"],
// },
// {
// // expression with arithmetic operations
// script: `Table14.data + 15`,
// expectedResults: ["Table14.data"],
// },
// {
// // expression with logical operations
// script: `Table15.data || [{}]`,
// expectedResults: ["Table15.data"],
// },
// ];
// cases.forEach((perCase) => {
// const references = extractIdentifiersFromCode(perCase.script);
// expect(references).toStrictEqual(perCase.expectedResults);
// });
// });
// });
// describe("parseJSObjectWithAST", () => {
// it("parse js object", () => {
// const body = `{
// myVar1: [],
// myVar2: {},
// myFun1: () => {
// //write code here
// },
// myFun2: async () => {
// //use async-await or promises
// }
// }`;
// const parsedObject = [
// {
// key: "myVar1",
// value: "[]",
// type: "ArrayExpression",
// },
// {
// key: "myVar2",
// value: "{}",
// type: "ObjectExpression",
// },
// {
// key: "myFun1",
// value: "() => {}",
// type: "ArrowFunctionExpression",
// arguments: [],
// },
// {
// key: "myFun2",
// value: "async () => {}",
// type: "ArrowFunctionExpression",
// arguments: [],
// },
// ];
// const resultParsedObject = parseJSObjectWithAST(body);
// expect(resultParsedObject).toStrictEqual(parsedObject);
// });
// it("parse js object with literal", () => {
// const body = `{
// myVar1: [],
// myVar2: {
// "a": "app",
// },
// myFun1: () => {
// //write code here
// },
// myFun2: async () => {
// //use async-await or promises
// }
// }`;
// const parsedObject = [
// {
// key: "myVar1",
// value: "[]",
// type: "ArrayExpression",
// },
// {
// key: "myVar2",
// value: '{\n "a": "app"\n}',
// type: "ObjectExpression",
// },
// {
// key: "myFun1",
// value: "() => {}",
// type: "ArrowFunctionExpression",
// arguments: [],
// },
// {
// key: "myFun2",
// value: "async () => {}",
// type: "ArrowFunctionExpression",
// arguments: [],
// },
// ];
// const resultParsedObject = parseJSObjectWithAST(body);
// expect(resultParsedObject).toStrictEqual(parsedObject);
// });
// it("parse js object with variable declaration inside function", () => {
// const body = `{
// myFun1: () => {
// const a = {
// conditions: [],
// requires: 1,
// testFunc: () => {},
// testFunc2: function(){}
// };
// },
// myFun2: async () => {
// //use async-await or promises
// }
// }`;
// const parsedObject = [
// {
// key: "myFun1",
// value: `() => {
// const a = {
// conditions: [],
// requires: 1,
// testFunc: () => {},
// testFunc2: function () {}
// };
// }`,
// type: "ArrowFunctionExpression",
// arguments: [],
// },
// {
// key: "myFun2",
// value: "async () => {}",
// type: "ArrowFunctionExpression",
// arguments: [],
// },
// ];
// const resultParsedObject = parseJSObjectWithAST(body);
// expect(resultParsedObject).toStrictEqual(parsedObject);
// });
// it("parse js object with params of all types", () => {
// const body = `{
// myFun2: async (a,b = Array(1,2,3),c = "", d = [], e = this.myVar1, f = {}, g = function(){}, h = Object.assign({}), i = String(), j = storeValue()) => {
// //use async-await or promises
// },
// }`;
// const parsedObject = [
// {
// key: "myFun2",
// value:
// 'async (a, b = Array(1, 2, 3), c = "", d = [], e = this.myVar1, f = {}, g = function () {}, h = Object.assign({}), i = String(), j = storeValue()) => {}',
// type: "ArrowFunctionExpression",
// arguments: [
// {
// paramName: "a",
// defaultValue: undefined,
// },
// {
// paramName: "b",
// defaultValue: undefined,
// },
// {
// paramName: "c",
// defaultValue: undefined,
// },
// {
// paramName: "d",
// defaultValue: undefined,
// },
// {
// paramName: "e",
// defaultValue: undefined,
// },
// {
// paramName: "f",
// defaultValue: undefined,
// },
// {
// paramName: "g",
// defaultValue: undefined,
// },
// {
// paramName: "h",
// defaultValue: undefined,
// },
// {
// paramName: "i",
// defaultValue: undefined,
// },
// {
// paramName: "j",
// defaultValue: undefined,
// },
// ],
// },
// ];
// const resultParsedObject = parseJSObjectWithAST(body);
// expect(resultParsedObject).toEqual(parsedObject);
// });
// });
const parsedObject = [
{
key: 'myFun2',
value:
'async (a, b = Array(1, 2, 3), c = "", d = [], e = this.myVar1, f = {}, g = function () {}, h = Object.assign({}), i = String(), j = storeValue()) => {}',
type: 'ArrowFunctionExpression',
arguments: [
{
paramName: 'a',
defaultValue: undefined,
},
{
paramName: 'b',
defaultValue: undefined,
},
{
paramName: 'c',
defaultValue: undefined,
},
{
paramName: 'd',
defaultValue: undefined,
},
{
paramName: 'e',
defaultValue: undefined,
},
{
paramName: 'f',
defaultValue: undefined,
},
{
paramName: 'g',
defaultValue: undefined,
},
{
paramName: 'h',
defaultValue: undefined,
},
{
paramName: 'i',
defaultValue: undefined,
},
{
paramName: 'j',
defaultValue: undefined,
},
],
},
];
const resultParsedObject = parseJSObjectWithAST(body);
expect(resultParsedObject).toEqual(parsedObject);
});
});

249
app/shared/ast/src/index.ts
View File

@ -1,8 +1,8 @@
import { parse, Node } from "acorn";
import { ancestor } from "acorn-walk";
import { ECMA_VERSION, NodeTypes } from "./constants";
import { isFinite, isString } from "lodash";
import { sanitizeScript } from "./utils";
import { parse, Node, SourceLocation, Options } from 'acorn';
import { ancestor, simple } from 'acorn-walk';
import { ECMA_VERSION, NodeTypes } from './constants/ast';
import { has, isFinite, isString, memoize, toPath } from 'lodash';
import { isTrueObject, sanitizeScript } from './utils';
/*
* Valuable links:
@ -90,9 +90,16 @@ export interface PropertyNode extends Node {
type: NodeTypes.Property;
key: LiteralNode | IdentifierNode;
value: Node;
kind: "init" | "get" | "set";
kind: 'init' | 'get' | 'set';
}
// Node with location details
type NodeWithLocation<NodeType> = NodeType & {
loc: SourceLocation;
};
type AstOptions = Omit<Options, 'ecmaVersion'>;
/* We need these functions to typescript casts the nodes with the correct types */
export const isIdentifierNode = (node: Node): node is IdentifierNode => {
return node.type === NodeTypes.Identifier;
@ -115,6 +122,11 @@ const isFunctionDeclaration = (node: Node): node is FunctionDeclarationNode => {
const isFunctionExpression = (node: Node): node is FunctionExpressionNode => {
return node.type === NodeTypes.FunctionExpression;
};
const isArrowFunctionExpression = (
node: Node
): node is ArrowFunctionExpressionNode => {
return node.type === NodeTypes.ArrowFunctionExpression;
};
export const isObjectExpression = (node: Node): node is ObjectExpression => {
return node.type === NodeTypes.ObjectExpression;
@ -158,27 +170,49 @@ const wrapCode = (code: string) => {
`;
};
export const getAST = (code: string) =>
parse(code, { ecmaVersion: ECMA_VERSION });
const getFunctionalParamNamesFromNode = (
node:
| FunctionDeclarationNode
| FunctionExpressionNode
| ArrowFunctionExpressionNode
) => {
return Array.from(getFunctionalParamsFromNode(node)).map(
(functionalParam) => functionalParam.paramName
);
};
// Memoize the ast generation code to improve performance.
// Since this will be used by both the server and the client, we want to prevent regeneration of ast
// for the the same code snippet
export const getAST = memoize((code: string, options?: AstOptions) =>
parse(code, { ...options, ecmaVersion: ECMA_VERSION })
);
/**
* An AST based extractor that fetches all possible identifiers in a given
* An AST based extractor that fetches all possible references in a given
* piece of code. We use this to get any references to the global entities in Appsmith
* and create dependencies on them. If the reference was updated, the given piece of code
* should run again.
* @param code: The piece of script where identifiers need to be extracted from
* @param code: The piece of script where references need to be extracted from
*/
export const extractIdentifiersFromCode = (
interface ExtractInfoFromCode {
references: string[];
functionalParams: string[];
variables: string[];
}
export const extractInfoFromCode = (
code: string,
evaluationVersion: number
): string[] => {
// List of all identifiers found
const identifiers = new Set<string>();
// List of variables declared within the script. This will be removed from identifier list
evaluationVersion: number,
invalidIdentifiers?: Record<string, unknown>
): ExtractInfoFromCode => {
// List of all references found
const references = new Set<string>();
// List of variables declared within the script. All identifiers and member expressions derived from declared variables will be removed
const variableDeclarations = new Set<string>();
// List of functionalParams found. This will be removed from the identifier list
let functionalParams = new Set<functionParams>();
let ast: Node = { end: 0, start: 0, type: "" };
// List of functional params declared within the script. All identifiers and member expressions derived from functional params will be removed
let functionalParams = new Set<string>();
let ast: Node = { end: 0, start: 0, type: '' };
try {
const sanitizedScript = sanitizeScript(code, evaluationVersion);
/* wrapCode - Wrapping code in a function, since all code/script get wrapped with a function during evaluation.
@ -194,15 +228,19 @@ export const extractIdentifiersFromCode = (
ast = getAST(wrappedCode);
} catch (e) {
if (e instanceof SyntaxError) {
// Syntax error. Ignore and return 0 identifiers
return [];
// Syntax error. Ignore and return empty list
return {
references: [],
functionalParams: [],
variables: [],
};
}
throw e;
}
/*
* We do an ancestor walk on the AST to get all identifiers. Since we need to know
* what surrounds the identifier, ancestor walk will give that information in the callback
* We do an ancestor walk on the AST in order to extract all references. For example, for member expressions and identifiers, we need to know
* what surrounds the identifier (its parent and ancestors), ancestor walk will give that information in the callback
* doc: https://github.com/acornjs/acorn/tree/master/acorn-walk
*/
ancestor(ast, {
@ -240,51 +278,70 @@ export const extractIdentifiersFromCode = (
}
if (isIdentifierNode(candidateTopLevelNode)) {
// If the node is an Identifier, just save that
identifiers.add(candidateTopLevelNode.name);
references.add(candidateTopLevelNode.name);
} else {
// For MemberExpression Nodes, we will construct a final reference string and then add
// it to the identifier list
// it to the references list
const memberExpIdentifier = constructFinalMemberExpIdentifier(
candidateTopLevelNode
);
identifiers.add(memberExpIdentifier);
references.add(memberExpIdentifier);
}
},
VariableDeclarator(node: Node) {
// keep a track of declared variables so they can be
// subtracted from the final list of identifiers
// removed from the final list of references
if (isVariableDeclarator(node)) {
variableDeclarations.add(node.id.name);
}
},
FunctionDeclaration(node: Node) {
// params in function declarations are also counted as identifiers so we keep
// track of them and remove them from the final list of identifiers
// params in function declarations are also counted as references so we keep
// track of them and remove them from the final list of references
if (!isFunctionDeclaration(node)) return;
functionalParams = new Set([
...functionalParams,
...getFunctionalParamsFromNode(node),
...getFunctionalParamNamesFromNode(node),
]);
},
FunctionExpression(node: Node) {
// params in function experssions are also counted as identifiers so we keep
// track of them and remove them from the final list of identifiers
// params in function expressions are also counted as references so we keep
// track of them and remove them from the final list of references
if (!isFunctionExpression(node)) return;
functionalParams = new Set([
...functionalParams,
...getFunctionalParamsFromNode(node),
...getFunctionalParamNamesFromNode(node),
]);
},
ArrowFunctionExpression(node: Node) {
// params in arrow function expressions are also counted as references so we keep
// track of them and remove them from the final list of references
if (!isArrowFunctionExpression(node)) return;
functionalParams = new Set([
...functionalParams,
...getFunctionalParamNamesFromNode(node),
]);
},
});
// Remove declared variables and function params
variableDeclarations.forEach((variable) => identifiers.delete(variable));
functionalParams.forEach((param) => identifiers.delete(param.paramName));
return Array.from(identifiers);
const referencesArr = Array.from(references).filter((reference) => {
// To remove references derived from declared variables and function params,
// We extract the topLevelIdentifier Eg. Api1.name => Api1
const topLevelIdentifier = toPath(reference)[0];
return !(
functionalParams.has(topLevelIdentifier) ||
variableDeclarations.has(topLevelIdentifier) ||
has(invalidIdentifiers, topLevelIdentifier)
);
});
return {
references: referencesArr,
functionalParams: Array.from(functionalParams),
variables: Array.from(variableDeclarations),
};
};
export type functionParams = { paramName: string; defaultValue: unknown };
export type functionParam = { paramName: string; defaultValue: unknown };
export const getFunctionalParamsFromNode = (
node:
@ -292,8 +349,8 @@ export const getFunctionalParamsFromNode = (
| FunctionExpressionNode
| ArrowFunctionExpressionNode,
needValue = false
): Set<functionParams> => {
const functionalParams = new Set<functionParams>();
): Set<functionParam> => {
const functionalParams = new Set<functionParam>();
node.params.forEach((paramNode) => {
if (isIdentifierNode(paramNode)) {
functionalParams.add({
@ -320,7 +377,7 @@ export const getFunctionalParamsFromNode = (
const constructFinalMemberExpIdentifier = (
node: MemberExpressionNode,
child = ""
child = ''
): string => {
const propertyAccessor = getPropertyAccessor(node.property);
if (isIdentifierNode(node.object)) {
@ -350,3 +407,111 @@ export const isTypeOfFunction = (type: string) => {
type === NodeTypes.FunctionExpression
);
};
export interface MemberExpressionData {
property: NodeWithLocation<IdentifierNode | LiteralNode>;
object: NodeWithLocation<IdentifierNode>;
}
/** Function returns Invalid top-level member expressions from code
* @param code
* @param data
* @param evaluationVersion
* @returns information about all invalid property/method assessment in code
* @example Given data {
* JSObject1: {
* name:"JSObject",
* data:[]
* },
* Api1:{
* name: "Api1",
* data: []
* }
* },
* For code {{Api1.name + JSObject.unknownProperty}}, function returns information about "JSObject.unknownProperty" node.
*/
export const extractInvalidTopLevelMemberExpressionsFromCode = (
code: string,
data: Record<string, any>,
evaluationVersion: number
): MemberExpressionData[] => {
const invalidTopLevelMemberExpressions = new Set<MemberExpressionData>();
const variableDeclarations = new Set<string>();
let functionalParams = new Set<string>();
let ast: Node = { end: 0, start: 0, type: '' };
try {
const sanitizedScript = sanitizeScript(code, evaluationVersion);
const wrappedCode = wrapCode(sanitizedScript);
ast = getAST(wrappedCode, { locations: true });
} catch (e) {
if (e instanceof SyntaxError) {
// Syntax error. Ignore and return empty list
return [];
}
throw e;
}
simple(ast, {
MemberExpression(node: Node) {
const { object, property } = node as MemberExpressionNode;
// We are only interested in top-level MemberExpression nodes
// Eg. for Api1.data.name, we are only interested in Api1.data
if (!isIdentifierNode(object)) return;
if (!(object.name in data) || !isTrueObject(data[object.name])) return;
// For computed member expressions (assessed via [], eg. JSObject1["name"] ),
// We are only interested in strings
if (
isLiteralNode(property) &&
isString(property.value) &&
!(property.value in data[object.name])
) {
invalidTopLevelMemberExpressions.add({
object,
property,
} as MemberExpressionData);
}
if (isIdentifierNode(property) && !(property.name in data[object.name])) {
invalidTopLevelMemberExpressions.add({
object,
property,
} as MemberExpressionData);
}
},
VariableDeclarator(node: Node) {
if (isVariableDeclarator(node)) {
variableDeclarations.add(node.id.name);
}
},
FunctionDeclaration(node: Node) {
if (!isFunctionDeclaration(node)) return;
functionalParams = new Set([
...functionalParams,
...getFunctionalParamNamesFromNode(node),
]);
},
FunctionExpression(node: Node) {
if (!isFunctionExpression(node)) return;
functionalParams = new Set([
...functionalParams,
...getFunctionalParamNamesFromNode(node),
]);
},
ArrowFunctionExpression(node: Node) {
if (!isArrowFunctionExpression(node)) return;
functionalParams = new Set([
...functionalParams,
...getFunctionalParamNamesFromNode(node),
]);
},
});
const invalidTopLevelMemberExpressionsArray = Array.from(
invalidTopLevelMemberExpressions
).filter((MemberExpression) => {
return !(
variableDeclarations.has(MemberExpression.object.name) ||
functionalParams.has(MemberExpression.object.name)
);
});
return invalidTopLevelMemberExpressionsArray;
};

18
app/shared/ast/src/jsObject/index.ts
View File

@ -1,21 +1,21 @@
import { Node } from "acorn";
import { getAST } from "../index";
import { generate } from "astring";
import { simple } from "acorn-walk";
import { Node } from 'acorn';
import { getAST } from '../index';
import { generate } from 'astring';
import { simple } from 'acorn-walk';
import {
getFunctionalParamsFromNode,
isPropertyAFunctionNode,
isVariableDeclarator,
isObjectExpression,
PropertyNode,
functionParams,
} from "../index";
functionParam,
} from '../index';
type JsObjectProperty = {
key: string;
value: string;
type: string;
arguments?: Array<functionParams>;
arguments?: Array<functionParam>;
};
export const parseJSObjectWithAST = (
@ -27,7 +27,7 @@ export const parseJSObjectWithAST = (
Keeping this just for sanity check if any caveat was missed.
*/
const jsObjectVariableName =
"____INTERNAL_JS_OBJECT_NAME_USED_FOR_PARSING_____";
'____INTERNAL_JS_OBJECT_NAME_USED_FOR_PARSING_____';
const jsCode = `var ${jsObjectVariableName} = ${jsObjectBody}`;
const ast = getAST(jsCode);
@ -49,7 +49,7 @@ export const parseJSObjectWithAST = (
});
JSObjectProperties.forEach((node) => {
let params = new Set<functionParams>();
let params = new Set<functionParam>();
const propertyNode = node;
let property: JsObjectProperty = {
key: generate(propertyNode.key),

12
app/shared/ast/src/utils.ts
View File

@ -1,4 +1,4 @@
import unescapeJS from "unescape-js";
import unescapeJS from 'unescape-js';
const beginsWithLineBreakRegex = /^\s+|\s+$/;
@ -6,6 +6,14 @@ export function sanitizeScript(js: string, evaluationVersion: number) {
// We remove any line breaks from the beginning of the script because that
// makes the final function invalid. We also unescape any escaped characters
// so that eval can happen
const trimmedJS = js.replace(beginsWithLineBreakRegex, "");
const trimmedJS = js.replace(beginsWithLineBreakRegex, '');
return evaluationVersion > 1 ? trimmedJS : unescapeJS(trimmedJS);
}
// For the times when you need to know if something truly an object like { a: 1, b: 2}
// typeof, lodash.isObject and others will return false positives for things like array, null, etc
export const isTrueObject = (
item: unknown
): item is Record<string, unknown> => {
return Object.prototype.toString.call(item) === '[object Object]';
};

4
yarn.lock Normal file
View File

@ -0,0 +1,4 @@
# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
# yarn lockfile v1