diff --git a/app/client/cypress/e2e/Regression/ClientSide/Autocomplete/Bugs_AC_Spec.ts b/app/client/cypress/e2e/Regression/ClientSide/Autocomplete/Bugs_AC_Spec.ts index 3fc3d6ef51..caee6aa86d 100644 --- a/app/client/cypress/e2e/Regression/ClientSide/Autocomplete/Bugs_AC_Spec.ts +++ b/app/client/cypress/e2e/Regression/ClientSide/Autocomplete/Bugs_AC_Spec.ts @@ -86,15 +86,6 @@ describe("Autocomplete bug fixes", function () { ); }); - it("6. feat #16426 Autocomplete for fast-xml-parser", function () { - entityExplorer.SelectEntityByName("Text1"); - propPane.TypeTextIntoField("Text", "{{xmlParser.j"); - agHelper.GetNAssertElementText(locators._hints, "j2xParser"); - - propPane.TypeTextIntoField("Text", "{{new xmlParser.j2xParser().p"); - agHelper.GetNAssertElementText(locators._hints, "parse"); - }); - it( "excludeForAirgap", "7. Installed library should show up in autocomplete", diff --git a/app/client/cypress/e2e/Regression/ClientSide/Binding/xmlParser_spec.js b/app/client/cypress/e2e/Regression/ClientSide/Binding/xmlParser_spec.js index 5cd7b98d3f..6adf665579 100644 --- a/app/client/cypress/e2e/Regression/ClientSide/Binding/xmlParser_spec.js +++ b/app/client/cypress/e2e/Regression/ClientSide/Binding/xmlParser_spec.js @@ -3,9 +3,21 @@ import * as _ from "../../../../support/Objects/ObjectsCore"; describe("xml2json text", function () { before(() => { - _.agHelper.AddDsl("xmlParser"); + _.homePage.NavigateToHome(); + _.homePage.ImportApp("xmlParser.json"); + _.homePage.AssertImportToast(); }); - it("1. Publish widget and validate the data displayed in text widget from xmlParser function", function () { + + it("1. Check if XMLparser v3 autocomplete works", function () { + _.entityExplorer.SelectEntityByName("Text2", "Widgets"); + _.propPane.TypeTextIntoField("Text", "{{xmlParser.j", true); + _.agHelper.GetNAssertElementText(_.locators._hints, "j2xParser"); + + _.propPane.TypeTextIntoField("Text", "{{new xmlParser.j2xParser().p", true); + _.agHelper.GetNAssertElementText(_.locators._hints, "parse"); + }); + + it("2. Publish widget and validate the data displayed in text widget from xmlParser function", function () { _.deployMode.DeployApp(); cy.get(publish.textWidget) .first() diff --git a/app/client/cypress/e2e/Regression/ClientSide/Git/ExistingApps/v1.9.24/DSCrudAndBindings_Spec.ts b/app/client/cypress/e2e/Regression/ClientSide/Git/ExistingApps/v1.9.24/DSCrudAndBindings_Spec.ts index 0504dea35d..a24ac844b6 100644 --- a/app/client/cypress/e2e/Regression/ClientSide/Git/ExistingApps/v1.9.24/DSCrudAndBindings_Spec.ts +++ b/app/client/cypress/e2e/Regression/ClientSide/Git/ExistingApps/v1.9.24/DSCrudAndBindings_Spec.ts @@ -53,11 +53,13 @@ describe("Import and validate older app (app created in older versions of Appsmi gitSync._gitStatusChanges, /[0-9] page(|s) modified/, ); - agHelper.GetNAssertElementText( - gitSync._gitStatusChanges, - "Application settings modified", - "not.contain.text", - ); + + // Commenting it as part of #28012 - to be added back later + // agHelper.GetNAssertElementText( + // gitSync._gitStatusChanges, + // "Application settings modified", + // "not.contain.text", + // ); agHelper.GetNAssertElementText( gitSync._gitStatusChanges, "Theme modified", @@ -73,7 +75,10 @@ describe("Import and validate older app (app created in older versions of Appsmi // ); agHelper.AssertContains(/[0-9] JS Object(|s) modified/, "not.exist"); - agHelper.AssertContains(/[0-9] librar(y|ies) modified/, "not.exist"); + + // Commenting it as part of #28012 - to be added back later + // agHelper.AssertContains(/[0-9] librar(y|ies) modified/, "not.exist"); + agHelper.GetNAssertElementText( gitSync._gitStatusChanges, "Some of the changes above are due to an improved file structure designed to reduce merge conflicts. You can safely commit them to your repository.", diff --git a/app/client/cypress/e2e/Regression/ClientSide/JSLibrary/Library_spec.ts b/app/client/cypress/e2e/Regression/ClientSide/JSLibrary/Library_spec.ts index 4c77d0fabf..e7282223c1 100644 --- a/app/client/cypress/e2e/Regression/ClientSide/JSLibrary/Library_spec.ts +++ b/app/client/cypress/e2e/Regression/ClientSide/JSLibrary/Library_spec.ts @@ -10,14 +10,13 @@ describe("excludeForAirgap", "Tests JS Libraries", () => { _.installer.assertUnInstall("uuidjs"); }); - it("2. Checks for naming collision", () => { + it("2. Installs the library against a unique namespace when there is a collision with the existing entity", () => { _.entityExplorer.DragDropWidgetNVerify(_.draggableWidgets.TABLE, 200, 200); _.entityExplorer.NavigateToSwitcher("Explorer"); _.entityExplorer.RenameEntityFromExplorer("Table1", "jsonwebtoken"); _.entityExplorer.ExpandCollapseEntity("Libraries"); _.installer.OpenInstaller(); - _.installer.installLibrary("jsonwebtoken", "jsonwebtoken", false); - _.agHelper.AssertContains("Name collision detected: jsonwebtoken"); + _.installer.installLibrary("jsonwebtoken", "jsonwebtoken_1", true); }); it("3. Checks jspdf library", () => { diff --git a/app/client/cypress/fixtures/xmlParser.json b/app/client/cypress/fixtures/xmlParser.json index ed63586a5c..65573652b8 100644 --- a/app/client/cypress/fixtures/xmlParser.json +++ b/app/client/cypress/fixtures/xmlParser.json @@ -1,64 +1,342 @@ { - "dsl": { - "widgetName": "MainContainer", - "backgroundColor": "none", - "rightColumn": 1224, - "snapColumns": 16, - "detachFromLayout": true, - "widgetId": "0", - "topRow": 0, - "bottomRow": 1280, - "containerStyle": "none", - "snapRows": 33, - "parentRowSpace": 1, - "type": "CANVAS_WIDGET", - "canExtend": true, - "version": 7, - "minHeight": 1292, - "parentColumnSpace": 1, - "leftColumn": 0, - "dynamicBindingPathList": [], - "children": [ + "clientSchemaVersion": 1.0, + "serverSchemaVersion": 6.0, + "exportedApplication": { + "name": "xml_paser_test", + "isPublic": false, + "pages": [ { - "isVisible": true, - "text": "{{xmlParser.parse(Input1.text) }}", - "textStyle": "LABEL", - "textAlign": "LEFT", - "widgetName": "Text1", - "type": "TEXT_WIDGET", - "isLoading": false, - "parentColumnSpace": 74, - "parentRowSpace": 40, - "leftColumn": 3, - "rightColumn": 8, - "topRow": 3, - "bottomRow": 10, - "parentId": "0", - "widgetId": "axlcnmjk4t", - "dynamicBindingPathList": [ - { - "key": "text" - } - ] - }, - { - "isVisible": true, - "inputType": "TEXT", - "label": "", - "widgetName": "Input1", - "type": "INPUT_WIDGET_V2", - "isLoading": false, - "parentColumnSpace": 74, - "parentRowSpace": 40, - "leftColumn": 3, - "rightColumn": 8, - "topRow": 2, - "bottomRow": 3, - "parentId": "0", - "widgetId": "8n5urob9mz", - "dynamicBindingPathList": [], - "defaultText": " Tove Jani Reminder Don't forget me this weekend! " + "id": "Page1", + "isDefault": true } - ] + ], + "publishedPages": [ + { + "id": "Page1", + "isDefault": true + } + ], + "viewMode": false, + "appIsExample": false, + "unreadCommentThreads": 0.0, + "color": "#EAEDFB", + "icon": "yen", + "slug": "xml-paser-test", + "unpublishedCustomJSLibs": [], + "publishedCustomJSLibs": [], + "evaluationVersion": 2.0, + "applicationVersion": 2.0, + "collapseInvisibleWidgets": true, + "isManualUpdate": false, + "deleted": false + }, + "datasourceList": [], + "customJSLibList": [], + "pageList": [ + { + "unpublishedPage": { + "name": "Page1", + "slug": "page1", + "layouts": [ + { + "viewMode": false, + "dsl": { + "widgetName": "MainContainer", + "backgroundColor": "none", + "rightColumn": 4896.0, + "snapColumns": 64.0, + "detachFromLayout": true, + "widgetId": "0", + "topRow": 0.0, + "bottomRow": 380.0, + "containerStyle": "none", + "snapRows": 124.0, + "parentRowSpace": 1.0, + "type": "CANVAS_WIDGET", + "canExtend": true, + "version": 87.0, + "minHeight": 1292.0, + "dynamicTriggerPathList": [], + "parentColumnSpace": 1.0, + "dynamicBindingPathList": [], + "leftColumn": 0.0, + "children": [ + { + "mobileBottomRow": 14.0, + "widgetName": "Text1", + "displayName": "Text", + "iconSVG": "https://release-appcdn.appsmith.com/static/media/icon.a47d6d5dbbb718c4dc4b2eb4f218c1b7.svg", + "searchTags": [ + "typography", + "paragraph", + "label" + ], + "topRow": 0.0, + "bottomRow": 16.0, + "parentRowSpace": 10.0, + "type": "TEXT_WIDGET", + "hideCard": false, + "mobileRightColumn": 23.0, + "animateLoading": true, + "overflow": "NONE", + "fontFamily": "{{appsmith.theme.fontFamily.appFont}}", + "parentColumnSpace": 14.953125, + "dynamicTriggerPathList": [], + "leftColumn": 0.0, + "dynamicBindingPathList": [ + { + "key": "truncateButtonColor" + }, + { + "key": "fontFamily" + }, + { + "key": "borderRadius" + }, + { + "key": "text" + } + ], + "shouldTruncate": false, + "truncateButtonColor": "{{appsmith.theme.colors.primaryColor}}", + "text": "{{xmlParser.parse(Input1.text)}}", + "key": "bdg0tfrf6x", + "isDeprecated": false, + "rightColumn": 13.0, + "textAlign": "LEFT", + "dynamicHeight": "AUTO_HEIGHT", + "widgetId": "b8blvt9b2z", + "minWidth": 450.0, + "isVisible": true, + "fontStyle": "BOLD", + "textColor": "#231F20", + "version": 1.0, + "parentId": "0", + "tags": [ + "Suggested", + "Content" + ], + "renderMode": "CANVAS", + "isLoading": false, + "mobileTopRow": 10.0, + "responsiveBehavior": "fill", + "originalTopRow": 0.0, + "borderRadius": "{{appsmith.theme.borderRadius.appBorderRadius}}", + "mobileLeftColumn": 7.0, + "maxDynamicHeight": 9000.0, + "originalBottomRow": 4.0, + "fontSize": "1rem", + "minDynamicHeight": 4.0 + }, + { + "boxShadow": "none", + "iconSVG": "https://release-appcdn.appsmith.com/static/media/icon.f2c34197dbcf03595098986de898928f.svg", + "topRow": 17.0, + "labelWidth": 5.0, + "type": "INPUT_WIDGET_V2", + "animateLoading": true, + "resetOnSubmit": true, + "leftColumn": 0.0, + "dynamicBindingPathList": [ + { + "key": "accentColor" + }, + { + "key": "borderRadius" + } + ], + "labelStyle": "", + "inputType": "TEXT", + "isDisabled": false, + "isRequired": false, + "dynamicHeight": "FIXED", + "accentColor": "{{appsmith.theme.colors.primaryColor}}", + "showStepArrows": false, + "isVisible": true, + "version": 2.0, + "tags": [ + "Suggested", + "Inputs" + ], + "isLoading": false, + "borderRadius": "{{appsmith.theme.borderRadius.appBorderRadius}}", + "originalBottomRow": 21.0, + "mobileBottomRow": 21.0, + "widgetName": "Input1", + "displayName": "Input", + "searchTags": [ + "form", + "text input", + "number", + "textarea" + ], + "bottomRow": 24.0, + "parentRowSpace": 10.0, + "autoFocus": false, + "hideCard": false, + "mobileRightColumn": 19.0, + "parentColumnSpace": 14.34375, + "dynamicTriggerPathList": [], + "labelPosition": "Top", + "key": "mx0emg6xlv", + "labelTextSize": "0.875rem", + "isDeprecated": false, + "rightColumn": 19.0, + "widgetId": "ni16fc0xys", + "minWidth": 450.0, + "label": "Label", + "parentId": "0", + "labelAlignment": "left", + "renderMode": "CANVAS", + "mobileTopRow": 14.0, + "responsiveBehavior": "fill", + "originalTopRow": 14.0, + "mobileLeftColumn": 0.0, + "maxDynamicHeight": 9000.0, + "iconAlign": "left", + "defaultText": " Tove Jani Reminder Don't forget me this weekend! ", + "minDynamicHeight": 4.0 + }, + { + "mobileBottomRow": 35.0, + "widgetName": "Text2", + "displayName": "Text", + "iconSVG": "https://release-appcdn.appsmith.com/static/media/icon.a47d6d5dbbb718c4dc4b2eb4f218c1b7.svg", + "searchTags": [ + "typography", + "paragraph", + "label" + ], + "topRow": 31.0, + "bottomRow": 35.0, + "parentRowSpace": 10.0, + "type": "TEXT_WIDGET", + "hideCard": false, + "mobileRightColumn": 16.0, + "animateLoading": true, + "overflow": "NONE", + "fontFamily": "{{appsmith.theme.fontFamily.appFont}}", + "parentColumnSpace": 14.34375, + "leftColumn": 0.0, + "dynamicBindingPathList": [ + { + "key": "truncateButtonColor" + }, + { + "key": "fontFamily" + }, + { + "key": "borderRadius" + }, + { + "key": "text" + } + ], + "shouldTruncate": false, + "truncateButtonColor": "{{appsmith.theme.colors.primaryColor}}", + "text": "Hello {{appsmith.user.name || appsmith.user.email}}", + "key": "bdg0tfrf6x", + "isDeprecated": false, + "rightColumn": 20.0, + "textAlign": "LEFT", + "dynamicHeight": "AUTO_HEIGHT", + "widgetId": "3m7c3st35v", + "minWidth": 450.0, + "isVisible": true, + "fontStyle": "BOLD", + "textColor": "#231F20", + "version": 1.0, + "parentId": "0", + "tags": [ + "Suggested", + "Content" + ], + "renderMode": "CANVAS", + "isLoading": false, + "mobileTopRow": 31.0, + "responsiveBehavior": "fill", + "borderRadius": "{{appsmith.theme.borderRadius.appBorderRadius}}", + "mobileLeftColumn": 0.0, + "maxDynamicHeight": 9000.0, + "fontSize": "1rem", + "minDynamicHeight": 4.0 + } + ] + }, + "layoutOnLoadActions": [], + "layoutOnLoadActionErrors": [], + "validOnPageLoadActions": true, + "id": "Page1", + "deleted": false, + "policies": [], + "userPermissions": [] + } + ], + "userPermissions": [], + "policies": [] + }, + "publishedPage": { + "name": "Page1", + "slug": "page1", + "layouts": [ + { + "viewMode": false, + "dsl": { + "widgetName": "MainContainer", + "backgroundColor": "none", + "rightColumn": 1224.0, + "snapColumns": 16.0, + "detachFromLayout": true, + "widgetId": "0", + "topRow": 0.0, + "bottomRow": 1250.0, + "containerStyle": "none", + "snapRows": 33.0, + "parentRowSpace": 1.0, + "type": "CANVAS_WIDGET", + "canExtend": true, + "version": 4.0, + "minHeight": 1292.0, + "dynamicTriggerPathList": [], + "parentColumnSpace": 1.0, + "dynamicBindingPathList": [], + "leftColumn": 0.0, + "children": [] + }, + "validOnPageLoadActions": true, + "id": "Page1", + "deleted": false, + "policies": [], + "userPermissions": [] + } + ], + "userPermissions": [], + "policies": [] + }, + "deleted": false, + "gitSyncId": "6529174c97a7581320fb6a4f_6529174c97a7581320fb6a51" + } + ], + "actionList": [], + "actionCollectionList": [], + "updatedResources": { + "customJSLibList": [], + "actionList": [], + "pageList": [ + "Page1" + ], + "actionCollectionList": [] + }, + "editModeTheme": { + "name": "Default-New", + "displayName": "Modern", + "isSystemTheme": true, + "deleted": false + }, + "publishedTheme": { + "name": "Default-New", + "displayName": "Modern", + "isSystemTheme": true, + "deleted": false } -} +} \ No newline at end of file diff --git a/app/client/package.json b/app/client/package.json index 2079505bdc..de934e1b83 100644 --- a/app/client/package.json +++ b/app/client/package.json @@ -104,7 +104,6 @@ "echarts-gl": "^2.0.9", "fast-deep-equal": "^3.1.3", "fast-sort": "^3.4.0", - "fast-xml-parser": "^3.17.5", "fastdom": "^1.0.11", "focus-trap-react": "^8.9.2", "fuse.js": "^3.4.5", diff --git a/app/client/src/actions/JSLibraryActions.ts b/app/client/src/actions/JSLibraryActions.ts index 9076263d3f..6ab4709057 100644 --- a/app/client/src/actions/JSLibraryActions.ts +++ b/app/client/src/actions/JSLibraryActions.ts @@ -1,5 +1,5 @@ import { ReduxActionTypes } from "@appsmith/constants/ReduxActionConstants"; -import type { TJSLibrary } from "workers/common/JSLibrary"; +import type { JSLibrary } from "workers/common/JSLibrary"; export function fetchJSLibraries(applicationId: string) { return { @@ -8,7 +8,7 @@ export function fetchJSLibraries(applicationId: string) { }; } -export function installLibraryInit(payload: Partial) { +export function installLibraryInit(payload: Partial) { return { type: ReduxActionTypes.INSTALL_LIBRARY_INIT, payload, @@ -22,7 +22,7 @@ export function toggleInstaller(payload: boolean) { }; } -export function uninstallLibraryInit(payload: TJSLibrary) { +export function uninstallLibraryInit(payload: JSLibrary) { return { type: ReduxActionTypes.UNINSTALL_LIBRARY_INIT, payload, diff --git a/app/client/src/api/LibraryAPI.tsx b/app/client/src/api/LibraryAPI.tsx index 123d4801e1..73b5c0d879 100644 --- a/app/client/src/api/LibraryAPI.tsx +++ b/app/client/src/api/LibraryAPI.tsx @@ -1,5 +1,5 @@ import { APP_MODE } from "entities/App"; -import type { TJSLibrary } from "workers/common/JSLibrary"; +import type { JSLibrary } from "workers/common/JSLibrary"; import Api from "./Api"; export default class LibraryApi extends Api { @@ -10,7 +10,7 @@ export default class LibraryApi extends Api { static async addLibrary( applicationId: string, - library: Partial & { defs: string }, + library: Partial & { defs: string }, ) { const url = LibraryApi.getUpdateLibraryBaseURL(applicationId) + "/add"; return Api.patch(url, library); @@ -18,7 +18,7 @@ export default class LibraryApi extends Api { static async removeLibrary( applicationId: string, - library: Partial, + library: Partial, ) { const url = LibraryApi.getUpdateLibraryBaseURL(applicationId) + "/remove"; return Api.patch(url, library); diff --git a/app/client/src/ce/selectors/entitiesSelector.ts b/app/client/src/ce/selectors/entitiesSelector.ts index 35aa61a050..484070825b 100644 --- a/app/client/src/ce/selectors/entitiesSelector.ts +++ b/app/client/src/ce/selectors/entitiesSelector.ts @@ -39,7 +39,7 @@ import { import { InstallState } from "reducers/uiReducers/libraryReducer"; import recommendedLibraries from "pages/Editor/Explorer/Libraries/recommendedLibraries"; -import type { TJSLibrary } from "workers/common/JSLibrary"; +import type { JSLibrary } from "workers/common/JSLibrary"; import { getEntityNameAndPropertyPath } from "@appsmith/workers/Evaluation/evaluationUtils"; import { getFormValues } from "redux-form"; import { TEMP_DATASOURCE_ID } from "constants/Datasource"; @@ -1099,7 +1099,7 @@ export const selectLibrariesForExplorer = createSelector( version: recommendedLibrary?.version || "", url: recommendedLibrary?.url || url, accessor: [], - } as TJSLibrary; + } as JSLibrary; }); return [...queuedInstalls, ...libs]; }, diff --git a/app/client/src/constants/defs/xmlParser.json b/app/client/src/constants/defs/xmlParser.json deleted file mode 100644 index 83f55cbe13..0000000000 --- a/app/client/src/constants/defs/xmlParser.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "!name": "LIB/xmlParser", - "xmlParser": { - "parse": { - "!doc": "converts xml string to json object", - "!type": "fn(xml: string, options?: object, validationOption?: object) -> object" - }, - "validate": { - "!doc": "validate xml data", - "!type": "fn(xml: string) -> bool" - }, - "convertToJson": { - "!type": "fn(node: ?, options: object) -> ?" - }, - "convertToJsonString": { - "!type": "fn(node: ?, options: object) -> string" - }, - "convertTonimn": { - "!type": "fn(node: ?, e_schema: ?, options: object) -> ?" - }, - "getTraversalObj": { - "!type": "fn(xmlData: ?, options: object) -> ?" - }, - "j2xParser": { - "!type": "fn(options: object) -> object", - "prototype": { - "parse": { - "!type": "fn(jObj: ?)" - }, - "j2x": { - "!type": "fn(jObj: ?, level: ?)" - } - } - }, - "parseToNimn": { - "!type": "fn(xmlData: ?, schema: ?, options: ?) -> ?" - } - } -} diff --git a/app/client/src/pages/Editor/Explorer/Libraries/Installer.tsx b/app/client/src/pages/Editor/Explorer/Libraries/Installer.tsx index 0451226a3f..1d04e4aa93 100644 --- a/app/client/src/pages/Editor/Explorer/Libraries/Installer.tsx +++ b/app/client/src/pages/Editor/Explorer/Libraries/Installer.tsx @@ -36,7 +36,7 @@ import recommendedLibraries from "pages/Editor/Explorer/Libraries/recommendedLib import type { AppState } from "@appsmith/reducers"; import { installLibraryInit } from "actions/JSLibraryActions"; import classNames from "classnames"; -import type { TJSLibrary } from "workers/common/JSLibrary"; +import type { JSLibrary } from "workers/common/JSLibrary"; import AnalyticsUtil from "utils/AnalyticsUtil"; import { EntityClassNames } from "pages/Editor/Explorer/Entity"; @@ -317,7 +317,7 @@ export function Installer() { }, [URL, isValid]); const installLibrary = useCallback( - (lib?: Partial) => { + (lib?: Partial) => { const url = lib?.url || URL; const isQueued = queuedLibraries.find((libURL) => libURL === url); if (isQueued) return; diff --git a/app/client/src/pages/Editor/Explorer/Libraries/index.tsx b/app/client/src/pages/Editor/Explorer/Libraries/index.tsx index 7e502331be..848dd8116a 100644 --- a/app/client/src/pages/Editor/Explorer/Libraries/index.tsx +++ b/app/client/src/pages/Editor/Explorer/Libraries/index.tsx @@ -33,7 +33,7 @@ import { uninstallLibraryInit, } from "actions/JSLibraryActions"; import EntityAddButton from "../Entity/AddButton"; -import type { TJSLibrary } from "workers/common/JSLibrary"; +import type { JSLibrary } from "workers/common/JSLibrary"; import { getCurrentPageId, getPagePermissions, @@ -177,7 +177,7 @@ const Version = styled.div<{ version?: string }>` margin: ${(props) => (props.version ? "0 8px" : "0")}; `; -const PrimaryCTA = function ({ lib }: { lib: TJSLibrary }) { +const PrimaryCTA = function ({ lib }: { lib: JSLibrary }) { const installationStatus = useSelector(selectInstallationStatus); const dispatch = useDispatch(); @@ -215,7 +215,7 @@ const PrimaryCTA = function ({ lib }: { lib: TJSLibrary }) { return null; }; -function LibraryEntity({ lib }: { lib: TJSLibrary }) { +function LibraryEntity({ lib }: { lib: JSLibrary }) { const openDocs = useCallback( (url?: string) => (e: React.MouseEvent) => { e?.stopPropagation(); diff --git a/app/client/src/pages/Editor/Explorer/Libraries/recommendedLibraries.ts b/app/client/src/pages/Editor/Explorer/Libraries/recommendedLibraries.ts index db64c2d001..2c7acb74db 100644 --- a/app/client/src/pages/Editor/Explorer/Libraries/recommendedLibraries.ts +++ b/app/client/src/pages/Editor/Explorer/Libraries/recommendedLibraries.ts @@ -5,9 +5,19 @@ export default [ author: "auth0", docsURL: "https://github.com/auth0/node-jsonwebtoken#readme", version: "8.5.1", - url: `/libraries/jsonwebtoken@8.5.1.js`, + url: "/libraries/jsonwebtoken@8.5.1.js", icon: "https://github.com/auth0.png?s=20", }, + { + name: "fast-xml-parser", + description: + "Validate XML, Parse XML to JS Object, or Build XML from JS Object without C/C++ based libraries and no callback.", + author: "NaturalIntelligence", + docsURL: "https://github.com/NaturalIntelligence/fast-xml-parser", + url: "https://cdnjs.cloudflare.com/ajax/libs/fast-xml-parser/4.3.2/fxparser.min.js", + version: "4.3.2", + icon: "https://img.jsdelivr.com/github.com/NaturalIntelligence.png", + }, { name: "jspdf", description: "PDF Document creation from JavaScript", diff --git a/app/client/src/plugins/Linting/types.ts b/app/client/src/plugins/Linting/types.ts index b3cfa00d0b..17cb175e4a 100644 --- a/app/client/src/plugins/Linting/types.ts +++ b/app/client/src/plugins/Linting/types.ts @@ -10,7 +10,7 @@ import type { } from "workers/Evaluation/evaluate"; import type { DependencyMap } from "utils/DynamicBindingUtils"; import type { TJSPropertiesState } from "workers/Evaluation/JSObject/jsPropertiesState"; -import type { TJSLibrary } from "workers/common/JSLibrary"; +import type { JSLibrary } from "workers/common/JSLibrary"; export enum LINT_WORKER_ACTIONS { LINT_TREE = "LINT_TREE", @@ -78,5 +78,5 @@ export interface getLintErrorsFromTreeResponse { export interface updateJSLibraryProps { add?: boolean; - libs: TJSLibrary[]; + libs: JSLibrary[]; } diff --git a/app/client/src/reducers/uiReducers/libraryReducer.ts b/app/client/src/reducers/uiReducers/libraryReducer.ts index a9ade8c1c6..5cda44347a 100644 --- a/app/client/src/reducers/uiReducers/libraryReducer.ts +++ b/app/client/src/reducers/uiReducers/libraryReducer.ts @@ -5,7 +5,7 @@ import { ReduxActionTypes, } from "@appsmith/constants/ReduxActionConstants"; import recommendedLibraries from "pages/Editor/Explorer/Libraries/recommendedLibraries"; -import type { TJSLibrary } from "workers/common/JSLibrary"; +import type { JSLibrary } from "workers/common/JSLibrary"; import { defaultLibraries } from "workers/common/JSLibrary"; export enum InstallState { @@ -17,14 +17,14 @@ export enum InstallState { export interface LibraryState { installationStatus: Record; - installedLibraries: TJSLibrary[]; + installedLibraries: JSLibrary[]; isInstallerOpen: boolean; } const initialState = { isInstallerOpen: false, installationStatus: {}, - installedLibraries: defaultLibraries.map((lib: TJSLibrary) => { + installedLibraries: defaultLibraries.map((lib: JSLibrary) => { return { name: lib.name, docsURL: lib.docsURL, @@ -38,7 +38,7 @@ const initialState = { const jsLibraryReducer = createImmerReducer(initialState, { [ReduxActionTypes.INSTALL_LIBRARY_INIT]: ( state: LibraryState, - action: ReduxAction>, + action: ReduxAction>, ) => { const { url } = action.payload; state.installationStatus[url as string] = @@ -91,7 +91,7 @@ const jsLibraryReducer = createImmerReducer(initialState, { }, [ReduxActionTypes.FETCH_JS_LIBRARIES_SUCCESS]: ( state: LibraryState, - action: ReduxAction, + action: ReduxAction, ) => { state.installedLibraries = action.payload.concat( initialState.installedLibraries, @@ -99,7 +99,7 @@ const jsLibraryReducer = createImmerReducer(initialState, { }, [ReduxActionTypes.UNINSTALL_LIBRARY_SUCCESS]: ( state: LibraryState, - action: ReduxAction, + action: ReduxAction, ) => { const uLib = action.payload; state.installedLibraries = state.installedLibraries.filter( diff --git a/app/client/src/sagas/JSLibrarySaga.ts b/app/client/src/sagas/JSLibrarySaga.ts index 17d57a7101..aa9237418c 100644 --- a/app/client/src/sagas/JSLibrarySaga.ts +++ b/app/client/src/sagas/JSLibrarySaga.ts @@ -29,7 +29,7 @@ import log from "loglevel"; import { APP_MODE } from "entities/App"; import { getAppMode } from "@appsmith/selectors/applicationSelectors"; import AnalyticsUtil from "utils/AnalyticsUtil"; -import type { TJSLibrary } from "workers/common/JSLibrary"; +import type { JSLibrary } from "workers/common/JSLibrary"; import { getUsedActionNames } from "selectors/actionSelectors"; import AppsmithConsole from "utils/AppsmithConsole"; import { selectInstalledLibraries } from "@appsmith/selectors/entitiesSelector"; @@ -73,7 +73,7 @@ function* handleInstallationFailure( log.error(message); } -export function* installLibrarySaga(lib: Partial) { +export function* installLibrarySaga(lib: Partial) { const { url } = lib; const takenNamesMap: Record = yield select( @@ -81,7 +81,7 @@ export function* installLibrarySaga(lib: Partial) { "", ); - const installedLibraries: TJSLibrary[] = yield select( + const installedLibraries: JSLibrary[] = yield select( selectInstalledLibraries, ); @@ -232,7 +232,7 @@ export function* installLibrarySaga(lib: Partial) { }); } -function* uninstallLibrarySaga(action: ReduxAction) { +function* uninstallLibrarySaga(action: ReduxAction) { const { accessor, name } = action.payload; const applicationId: string = yield select(getCurrentApplicationId); @@ -320,7 +320,7 @@ function* fetchJSLibraries(action: ReduxAction) { const isValidResponse: boolean = yield validateResponse(response); if (!isValidResponse) return; - const libraries = response.data as Array; + const libraries = response.data as Array; const { message, success }: { success: boolean; message: string } = yield call( @@ -404,7 +404,7 @@ function* startInstallationRequestChannel() { ReduxActionTypes.INSTALL_LIBRARY_INIT, ]); while (true) { - const action: ReduxAction> = + const action: ReduxAction> = yield take(queueInstallChannel); yield put({ type: ReduxActionTypes.INSTALL_LIBRARY_START, diff --git a/app/client/src/sagas/LintingSagas.ts b/app/client/src/sagas/LintingSagas.ts index ea0c96edbd..b2a44292ec 100644 --- a/app/client/src/sagas/LintingSagas.ts +++ b/app/client/src/sagas/LintingSagas.ts @@ -4,7 +4,7 @@ import { ReduxActionTypes } from "@appsmith/constants/ReduxActionConstants"; import { APP_MODE } from "entities/App"; import { call, put, select, takeEvery } from "redux-saga/effects"; import { getAppMode } from "@appsmith/selectors/entitiesSelector"; -import type { TJSLibrary } from "workers/common/JSLibrary"; +import type { JSLibrary } from "workers/common/JSLibrary"; import { logLatestLintPropertyErrors } from "./PostLintingSagas"; import { getAppsmithConfigs } from "@appsmith/configs"; import type { AppState } from "@appsmith/reducers"; @@ -29,7 +29,7 @@ const APPSMITH_CONFIGS = getAppsmithConfigs(); export const lintWorker = new Linter(); function* updateLintGlobals( - action: ReduxAction<{ add?: boolean; libs: TJSLibrary[] }>, + action: ReduxAction<{ add?: boolean; libs: JSLibrary[] }>, ) { const appMode: APP_MODE = yield select(getAppMode); const isEditorMode = appMode === APP_MODE.EDIT; diff --git a/app/client/src/selectors/actionSelectors.tsx b/app/client/src/selectors/actionSelectors.tsx index 47f3d97b49..4318ac8fd2 100644 --- a/app/client/src/selectors/actionSelectors.tsx +++ b/app/client/src/selectors/actionSelectors.tsx @@ -2,7 +2,7 @@ import type { DataTree } from "entities/DataTree/dataTreeTypes"; import { createSelector } from "reselect"; import WidgetFactory from "WidgetProvider/factory"; import type { FlattenedWidgetProps } from "WidgetProvider/constants"; -import type { TJSLibrary } from "workers/common/JSLibrary"; +import type { JSLibrary } from "workers/common/JSLibrary"; import { getDataTree } from "./dataTreeSelectors"; import { getExistingPageNames, @@ -28,7 +28,7 @@ export const getUsedActionNames = createSelector( pageNames: Record, dataTree: DataTree, parentWidget: FlattenedWidgetProps | undefined, - installedLibraries: TJSLibrary[], + installedLibraries: JSLibrary[], ) => { const map: Record = {}; // The logic has been copied from Explorer/Entity/Name.tsx Component. diff --git a/app/client/src/workers/Evaluation/handlers/__tests__/jsLibrary.test.ts b/app/client/src/workers/Evaluation/handlers/__tests__/jsLibrary.test.ts index 8ee15ccfb7..7f193a7215 100644 --- a/app/client/src/workers/Evaluation/handlers/__tests__/jsLibrary.test.ts +++ b/app/client/src/workers/Evaluation/handlers/__tests__/jsLibrary.test.ts @@ -7,14 +7,14 @@ import * as mod from "../../../common/JSLibrary/ternDefinitionGenerator"; jest.mock("../../../common/JSLibrary/ternDefinitionGenerator"); +declare const self: WorkerGlobalScope; + describe("Tests to assert install/uninstall flows", function () { beforeAll(() => { self.importScripts = jest.fn(() => { - //@ts-expect-error importScripts is not defined in the test environment self.lodash = {}; }); - //@ts-expect-error importScripts is not defined in the test environment self.import = jest.fn(); const mockTernDefsGenerator = jest.fn(() => ({})); @@ -49,7 +49,7 @@ describe("Tests to assert install/uninstall flows", function () { }); }); - it("Reinstalling a different version of the same installed library should fail", async function () { + it("Reinstalling a different version of the same installed library should create a new accessor", async function () { const res = await installLibrary({ data: { url: "https://cdnjs.cloudflare.com/ajax/libs/lodash.js/4.16.0/lodash.min.js", @@ -59,39 +59,43 @@ describe("Tests to assert install/uninstall flows", function () { method: EVAL_WORKER_ASYNC_ACTION.INSTALL_LIBRARY, }); expect(res).toEqual({ - success: false, - defs: {}, - error: expect.any(Error), + success: true, + defs: { + "!name": "LIB/lodash_1", + lodash_1: undefined, + }, + accessor: ["lodash_1"], }); }); - it("Detects name space collision where there is another entity(api, widget or query) with the same name", async function () { - //@ts-expect-error ignore - delete self.lodash; + it("Detects name space collision where there is another entity(api, widget or query) with the same name and creates a unique accessor", async function () { + delete self["lodash"]; const res = await installLibrary({ data: { url: "https://cdnjs.cloudflare.com/ajax/libs/lodash.js/4.16.0/lodash.min.js", - takenAccessors: [], + takenAccessors: ["lodash_1"], takenNamesMap: { lodash: true }, }, method: EVAL_WORKER_ASYNC_ACTION.INSTALL_LIBRARY, }); expect(res).toEqual({ - success: false, - defs: {}, - error: expect.any(Error), + success: true, + defs: { + "!name": "LIB/lodash_2", + lodash_2: undefined, + }, + accessor: ["lodash_2"], }); + delete self["lodash_2"]; }); - it("Removes or set the accessors to undefined on the global object on uninstallation", async function () { - //@ts-expect-error ignore + it("Removes or set the accessors to undefined on the global object on un-installation", async function () { self.lodash = {}; const res = await uninstallLibrary({ data: ["lodash"], method: EVAL_WORKER_SYNC_ACTION.UNINSTALL_LIBRARY, }); expect(res).toEqual({ success: true }); - //@ts-expect-error ignore expect(self.lodash).toBeUndefined(); }); }); diff --git a/app/client/src/workers/Evaluation/handlers/jsLibrary.ts b/app/client/src/workers/Evaluation/handlers/jsLibrary.ts index f45992d918..9e20f4ceea 100644 --- a/app/client/src/workers/Evaluation/handlers/jsLibrary.ts +++ b/app/client/src/workers/Evaluation/handlers/jsLibrary.ts @@ -9,10 +9,20 @@ import { JSLibraries, libraryReservedIdentifiers, } from "../../common/JSLibrary"; +import type { JSLibrary } from "../../common/JSLibrary"; import { resetJSLibraries } from "../../common/JSLibrary/resetJSLibraries"; import { makeTernDefs } from "../../common/JSLibrary/ternDefinitionGenerator"; import type { EvalWorkerASyncRequest, EvalWorkerSyncRequest } from "../types"; import { dataTreeEvaluator } from "./evalTree"; +import log from "loglevel"; + +declare global { + interface WorkerGlobalScope { + [x: string]: any; + } +} + +declare const self: WorkerGlobalScope; enum LibraryInstallError { NameCollisionError, @@ -21,16 +31,6 @@ enum LibraryInstallError { LibraryOverrideError, } -class NameCollisionError extends Error { - code = LibraryInstallError.NameCollisionError; - constructor(accessors: string) { - super( - createMessage(customJSLibraryMessages.NAME_COLLISION_ERROR, accessors), - ); - this.name = "NameCollisionError"; - } -} - class ImportError extends Error { code = LibraryInstallError.ImportError; constructor(url: string) { @@ -47,25 +47,13 @@ class TernDefinitionError extends Error { } } -class LibraryOverrideError extends Error { - code = LibraryInstallError.LibraryOverrideError; - data: any; - constructor(name: string, data: any) { - super(createMessage(customJSLibraryMessages.LIB_OVERRIDE_ERROR, name)); - this.name = "LibraryOverrideError"; - this.data = data; - } -} - const removeDataTreeFromContext = () => { if (!dataTreeEvaluator) return {}; const evalTree = dataTreeEvaluator?.getEvalTree(); const dataTreeEntityNames = Object.keys(evalTree); const tempDataTreeStore: Record = {}; for (const entityName of dataTreeEntityNames) { - // @ts-expect-error: self is a global variable tempDataTreeStore[entityName] = self[entityName]; - // @ts-expect-error: self is a global variable delete self[entityName]; } return tempDataTreeStore; @@ -76,12 +64,17 @@ function addTempStoredDataTreeToContext( ) { const dataTreeEntityNames = Object.keys(tempDataTreeStore); for (const entityName of dataTreeEntityNames) { - // @ts-expect-error: self is a global variable self[entityName] = tempDataTreeStore[entityName]; } } -export async function installLibrary(request: EvalWorkerASyncRequest) { +export async function installLibrary( + request: EvalWorkerASyncRequest<{ + url: string; + takenNamesMap: Record; + takenAccessors: Array; + }>, +) { const { data } = request; const { takenAccessors, takenNamesMap, url } = data; const defs: Def = {}; @@ -91,73 +84,105 @@ export async function installLibrary(request: EvalWorkerASyncRequest) { * We store the data tree in a temporary variable and add it back to the global scope after the library is imported. */ const tempDataTreeStore = removeDataTreeFromContext(); + + // Map of all the currently installed libraries + const libStore = takenAccessors.reduce( + (acc: Record, a: string) => { + acc[a] = self[a]; + return acc; + }, + {}, + ); + try { - const currentEnvKeys = Object.keys(self); + const envKeysBeforeInstallation = Object.keys(self); - //@ts-expect-error Find libraries that were uninstalled. - const unsetKeys = currentEnvKeys.filter((key) => self[key] === undefined); + /** Holds keys of uninstalled libraries that cannot be removed from worker global. + * Certain libraries are added to the global scope with { configurable: false } + */ + const unsetLibraryKeys = envKeysBeforeInstallation.filter( + (k) => self[k] === undefined, + ); + + const accessors: string[] = []; - const existingLibraries: Record = {}; - for (const acc of takenAccessors) { - existingLibraries[acc] = self[acc]; - } let module = null; try { self.importScripts(url); + // Find keys add that were installed to the global scope. + const keysAfterInstallation = Object.keys(self); + accessors.push( + ...difference(keysAfterInstallation, envKeysBeforeInstallation), + ); + + // Check the list of installed library to see if their values have changed. + // This is to check if the newly installed library overwrites an already existing + accessors.push( + ...Object.keys(libStore).filter((k) => { + return libStore[k] !== self[k]; + }), + ); + + accessors.push(...unsetLibraryKeys.filter((k) => self[k] !== undefined)); + + for (let i = 0; i < accessors.length; i++) { + if ( + takenNamesMap.hasOwnProperty(accessors[i]) || + takenAccessors.includes(accessors[i]) + ) { + const uniqueName = generateUniqueAccessor( + accessors[i], + takenAccessors, + takenNamesMap, + ); + self[uniqueName] = self[accessors[i]]; + accessors[i] = uniqueName; + } + } } catch (e) { + log.debug(e, `importScripts failed for ${url}`); try { module = await import(/* webpackIgnore: true */ url); + if (module && typeof module === "object") { + const uniqAccessor = generateUniqueAccessor( + url, + takenAccessors, + takenNamesMap, + ); + self[uniqAccessor] = flattenModule(module); + accessors.push(uniqAccessor); + } } catch (e) { + log.debug(e, `dynamic import failed for ${url}`); throw new ImportError(url); } } - // Find keys add that were installed to the global scope. - const accessors = difference( - Object.keys(self), - currentEnvKeys, - ) as Array; - - // If no keys were added to the global scope, check if the module is a ESM module. + // If no accessors at this point, installation likely failed. if (accessors.length === 0) { - if (module && typeof module === "object") { - const uniqAccessor = generateUniqueAccessor( - url, - takenAccessors, - takenNamesMap, - ); - // @ts-expect-error no types - self[uniqAccessor] = flattenModule(module); - accessors.push(uniqAccessor); - } + throw new Error("Unable to determine a unique accessor"); } - addTempStoredDataTreeToContext(tempDataTreeStore); - checkForNameCollision(accessors, takenNamesMap); - checkIfUninstalledEarlier(accessors, unsetKeys); - checkForOverrides(url, accessors, takenAccessors, existingLibraries); - if (accessors.length === 0) - return { status: false, defs, accessor: accessors }; - - //Reserves accessor names. const name = accessors[accessors.length - 1]; defs["!name"] = `LIB/${name}`; try { for (const key of accessors) { - //@ts-expect-error no types defs[key] = makeTernDefs(self[key]); } } catch (e) { for (const acc of accessors) { - //@ts-expect-error no types self[acc] = undefined; } + log.debug(e, `ternDefinitions failed for ${url}`); throw new TernDefinitionError( `Failed to generate autocomplete definitions: ${name}`, ); } + // All the existing library and the newly installed one is added to global. + Object.keys(libStore).forEach((k) => (self[k] = libStore[k])); + //Reserve accessor names. for (const acc of accessors) { //we have to update invalidEntityIdentifiers as well @@ -167,21 +192,20 @@ export async function installLibrary(request: EvalWorkerASyncRequest) { return { success: true, defs, accessor: accessors }; } catch (error) { + addTempStoredDataTreeToContext(tempDataTreeStore); + takenAccessors.forEach((k) => (self[k] = libStore[k])); return { success: false, defs, error }; } } -export function uninstallLibrary(request: EvalWorkerSyncRequest) { +export function uninstallLibrary( + request: EvalWorkerSyncRequest>, +) { const { data } = request; const accessor = data; try { for (const key of accessor) { - try { - delete self[key]; - } catch (e) { - //@ts-expect-error ignore - self[key] = undefined; - } + self[key] = undefined; //we have to update invalidEntityIdentifiers as well delete libraryReservedIdentifiers[key]; delete invalidEntityIdentifiers[key]; @@ -192,39 +216,60 @@ export function uninstallLibrary(request: EvalWorkerSyncRequest) { } } -export async function loadLibraries(request: EvalWorkerASyncRequest) { +export async function loadLibraries( + request: EvalWorkerASyncRequest, +) { resetJSLibraries(); - //Add types const { data: libs } = request; let message = ""; + const libStore: Record = {}; try { for (const lib of libs) { - const url = lib.url; - const accessor = lib.accessor; + const url = lib.url as string; + const accessors = lib.accessor; const keysBefore = Object.keys(self); let module = null; try { self.importScripts(url); + const keysAfter = Object.keys(self); + const defaultAccessors = difference(keysAfter, keysBefore); + + /** + * Installing 2 different version of lodash tries to add the same accessor on the self object. Let take version a & b for example. + * Installation of version a, will add _ to the self object and can be detected by looking at the differences in the previous step. + * Now when version b is installed, differences will be [], since _ already exists in the self object. + * We add all the installations to the libStore and see if the reference it points to in the self object changes. + * If the references changes it means that it a valid accessor. + */ + defaultAccessors.push( + ...Object.keys(libStore).filter((k) => libStore[k] !== self[k]), + ); + + /** Sort the accessor list from backend and installed accessor list using the same rule to apply all modifications. + * This is required only for UMD builds, since we always generate unique names for ESM. + */ + accessors.sort(); + defaultAccessors.sort(); + + for (let i = 0; i < defaultAccessors.length; i++) { + self[accessors[i]] = self[defaultAccessors[i]]; + libStore[defaultAccessors[i]] = self[defaultAccessors[i]]; + libraryReservedIdentifiers[accessors[i]] = true; + invalidEntityIdentifiers[accessors[i]] = true; + } } catch (e) { - message = (e as Error).message; try { module = await import(/* webpackIgnore: true */ url); + if (!module || typeof module !== "object") throw "Not an ESM module"; + const key = accessors[0]; + libStore[key] = module; + libraryReservedIdentifiers[key] = true; + invalidEntityIdentifiers[key] = true; } catch (e) { - message = (e as Error).message; + throw new ImportError(url); } } - const keysAfter = Object.keys(self); - const newKeys = difference(keysAfter, keysBefore); - if (newKeys.length === 0 && module && typeof module === "object") { - self[accessor[0]] = flattenModule(module); - newKeys.push(accessor[0]); - } - for (const key of newKeys) { - //we have to update invalidEntityIdentifiers as well - libraryReservedIdentifiers[key] = true; - invalidEntityIdentifiers[key] = true; - } } JSLibraries.push(...libs); return { success: true, message }; @@ -234,48 +279,6 @@ export async function loadLibraries(request: EvalWorkerASyncRequest) { } } -function checkForNameCollision( - accessor: string[], - takenNamesMap: Record, -) { - const collidingNames = accessor.filter((key: string) => takenNamesMap[key]); - if (collidingNames.length) { - for (const acc of accessor) { - //@ts-expect-error no types - self[acc] = undefined; - } - throw new NameCollisionError(collidingNames.join(", ")); - } -} - -function checkIfUninstalledEarlier(accessor: string[], unsetKeys: string[]) { - if (accessor.length > 0) return; - for (const key of unsetKeys) { - //@ts-expect-error no types - if (!self[key]) continue; - accessor.push(key); - } -} - -function checkForOverrides( - url: string, - accessor: string[], - takenAccessors: string[], - existingLibraries: Record, -) { - if (accessor.length > 0) return; - const overriddenAccessors: Array = []; - for (const acc of takenAccessors) { - //@ts-expect-error no types - if (existingLibraries[acc] === self[acc]) continue; - //@ts-expect-error no types - self[acc] = existingLibraries[acc]; - overriddenAccessors.push(acc); - } - if (overriddenAccessors.length === 0) return; - throw new LibraryOverrideError(url, overriddenAccessors); -} - /** * This function is called only for ESM modules and generates a unique namespace for the module. * @param url @@ -284,22 +287,24 @@ function checkForOverrides( * @returns */ function generateUniqueAccessor( - url: string, + urlOrName: string, takenAccessors: Array, takenNamesMap: Record, ) { + let name = urlOrName; // extract file name from url - const urlObject = new URL(url); - // URL pattern for ESM modules from jsDelivr - https://cdn.jsdelivr.net/npm/stripe@13.3.0/+esm - // Assuming the file name is the last part of the path - const urlPathParts = urlObject.pathname.split("/"); - let fileName = urlPathParts.pop(); - fileName = fileName?.includes("esm") ? urlPathParts.pop() : fileName; + try { + // Checks to see if a URL was passed + const urlObject = new URL(urlOrName); + // URL pattern for ESM modules from jsDelivr - https://cdn.jsdelivr.net/npm/stripe@13.3.0/+esm + // Assuming the file name is the last part of the path + const urlPathParts = urlObject.pathname.split("/"); + name = urlPathParts.pop() as string; + name = name?.includes("+esm") ? (urlPathParts.pop() as string) : name; + } catch (e) {} - // This should never happen. This is just to avoid the typescript error. - if (!fileName) throw new Error("Unable to generate a unique accessor"); // Replace all non-alphabetic characters with underscores and remove trailing underscores - const validVar = fileName.replace(/[^a-zA-Z]/g, "_").replace(/_+$/, ""); + const validVar = name.replace(/[^a-zA-Z]/g, "_").replace(/_+$/, ""); if ( !takenAccessors.includes(validVar) && !takenNamesMap.hasOwnProperty(validVar) @@ -308,7 +313,7 @@ function generateUniqueAccessor( } let index = 0; while (index++ < 100) { - const name = `Library_${index}`; + const name = `${validVar}_${index}`; if (!takenAccessors.includes(name) && !takenNamesMap.hasOwnProperty(name)) { return name; } diff --git a/app/client/src/workers/Evaluation/types.ts b/app/client/src/workers/Evaluation/types.ts index ee5f1f7f82..d416e8735b 100644 --- a/app/client/src/workers/Evaluation/types.ts +++ b/app/client/src/workers/Evaluation/types.ts @@ -19,9 +19,12 @@ import type { WorkerRequest } from "@appsmith/workers/common/types"; import type { DataTreeDiff } from "@appsmith/workers/Evaluation/evaluationUtils"; import type { APP_MODE } from "entities/App"; -export type EvalWorkerSyncRequest = WorkerRequest; -export type EvalWorkerASyncRequest = WorkerRequest< - any, +export type EvalWorkerSyncRequest = WorkerRequest< + T, + EVAL_WORKER_SYNC_ACTION +>; +export type EvalWorkerASyncRequest = WorkerRequest< + T, EVAL_WORKER_ASYNC_ACTION >; export type EvalWorkerResponse = EvalTreeResponseData | boolean | unknown; diff --git a/app/client/src/workers/Tern/tern.worker.ts b/app/client/src/workers/Tern/tern.worker.ts index 7c3d77c923..97b3f2bcc4 100644 --- a/app/client/src/workers/Tern/tern.worker.ts +++ b/app/client/src/workers/Tern/tern.worker.ts @@ -6,7 +6,6 @@ import ecma from "constants/defs/ecmascript.json"; import lodash from "constants/defs/lodash.json"; import base64 from "constants/defs/base64-js.json"; import moment from "constants/defs/moment.json"; -import xmlJs from "constants/defs/xmlParser.json"; import forge from "constants/defs/forge.json"; import browser from "constants/defs/browser.json"; import { @@ -64,7 +63,6 @@ function startServer(plugins = {}, scripts?: string[]) { lodash, base64, moment, - xmlJs, forge, ] as Def[], plugins: plugins, diff --git a/app/client/src/workers/common/JSLibrary/index.ts b/app/client/src/workers/common/JSLibrary/index.ts index 05d772d52e..9f23b48d71 100644 --- a/app/client/src/workers/common/JSLibrary/index.ts +++ b/app/client/src/workers/common/JSLibrary/index.ts @@ -1,7 +1,7 @@ import lodashPackageJson from "lodash/package.json"; import momentPackageJson from "moment-timezone/package.json"; -export interface TJSLibrary { +export interface JSLibrary { version?: string; docsURL: string; name: string; @@ -9,7 +9,7 @@ export interface TJSLibrary { url?: string; } -export const defaultLibraries: TJSLibrary[] = [ +export const defaultLibraries: JSLibrary[] = [ { accessor: ["_"], version: lodashPackageJson.version, @@ -22,12 +22,6 @@ export const defaultLibraries: TJSLibrary[] = [ docsURL: `https://momentjs.com/docs/`, name: "moment", }, - { - accessor: ["xmlParser"], - version: "3.17.5", - docsURL: "https://github.com/NaturalIntelligence/fast-xml-parser", - name: "xmlParser", - }, { accessor: ["forge"], version: "1.3.0", diff --git a/app/client/src/workers/common/JSLibrary/resetJSLibraries.ts b/app/client/src/workers/common/JSLibrary/resetJSLibraries.ts index 49e45ca18c..250c6c6132 100644 --- a/app/client/src/workers/common/JSLibrary/resetJSLibraries.ts +++ b/app/client/src/workers/common/JSLibrary/resetJSLibraries.ts @@ -1,6 +1,5 @@ import _ from "./lodash-wrapper"; import moment from "moment-timezone"; -import parser from "fast-xml-parser"; import forge from "node-forge"; import { defaultLibraries } from "./index"; import { JSLibraries, libraryReservedIdentifiers } from "./index"; @@ -8,7 +7,6 @@ import { invalidEntityIdentifiers } from "../DependencyMap/utils"; const defaultLibImplementations = { lodash: _, moment: moment, - xmlParser: parser, // We are removing some functionalities of node-forge because they wont // work in the worker thread forge: /*#__PURE*/ _.omit(forge, ["tls", "http", "xhr", "socket", "task"]), diff --git a/app/client/yarn.lock b/app/client/yarn.lock index 6706b5b8ae..dcb96e034d 100644 --- a/app/client/yarn.lock +++ b/app/client/yarn.lock @@ -10511,7 +10511,6 @@ __metadata: factory.ts: ^0.5.1 fast-deep-equal: ^3.1.3 fast-sort: ^3.4.0 - fast-xml-parser: ^3.17.5 fastdom: ^1.0.11 focus-trap-react: ^8.9.2 fuse.js: ^3.4.5 @@ -16679,15 +16678,6 @@ __metadata: languageName: node linkType: hard -"fast-xml-parser@npm:^3.17.5": - version: 3.17.5 - resolution: "fast-xml-parser@npm:3.17.5" - bin: - xml2js: cli.js - checksum: 6c436f540a4dcab67d395c0f6e8dc70090e6ced2ff73ed5fec181c1b25df755ed9fd9ba8271d6f70096fbe3add8b4eac130a5f4daeb12970427f72403a56934e - languageName: node - linkType: hard - "fastdom@npm:^1.0.11": version: 1.0.11 resolution: "fastdom@npm:1.0.11" diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/constants/ApplicationConstants.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/constants/ApplicationConstants.java index 0a7b2d2a0a..e79e33547a 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/constants/ApplicationConstants.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/constants/ApplicationConstants.java @@ -1,8 +1,34 @@ package com.appsmith.server.constants; +import com.appsmith.server.domains.CustomJSLib; + +import java.util.Set; + public class ApplicationConstants { public static final String[] APP_CARD_COLORS = { "#FFDEDE", "#FFEFDB", "#F3F1C7", "#F4FFDE", "#C7F3F0", "#D9E7FF", "#E3DEFF", "#F1DEFF", "#C7F3E3", "#F5D1D1", "#ECECEC", "#FBF4ED", "#D6D1F2", "#FFEBFB", "#EAEDFB" }; + + public static CustomJSLib getDefaultParserCustomJsLibCompatibilityDTO() { + CustomJSLib customJSLib = new CustomJSLib(); + customJSLib.setName("xmlParser"); + customJSLib.setVersion("3.17.5"); + customJSLib.setAccessor(Set.of("xmlParser")); + customJSLib.setUrl("https://cdnjs.cloudflare.com/ajax/libs/fast-xml-parser/3.17.5/parser.min.js"); + customJSLib.setDefs( + "{\"!name\":\"LIB/xmlParser\",\"xmlParser\":{\"parse\":{\"!type\":\"fn()\",\"prototype\":{}},\"convertTonimn\":{\"!type\":\"fn()\",\"prototype\":{}},\"getTraversalObj\":{\"!type\":\"fn()\",\"prototype\":{}},\"convertToJson\":{\"!type\":\"fn()\",\"prototype\":{}},\"convertToJsonString\":{\"!type\":\"fn()\",\"prototype\":{}},\"validate\":{\"!type\":\"fn()\",\"prototype\":{}},\"j2xParser\":{\"!type\":\"fn()\",\"prototype\":{\"parse\":{\"!type\":\"fn()\",\"prototype\":{}},\"j2x\":{\"!type\":\"fn()\",\"prototype\":{}}}},\"parseToNimn\":{\"!type\":\"fn()\",\"prototype\":{}}}}"); + customJSLib.setUidString(XML_PARSER_LIBRARY_UID); + return customJSLib; + } + + /** + * Appsmith provides xmlParser v 3.17.5 and few other customJSLibraries by default, xmlParser has been + * flagged because it has some vulnerabilities. Appsmith is stopping natively providing support for xmlParser. + * This however, would break existing applications which are using xmlParser. In order to prevent this, + * we are adding xmlParser as an add-onn to all existing applications and applications which will be imported + * This CustomJSLib UID needs to be added to all the imported applications where we don't have any later versions of xmlParser present. + */ + public static final String XML_PARSER_LIBRARY_UID = + "xmlParser_https://cdnjs.cloudflare.com/ajax/libs/fast-xml-parser/3.17.5/parser.min.js"; } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/imports/internal/ImportApplicationServiceCEImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/imports/internal/ImportApplicationServiceCEImpl.java index 0f45d9f0b4..1de6ffcb7b 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/imports/internal/ImportApplicationServiceCEImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/imports/internal/ImportApplicationServiceCEImpl.java @@ -16,6 +16,7 @@ import com.appsmith.external.models.DefaultResources; import com.appsmith.external.models.OAuth2; import com.appsmith.external.models.Policy; import com.appsmith.server.actioncollections.base.ActionCollectionService; +import com.appsmith.server.constants.ApplicationConstants; import com.appsmith.server.constants.FieldName; import com.appsmith.server.constants.ResourceModes; import com.appsmith.server.datasources.base.DatasourceService; @@ -1135,6 +1136,8 @@ public class ImportApplicationServiceCEImpl implements ImportApplicationServiceC customJSLibs = new ArrayList<>(); } + ensureXmlParserPresenceInCustomJsLibList(customJSLibs); + return Flux.fromIterable(customJSLibs) .flatMap(customJSLib -> { customJSLib.setId(null); @@ -1154,6 +1157,32 @@ public class ImportApplicationServiceCEImpl implements ImportApplicationServiceC }); } + /** + * This method takes customJSLibList from application JSON, checks if an entry for XML parser exists, + * otherwise adds the entry. + * This has been done to add the xmlParser entry in imported application as appsmith is stopping native support + * for xml parser. + * Read More: https://github.com/appsmithorg/appsmith/pull/28012 + * + * @param customJSLibList + */ + public void ensureXmlParserPresenceInCustomJsLibList(List customJSLibList) { + boolean isXmlParserLibFound = false; + for (CustomJSLib customJSLib : customJSLibList) { + if (!customJSLib.getUidString().equals(ApplicationConstants.XML_PARSER_LIBRARY_UID)) { + continue; + } + + isXmlParserLibFound = true; + break; + } + + if (!isXmlParserLibFound) { + CustomJSLib xmlParserJsLib = ApplicationConstants.getDefaultParserCustomJsLibCompatibilityDTO(); + customJSLibList.add(xmlParserJsLib); + } + } + private Mono> getExistingDatasourceMono(String applicationId, Flux datasourceFlux) { Mono> existingDatasourceMono; // Check if the request is to hydrate the application to DB for particular branch diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/MigrationHelperMethods.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/MigrationHelperMethods.java index 6b14b9b64d..a361eb3965 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/MigrationHelperMethods.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/MigrationHelperMethods.java @@ -3,6 +3,7 @@ package com.appsmith.server.migrations; import com.appsmith.external.models.ActionDTO; import com.appsmith.external.models.BaseDomain; import com.appsmith.external.models.InvisibleActionFields; +import com.appsmith.server.constants.FieldName; import com.appsmith.server.constants.ResourceModes; import com.appsmith.server.domains.ApplicationPage; import com.appsmith.server.domains.NewAction; @@ -236,4 +237,23 @@ public class MigrationHelperMethods { mongoTemplate.find(query(where(fieldName(path)).is(id)), type); return domainObject; } + + /** + * The method provides the criteria for any document to qualify as not deleted + * @return Criteria + */ + public static Criteria notDeleted() { + return new Criteria() + .andOperator( + // Older check for deleted + new Criteria() + .orOperator( + where(FieldName.DELETED).exists(false), + where(FieldName.DELETED).is(false)), + // New check for deleted + new Criteria() + .orOperator( + where(FieldName.DELETED_AT).exists(false), + where(FieldName.DELETED_AT).is(null))); + } } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/db/ce/Migration032AddingXmlParserToApplicationLibraries.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/db/ce/Migration032AddingXmlParserToApplicationLibraries.java new file mode 100644 index 0000000000..4cccb969ac --- /dev/null +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/db/ce/Migration032AddingXmlParserToApplicationLibraries.java @@ -0,0 +1,90 @@ +package com.appsmith.server.migrations.db.ce; + +import com.appsmith.server.constants.ApplicationConstants; +import com.appsmith.server.domains.Application; +import com.appsmith.server.domains.CustomJSLib; +import com.appsmith.server.domains.QApplication; +import com.appsmith.server.dtos.CustomJSLibApplicationDTO; +import com.appsmith.server.exceptions.AppsmithError; +import com.appsmith.server.exceptions.AppsmithException; +import io.mongock.api.annotations.ChangeUnit; +import io.mongock.api.annotations.Execution; +import io.mongock.api.annotations.RollbackExecution; +import lombok.extern.slf4j.Slf4j; +import org.springframework.dao.DuplicateKeyException; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.data.mongodb.core.query.Criteria; +import org.springframework.data.mongodb.core.query.Query; +import org.springframework.data.mongodb.core.query.Update; + +import static com.appsmith.server.constants.ApplicationConstants.XML_PARSER_LIBRARY_UID; +import static com.appsmith.server.migrations.MigrationHelperMethods.notDeleted; +import static com.appsmith.server.repositories.ce.BaseAppsmithRepositoryCEImpl.fieldName; + +/** + * Appsmith provides xmlParser v 3.17.5 and few other customJSLibraries by default, xmlParser has been + * flagged because it has some vulnerabilities. Appsmith is stopping natively providing support for xmlParser. + * This however, would break existing applications which are using xmlParser. In order to prevent this, + * applications require to have xmlParser as added library. + * + * This migration takes care of adding a document in customJsLib for xml-parser and adding corresponding entry + * in application collection + * + */ +@Slf4j +@ChangeUnit(order = "032", id = "add-xml-parser-to-application-jslibs", author = " ") +public class Migration032AddingXmlParserToApplicationLibraries { + + private final MongoTemplate mongoTemplate; + private static final String UNPUBLISHED_CUSTOM_JS_LIBS = + fieldName(QApplication.application.unpublishedCustomJSLibs); + private static final String PUBLISHED_CUSTOM_JS_LIBS = fieldName(QApplication.application.publishedCustomJSLibs); + + public Migration032AddingXmlParserToApplicationLibraries(MongoTemplate mongoTemplate) { + this.mongoTemplate = mongoTemplate; + } + + @RollbackExecution + public void rollbackExecution() {} + + @Execution + public void addXmlParserEntryToEachApplication() { + // add the xml-parser to customJSLib if it's not already present + CustomJSLib customJSLib = generateXmlParserJSLibObject(); + try { + mongoTemplate.save(customJSLib); + } catch (DuplicateKeyException duplicateKeyException) { + log.debug( + "Addition of xmlParser object in customJSLib failed, because object with similar UID already exists"); + } catch (Exception exception) { + throw new AppsmithException( + AppsmithError.MIGRATION_FAILED, + "Migration032AddingXmlParserToApplicationLibraries", + exception.getMessage(), + "Unable to insert xml parser library in CustomJSLib collection"); + } + + // add uid entry in all these custom js libs + Update pushXmlParserUpdate = new Update() + .addToSet(PUBLISHED_CUSTOM_JS_LIBS, getXmlParserCustomJSLibApplicationDTO()) + .addToSet(UNPUBLISHED_CUSTOM_JS_LIBS, getXmlParserCustomJSLibApplicationDTO()); + + log.debug("Going to add Xml Parser uid in all application DTOs"); + mongoTemplate.updateMulti( + new Query().addCriteria(getMigrationCriteria()), pushXmlParserUpdate, Application.class); + } + + private CustomJSLibApplicationDTO getXmlParserCustomJSLibApplicationDTO() { + CustomJSLibApplicationDTO xmlParserApplicationDTO = new CustomJSLibApplicationDTO(); + xmlParserApplicationDTO.setUidString(XML_PARSER_LIBRARY_UID); + return xmlParserApplicationDTO; + } + + private static CustomJSLib generateXmlParserJSLibObject() { + return ApplicationConstants.getDefaultParserCustomJsLibCompatibilityDTO(); + } + + private static Criteria getMigrationCriteria() { + return notDeleted(); + } +} diff --git a/app/server/appsmith-server/src/test/java/com/appsmith/server/solutions/ImportApplicationServiceTests.java b/app/server/appsmith-server/src/test/java/com/appsmith/server/solutions/ImportApplicationServiceTests.java index 56cc8fe2e9..e799141df3 100644 --- a/app/server/appsmith-server/src/test/java/com/appsmith/server/solutions/ImportApplicationServiceTests.java +++ b/app/server/appsmith-server/src/test/java/com/appsmith/server/solutions/ImportApplicationServiceTests.java @@ -968,7 +968,10 @@ public class ImportApplicationServiceTests { final List actionCollectionList = tuple.getT5(); final List importedJSLibList = tuple.getT6(); - assertEquals(1, importedJSLibList.size()); + // although the imported list had only one jsLib entry, the other entry comes from ensuring an xml + // parser entry + // for backward compatibility + assertEquals(2, importedJSLibList.size()); CustomJSLib importedJSLib = (CustomJSLib) importedJSLibList.toArray()[0]; CustomJSLib expectedJSLib = new CustomJSLib( "TestLib", Set.of("accessor1"), "url", "docsUrl", "1" + ".0", "defs_string"); @@ -978,10 +981,10 @@ public class ImportApplicationServiceTests { assertEquals(expectedJSLib.getDocsUrl(), importedJSLib.getDocsUrl()); assertEquals(expectedJSLib.getVersion(), importedJSLib.getVersion()); assertEquals(expectedJSLib.getDefs(), importedJSLib.getDefs()); - assertEquals(1, application.getUnpublishedCustomJSLibs().size()); - assertEquals( - getDTOFromCustomJSLib(expectedJSLib), - application.getUnpublishedCustomJSLibs().toArray()[0]); + // although the imported list had only one jsLib entry, the other entry comes from ensuring an xml + // parser entry + // for backward compatibility + assertEquals(2, application.getUnpublishedCustomJSLibs().size()); assertThat(application.getName()).isEqualTo("valid_application"); assertThat(application.getWorkspaceId()).isNotNull();