From 5e0c4d6d2f8528827ba3af37a95c631cbf1ad0f8 Mon Sep 17 00:00:00 2001 From: Trisha Anand Date: Fri, 17 Jun 2022 15:01:52 +0530 Subject: [PATCH 01/12] fix: Fix migration for overwriting recently used workspaces (#14634) --- .../server/migrations/DatabaseChangelog2.java | 51 ++----------------- 1 file changed, 5 insertions(+), 46 deletions(-) diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/DatabaseChangelog2.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/DatabaseChangelog2.java index df5f2dbc3d..090483443f 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/DatabaseChangelog2.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/DatabaseChangelog2.java @@ -1,89 +1,50 @@ package com.appsmith.server.migrations; -import com.appsmith.external.models.ApiTemplate; -import com.appsmith.external.models.BaseDomain; -import com.appsmith.external.models.Category; import com.appsmith.external.models.Datasource; import com.appsmith.external.models.Property; -import com.appsmith.external.models.Provider; import com.appsmith.external.models.QBaseDomain; import com.appsmith.external.models.QDatasource; -import com.appsmith.server.acl.AppsmithRole; import com.appsmith.server.constants.FieldName; -import com.appsmith.server.domains.Action; import com.appsmith.server.domains.ActionCollection; import com.appsmith.server.domains.Application; import com.appsmith.server.domains.ApplicationPage; -import com.appsmith.server.domains.Asset; -import com.appsmith.server.domains.Collection; import com.appsmith.server.domains.Comment; -import com.appsmith.server.domains.CommentNotification; import com.appsmith.server.domains.CommentThread; -import com.appsmith.server.domains.CommentThreadNotification; -import com.appsmith.server.domains.Config; -import com.appsmith.server.domains.GitDeployKeys; -import com.appsmith.server.domains.Group; -import com.appsmith.server.domains.InviteUser; -import com.appsmith.server.domains.Layout; import com.appsmith.server.domains.NewAction; import com.appsmith.server.domains.NewPage; -import com.appsmith.server.domains.Notification; import com.appsmith.server.domains.Organization; -import com.appsmith.server.domains.Page; -import com.appsmith.server.domains.PasswordResetToken; import com.appsmith.server.domains.Plugin; -import com.appsmith.server.domains.QAction; -import com.appsmith.server.domains.QActionCollection; import com.appsmith.server.domains.PricingPlan; +import com.appsmith.server.domains.QActionCollection; import com.appsmith.server.domains.QApplication; -import com.appsmith.server.domains.QCollection; import com.appsmith.server.domains.QComment; import com.appsmith.server.domains.QCommentThread; -import com.appsmith.server.domains.QConfig; -import com.appsmith.server.domains.QGroup; -import com.appsmith.server.domains.QInviteUser; import com.appsmith.server.domains.QNewAction; import com.appsmith.server.domains.QNewPage; import com.appsmith.server.domains.QOrganization; import com.appsmith.server.domains.QPlugin; +import com.appsmith.server.domains.QTenant; import com.appsmith.server.domains.QTheme; import com.appsmith.server.domains.QUser; import com.appsmith.server.domains.QUserData; import com.appsmith.server.domains.QWorkspace; -import com.appsmith.server.domains.Role; -import com.appsmith.server.domains.Sequence; -import com.appsmith.server.domains.Theme; -import com.appsmith.server.domains.UsagePulse; -import com.appsmith.server.domains.User; -import com.appsmith.server.domains.UserData; -import com.appsmith.server.domains.QTenant; import com.appsmith.server.domains.Sequence; import com.appsmith.server.domains.Tenant; +import com.appsmith.server.domains.Theme; import com.appsmith.server.domains.User; +import com.appsmith.server.domains.UserData; import com.appsmith.server.domains.Workspace; -import com.appsmith.server.domains.WorkspacePlugin; import com.appsmith.server.dtos.ActionDTO; -import com.appsmith.server.dtos.ApplicationTemplate; -import com.appsmith.server.dtos.ResetUserPasswordDTO; import com.appsmith.server.exceptions.AppsmithError; import com.appsmith.server.exceptions.AppsmithException; import com.appsmith.server.helpers.TextUtils; -import com.appsmith.server.services.ce.ConfigServiceCE; -import com.appsmith.server.services.ce.ConfigServiceCEImpl; import com.github.cloudyrock.mongock.ChangeLog; import com.github.cloudyrock.mongock.ChangeSet; import com.github.cloudyrock.mongock.driver.mongodb.springdata.v3.decorator.impl.MongockTemplate; import com.google.gson.Gson; import lombok.extern.slf4j.Slf4j; -import reactor.core.publisher.Flux; - -import org.bson.BsonArray; -import org.bson.Document; -import org.springframework.data.mongodb.core.aggregation.AggregationOperation; -import org.springframework.data.mongodb.core.aggregation.AggregationPipeline; import org.springframework.data.mongodb.core.aggregation.AggregationUpdate; import org.springframework.data.mongodb.core.aggregation.Fields; -import org.springframework.data.mongodb.core.aggregation.SetOperation; import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Query; import org.springframework.data.mongodb.core.query.Update; @@ -94,14 +55,12 @@ import org.springframework.util.StringUtils; import reactor.core.publisher.Flux; import java.time.Instant; -import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.Map.Entry; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -967,7 +926,7 @@ public class DatabaseChangelog2 { AggregationUpdate.update().set(fieldName(QTheme.theme.workspaceId)).toValueOf(Fields.field(fieldName(QTheme.theme.organizationId))), Theme.class); mongockTemplate.updateMulti(new Query(), - AggregationUpdate.update().set(fieldName(QUserData.userData.recentlyUsedOrgIds)).toValueOf(Fields.field(fieldName(QUserData.userData.recentlyUsedWorkspaceIds))), + AggregationUpdate.update().set(fieldName(QUserData.userData.recentlyUsedWorkspaceIds)).toValueOf(Fields.field(fieldName(QUserData.userData.recentlyUsedOrgIds))), UserData.class); mongockTemplate.updateMulti(new Query(), AggregationUpdate.update().set(fieldName(QWorkspace.workspace.isAutoGeneratedWorkspace)).toValueOf(Fields.field(fieldName(QWorkspace.workspace.isAutoGeneratedOrganization))), From 7ae511f402e8907d024f593da22453d4bed67534 Mon Sep 17 00:00:00 2001 From: Arpit Mohan Date: Wed, 22 Jun 2022 06:27:40 +0200 Subject: [PATCH 02/12] ci: Revert "ci: Use buildjet runners for Cypress tests, and fewer of them" Reverts #14636 Reverting this change for now because there is an issue when re-running the failed specs. The action is unable to find the correct directory. --- .github/workflows/test-build-docker-image.yml | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-build-docker-image.yml b/.github/workflows/test-build-docker-image.yml index cfca4329ec..d2f1a66784 100644 --- a/.github/workflows/test-build-docker-image.yml +++ b/.github/workflows/test-build-docker-image.yml @@ -761,7 +761,7 @@ jobs: (github.event_name == 'pull_request_review' && github.event.review.state == 'approved' && github.event.pull_request.head.repo.full_name == github.repository)) - runs-on: buildjet-4vcpu-ubuntu-2004 + runs-on: ubuntu-latest defaults: run: working-directory: app/client @@ -778,6 +778,23 @@ jobs: 4, 5, 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, ] # Service containers to run with this job. Required for running tests From dfab0570452f80116de7816edd2b02a7c24a50fc Mon Sep 17 00:00:00 2001 From: Aishwarya-U-R <91450662+Aishwarya-U-R@users.noreply.github.com> Date: Wed, 6 Jul 2022 20:52:24 +0530 Subject: [PATCH 03/12] test: Script fixes for failing cases due to UQI css changes (#15037) * flaky fixes * Eval value pop up hinder fix * JS delete from EE fix * ActionContextMenuByEntityName() fix * Adding Escape() * Toasts handling --- .../ClientSideTests/BugTests/Bug14299_Spec.ts | 5 +++-- .../ExplorerTests/JSEditorContextMenu_Spec.ts | 2 +- .../ClientSideTests/Widgets/TableBugs_Spec.ts | 2 +- .../ClientSideTests/Widgets/TableFilter_Spec.ts | 2 +- .../ServerSideTests/GenerateCRUD/Postgres_Spec.ts | 2 ++ .../JsFunctionExecution/JSFunctionExecution_spec.ts | 2 ++ .../ServerSideTests/OnLoadTests/JSOnLoad_Spec.ts | 6 +++--- .../ServerSideTests/Params/PassingParams_Spec.ts | 3 +-- app/client/cypress/support/Pages/AggregateHelper.ts | 6 +++++- app/client/cypress/support/Pages/EntityExplorer.ts | 5 +++++ 10 files changed, 24 insertions(+), 11 deletions(-) diff --git a/app/client/cypress/integration/Smoke_TestSuite/ClientSideTests/BugTests/Bug14299_Spec.ts b/app/client/cypress/integration/Smoke_TestSuite/ClientSideTests/BugTests/Bug14299_Spec.ts index 6220978d54..a18161cdf1 100644 --- a/app/client/cypress/integration/Smoke_TestSuite/ClientSideTests/BugTests/Bug14299_Spec.ts +++ b/app/client/cypress/integration/Smoke_TestSuite/ClientSideTests/BugTests/Bug14299_Spec.ts @@ -119,13 +119,14 @@ describe("[Bug]: The data from the query does not show up on the widget #14299", }); it("4. Verify Deletion of the datasource after all created queries are Deleted", () => { - deployMode.NavigateBacktoEditor() + deployMode.NavigateBacktoEditor(); + agHelper.WaitUntilToastDisappear("ran successfully"); //runAstros triggered on PageLaoad of Edit page! ee.ExpandCollapseEntity("QUERIES/JS"); ee.ActionContextMenuByEntityName("getAstronauts", "Delete", "Are you sure?"); ee.ActionContextMenuByEntityName( "JSObject1", "Delete", - "Are you sure?", + "Are you sure?", true ); deployMode.DeployApp(locator._widgetInDeployed("tablewidget"), false); deployMode.NavigateBacktoEditor(); diff --git a/app/client/cypress/integration/Smoke_TestSuite/ClientSideTests/ExplorerTests/JSEditorContextMenu_Spec.ts b/app/client/cypress/integration/Smoke_TestSuite/ClientSideTests/ExplorerTests/JSEditorContextMenu_Spec.ts index 136316756b..0639299d9d 100644 --- a/app/client/cypress/integration/Smoke_TestSuite/ClientSideTests/ExplorerTests/JSEditorContextMenu_Spec.ts +++ b/app/client/cypress/integration/Smoke_TestSuite/ClientSideTests/ExplorerTests/JSEditorContextMenu_Spec.ts @@ -62,7 +62,7 @@ describe("Validate basic operations on Entity explorer JSEditor structure", () = ee.ActionContextMenuByEntityName( "ExplorerRenamed", "Delete", - "Are you sure?", + "Are you sure?", true ); ee.AssertEntityAbsenceInExplorer("ExplorerRenamed"); }); diff --git a/app/client/cypress/integration/Smoke_TestSuite/ClientSideTests/Widgets/TableBugs_Spec.ts b/app/client/cypress/integration/Smoke_TestSuite/ClientSideTests/Widgets/TableBugs_Spec.ts index 78925817b7..daee7c7eb9 100644 --- a/app/client/cypress/integration/Smoke_TestSuite/ClientSideTests/Widgets/TableBugs_Spec.ts +++ b/app/client/cypress/integration/Smoke_TestSuite/ClientSideTests/Widgets/TableBugs_Spec.ts @@ -18,7 +18,7 @@ describe("Verify various Table property bugs", function () { ee.DragDropWidgetNVerify("tablewidget", 250, 250); propPane.UpdatePropertyFieldValue("Table Data", JSON.stringify(dataSet.TableURLColumnType)); agHelper.ValidateNetworkStatus("@updateLayout", 200); - cy.get('body').type("{esc}"); + agHelper.Escape(); }); it("2. Bug 13299 - Verify Display Text does not contain garbage value for URL column type when empty", function () { diff --git a/app/client/cypress/integration/Smoke_TestSuite/ClientSideTests/Widgets/TableFilter_Spec.ts b/app/client/cypress/integration/Smoke_TestSuite/ClientSideTests/Widgets/TableFilter_Spec.ts index 1dc81a44c1..8300f06391 100644 --- a/app/client/cypress/integration/Smoke_TestSuite/ClientSideTests/Widgets/TableFilter_Spec.ts +++ b/app/client/cypress/integration/Smoke_TestSuite/ClientSideTests/Widgets/TableFilter_Spec.ts @@ -19,7 +19,7 @@ describe("Verify various Table_Filter combinations", function () { ee.DragDropWidgetNVerify("tablewidget", 250, 250); propPane.UpdatePropertyFieldValue("Table Data", JSON.stringify(dataSet.TableInput)); agHelper.ValidateNetworkStatus("@updateLayout", 200); - cy.get('body').type("{esc}"); + agHelper.Escape(); deployMode.DeployApp() }); diff --git a/app/client/cypress/integration/Smoke_TestSuite/ServerSideTests/GenerateCRUD/Postgres_Spec.ts b/app/client/cypress/integration/Smoke_TestSuite/ServerSideTests/GenerateCRUD/Postgres_Spec.ts index dde7414956..0c61408b13 100644 --- a/app/client/cypress/integration/Smoke_TestSuite/ServerSideTests/GenerateCRUD/Postgres_Spec.ts +++ b/app/client/cypress/integration/Smoke_TestSuite/ServerSideTests/GenerateCRUD/Postgres_Spec.ts @@ -276,6 +276,7 @@ describe("Validate Postgres Generate CRUD with JSON Form", () => { ee.SelectEntityByName("UpdateQuery", "QUERIES/JS"); dataSources.EnterQuery(updateQuery); + agHelper.Escape(); agHelper.AssertAutoSave(); ee.ExpandCollapseEntity("QUERIES/JS", false); }); @@ -518,6 +519,7 @@ describe("Validate Postgres Generate CRUD with JSON Form", () => { ee.SelectEntityByName("InsertQuery", "QUERIES/JS"); dataSources.EnterQuery(insertQuery); + agHelper.Escape(); agHelper.AssertAutoSave(); ee.ExpandCollapseEntity("QUERIES/JS", false); }); diff --git a/app/client/cypress/integration/Smoke_TestSuite/ServerSideTests/JsFunctionExecution/JSFunctionExecution_spec.ts b/app/client/cypress/integration/Smoke_TestSuite/ServerSideTests/JsFunctionExecution/JSFunctionExecution_spec.ts index 604996c0b4..dd2fbf14dd 100644 --- a/app/client/cypress/integration/Smoke_TestSuite/ServerSideTests/JsFunctionExecution/JSFunctionExecution_spec.ts +++ b/app/client/cypress/integration/Smoke_TestSuite/ServerSideTests/JsFunctionExecution/JSFunctionExecution_spec.ts @@ -244,6 +244,7 @@ describe("JS Function Execution", function() { // Re-introduce parse errors jsEditor.EditJSObj(JS_OBJECT_WITH_PARSE_ERROR); agHelper.GetNClick(jsEditor._runButton); + agHelper.WaitUntilToastDisappear("ran successfully"); //to not hinder with next toast msg in next case! // Assert that there is a function execution parse error jsEditor.AssertParseError(true, true); @@ -255,6 +256,7 @@ describe("JS Function Execution", function() { "TypeError: Cannot read properties of undefined (reading 'name')", ).should("not.exist"); }); + it("6. Supports the use of large JSON data (doesn't crash)", () => { const jsObjectWithLargeJSONData = `export default{ largeData: ${JSON.stringify(largeJSONData)}, diff --git a/app/client/cypress/integration/Smoke_TestSuite/ServerSideTests/OnLoadTests/JSOnLoad_Spec.ts b/app/client/cypress/integration/Smoke_TestSuite/ServerSideTests/OnLoadTests/JSOnLoad_Spec.ts index 1015cb68cb..ed10e9d16f 100644 --- a/app/client/cypress/integration/Smoke_TestSuite/ServerSideTests/OnLoadTests/JSOnLoad_Spec.ts +++ b/app/client/cypress/integration/Smoke_TestSuite/ServerSideTests/OnLoadTests/JSOnLoad_Spec.ts @@ -170,7 +170,7 @@ describe("JSObjects OnLoad Actions tests", function() { ee.ActionContextMenuByEntityName( jsName as string, "Delete", - "Are you sure?", + "Are you sure?", true ); ee.ActionContextMenuByEntityName("GetUser", "Delete", "Are you sure?"); @@ -220,7 +220,7 @@ describe("JSObjects OnLoad Actions tests", function() { ee.ActionContextMenuByEntityName( jsName as string, "Delete", - "Are you sure?", + "Are you sure?", true ); }); }); @@ -559,7 +559,7 @@ describe("JSObjects OnLoad Actions tests", function() { ee.ActionContextMenuByEntityName( jsName as string, "Delete", - "Are you sure?", + "Are you sure?", true ); }); diff --git a/app/client/cypress/integration/Smoke_TestSuite/ServerSideTests/Params/PassingParams_Spec.ts b/app/client/cypress/integration/Smoke_TestSuite/ServerSideTests/Params/PassingParams_Spec.ts index ef740ff871..696b9107d2 100644 --- a/app/client/cypress/integration/Smoke_TestSuite/ServerSideTests/Params/PassingParams_Spec.ts +++ b/app/client/cypress/integration/Smoke_TestSuite/ServerSideTests/Params/PassingParams_Spec.ts @@ -231,9 +231,8 @@ describe("[Bug] - 10784 - Passing params from JS to SQL query should not break", ee.ActionContextMenuByEntityName( jsName as string, "Delete", - "Are you sure?", + "Are you sure?", true ); - agHelper.ValidateNetworkStatus("@deleteJSCollection", 200); // //Bug 12532 // ee.expandCollapseEntity('DATASOURCES') // ee.ActionContextMenuByEntityName(guid, 'Delete', 'Are you sure?') diff --git a/app/client/cypress/support/Pages/AggregateHelper.ts b/app/client/cypress/support/Pages/AggregateHelper.ts index 8b52234717..9259968ce1 100644 --- a/app/client/cypress/support/Pages/AggregateHelper.ts +++ b/app/client/cypress/support/Pages/AggregateHelper.ts @@ -330,12 +330,16 @@ export class AggregateHelper { } // //closing multiselect dropdown - cy.get("body").type("{esc}"); + this.Escape(); // cy.get(this.locator._widgetInDeployed(endpoint)) // .eq(index) // .click() } + public Escape(){ + cy.get('body').type("{esc}"); + } + public RemoveMultiSelectItems(items: string[]) { items.forEach(($each) => { cy.xpath(this.locator._multiSelectItem($each)) diff --git a/app/client/cypress/support/Pages/EntityExplorer.ts b/app/client/cypress/support/Pages/EntityExplorer.ts index 6d8862f436..5f23c92ae9 100644 --- a/app/client/cypress/support/Pages/EntityExplorer.ts +++ b/app/client/cypress/support/Pages/EntityExplorer.ts @@ -104,6 +104,7 @@ export class EntityExplorer { entityNameinLeftSidebar: string, action = "Delete", subAction = "", + jsDelete = false, ) { this.agHelper.Sleep(); cy.xpath(this._contextMenu(entityNameinLeftSidebar)) @@ -115,6 +116,10 @@ export class EntityExplorer { cy.xpath(this._contextMenuItem(subAction)).click({ force: true }); this.agHelper.Sleep(500); } + if (action == "Delete") { + jsDelete && this.agHelper.ValidateNetworkStatus("@deleteJSCollection"); + jsDelete && this.agHelper.WaitUntilToastDisappear("deleted successfully"); + } } public ActionTemplateMenuByEntityName( From 887d57b86b80c9179a4a3094a008483cc2b14650 Mon Sep 17 00:00:00 2001 From: Arsalan Yaldram Date: Wed, 13 Jul 2022 13:04:23 +0530 Subject: [PATCH 04/12] fix: using old versions of marked (#15151) (cherry picked from commit 8428ae506a02ec477027b82936ff003c0c53cafb) --- app/client/package.json | 4 ++-- .../GlobalSearch/parseDocumentationContent.ts | 7 +++---- app/client/yarn.lock | 16 ++++++++-------- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/app/client/package.json b/app/client/package.json index cc69c86560..80c9762b6a 100644 --- a/app/client/package.json +++ b/app/client/package.json @@ -78,7 +78,7 @@ "loglevel": "^1.7.1", "lottie-web": "^5.7.4", "mammoth": "^1.4.19", - "marked": "^4.0.17", + "marked": "^2.0.0", "memoize-one": "^5.2.1", "micro-memoize": "^4.0.10", "moment": "2.29.3", @@ -201,7 +201,7 @@ "@types/js-beautify": "^1.13.2", "@types/jshint": "^2.12.0", "@types/lodash": "^4.14.120", - "@types/marked": "^4.0.3", + "@types/marked": "^1.2.2", "@types/moment-timezone": "^0.5.10", "@types/nanoid": "^2.0.0", "@types/node": "^10.12.18", diff --git a/app/client/src/components/editorComponents/GlobalSearch/parseDocumentationContent.ts b/app/client/src/components/editorComponents/GlobalSearch/parseDocumentationContent.ts index b66a8db3cc..ca78fee185 100644 --- a/app/client/src/components/editorComponents/GlobalSearch/parseDocumentationContent.ts +++ b/app/client/src/components/editorComponents/GlobalSearch/parseDocumentationContent.ts @@ -1,8 +1,7 @@ import { HelpBaseURL } from "constants/HelpConstants"; import { algoliaHighlightTag } from "./utils"; import log from "loglevel"; - -const { marked } = require("marked"); +import marked, { Token } from "marked"; /** * @param {String} HTML representing a single element @@ -123,8 +122,8 @@ const parseMarkdown = (value: string) => { value = replaceHintTagsWithCode(stripDescriptionMarkdown(value)); marked.use({ - walkTokens(token: any) { - const currentToken = token; + walkTokens(token: unknown) { + const currentToken = token as Token; if ("type" in currentToken && currentToken.type === "link") { let href = currentToken.href; try { diff --git a/app/client/yarn.lock b/app/client/yarn.lock index 2a38d1590e..ab64d4f6b0 100644 --- a/app/client/yarn.lock +++ b/app/client/yarn.lock @@ -3057,10 +3057,10 @@ version "4.14.169" resolved "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.169.tgz" -"@types/marked@^4.0.3": - version "4.0.3" - resolved "https://registry.yarnpkg.com/@types/marked/-/marked-4.0.3.tgz#2098f4a77adaba9ce881c9e0b6baf29116e5acc4" - integrity sha512-HnMWQkLJEf/PnxZIfbm0yGJRRZYYMhb++O9M36UCTA9z53uPvVoSlAwJr3XOpDEryb7Hwl1qAx/MV6YIW1RXxg== +"@types/marked@^1.2.2": + version "1.2.2" + resolved "https://registry.yarnpkg.com/@types/marked/-/marked-1.2.2.tgz#1f858a0e690247ecf3b2eef576f98f86e8d960d4" + integrity sha512-wLfw1hnuuDYrFz97IzJja0pdVsC0oedtS4QsKH1/inyW9qkLQbXgMUqEQT0MVtUBx3twjWeInUfjQbhBVLECXw== "@types/mime@^1": version "1.3.2" @@ -10476,10 +10476,10 @@ map-obj@^4.0.0: version "4.2.1" resolved "https://registry.npmjs.org/map-obj/-/map-obj-4.2.1.tgz" -marked@^4.0.17: - version "4.0.17" - resolved "https://registry.yarnpkg.com/marked/-/marked-4.0.17.tgz#1186193d85bb7882159cdcfc57d1dfccaffb3fe9" - integrity sha512-Wfk0ATOK5iPxM4ptrORkFemqroz0ZDxp5MWfYA7H/F+wO17NRWV5Ypxi6p3g2Xmw2bKeiYOl6oVnLHKxBA0VhA== +marked@^2.0.0: + version "2.1.3" + resolved "https://registry.yarnpkg.com/marked/-/marked-2.1.3.tgz#bd017cef6431724fd4b27e0657f5ceb14bff3753" + integrity sha512-/Q+7MGzaETqifOMWYEA7HVMaZb4XbcRfaOzcSsHZEith83KGlvaSG33u0SKu89Mj5h+T8V2hM+8O45Qc5XTgwA== marker-clusterer-plus@^2.1.4: version "2.1.4" From fa82cead9883e2a420e787406f37d1bcc185131f Mon Sep 17 00:00:00 2001 From: Anand Srinivasan <66776129+eco-monk@users.noreply.github.com> Date: Wed, 13 Jul 2022 19:24:05 +0530 Subject: [PATCH 05/12] fix: curl import error (#15178) * send only pageId on curl import creation * remove variables * fix data source redirect (cherry picked from commit 59f91b618c79e3fb8931f9a4565215a72ae3f3e6) --- .../GlobalSearch/GlobalSearchHooks.tsx | 6 +----- .../src/pages/Editor/Explorer/Files/Submenu.tsx | 15 ++------------- app/client/src/utils/AnalyticsUtil.tsx | 3 ++- 3 files changed, 5 insertions(+), 19 deletions(-) diff --git a/app/client/src/components/editorComponents/GlobalSearch/GlobalSearchHooks.tsx b/app/client/src/components/editorComponents/GlobalSearch/GlobalSearchHooks.tsx index db7fd883a8..f0591ed4d9 100644 --- a/app/client/src/components/editorComponents/GlobalSearch/GlobalSearchHooks.tsx +++ b/app/client/src/components/editorComponents/GlobalSearch/GlobalSearchHooks.tsx @@ -136,11 +136,7 @@ export const useFilteredFileOperations = (query = "") => { ), kind: SEARCH_ITEM_TYPES.actionOperation, - redirect: ( - applicationSlug: string, - pageSlug: string, - pageId: string, - ) => { + redirect: (pageId: string) => { history.push( integrationEditorURL({ pageId, diff --git a/app/client/src/pages/Editor/Explorer/Files/Submenu.tsx b/app/client/src/pages/Editor/Explorer/Files/Submenu.tsx index 10e79924f0..65622f75cb 100644 --- a/app/client/src/pages/Editor/Explorer/Files/Submenu.tsx +++ b/app/client/src/pages/Editor/Explorer/Files/Submenu.tsx @@ -8,11 +8,7 @@ import { import styled from "constants/DefaultTheme"; import React, { useCallback, useEffect, useMemo, useState } from "react"; import { useDispatch, useSelector } from "react-redux"; -import { - getCurrentPageId, - selectCurrentApplicationSlug, - selectPageSlugToIdMap, -} from "selectors/editorSelectors"; +import { getCurrentPageId } from "selectors/editorSelectors"; import EntityAddButton from "../Entity/AddButton"; import { ReactComponent as SearchIcon } from "assets/icons/ads/search.svg"; import { ReactComponent as CrossIcon } from "assets/icons/ads/cross.svg"; @@ -66,8 +62,6 @@ export default function ExplorerSubMenu({ const [show, setShow] = useState(openMenu); const fileOperations = useFilteredFileOperations(query); const pageId = useSelector(getCurrentPageId); - const applicationSlug = useSelector(selectCurrentApplicationSlug); - const pageIdToSlugMap = useSelector(selectPageSlugToIdMap); const dispatch = useDispatch(); const plugins = useSelector((state: AppState) => { return state.entities.plugins.list; @@ -128,12 +122,7 @@ export default function ExplorerSubMenu({ if (item.action) { dispatch(item.action(pageId, "SUBMENU")); } else if (item.redirect) { - item.redirect( - applicationSlug, - pageIdToSlugMap[pageId], - pageId, - "SUBMENU", - ); + item.redirect(pageId, "SUBMENU"); } setShow(false); }, diff --git a/app/client/src/utils/AnalyticsUtil.tsx b/app/client/src/utils/AnalyticsUtil.tsx index e0e16ea857..903d95afd4 100644 --- a/app/client/src/utils/AnalyticsUtil.tsx +++ b/app/client/src/utils/AnalyticsUtil.tsx @@ -22,7 +22,8 @@ export type EventLocation = | "QUERY_PANE" | "QUERY_TEMPLATE" | "QUICK_COMMANDS" - | "OMNIBAR"; + | "OMNIBAR" + | "SUBMENU"; export type EventName = | "APP_CRASH" From 64c693b9cdc52082cf7ba6f2215b427cf2dcd231 Mon Sep 17 00:00:00 2001 From: balajisoundar Date: Wed, 20 Jul 2022 11:42:59 +0530 Subject: [PATCH 06/12] fix: cell background for edit actions column in Table v2 (#15233) (cherry picked from commit 71e581d69b54db140e8cd47e3f41b1675ca97b99) --- .../cypress/fixtures/tableV2NewDsl.json | 1 + .../Widgets/TableV2/TableV2_Color_spec.js | 19 ++++++++- .../src/components/ads/DraggableList.tsx | 41 +++++++++++-------- .../PrimaryColumnsControlV2.tsx | 2 +- app/client/src/sagas/WidgetAdditionSagas.ts | 12 +++++- .../cellComponents/EditActionsCell.tsx | 2 + .../propertyConfig/PanelConfig/Styles.ts | 9 +--- 7 files changed, 57 insertions(+), 29 deletions(-) diff --git a/app/client/cypress/fixtures/tableV2NewDsl.json b/app/client/cypress/fixtures/tableV2NewDsl.json index d4e9e78e72..f19b88f4aa 100644 --- a/app/client/cypress/fixtures/tableV2NewDsl.json +++ b/app/client/cypress/fixtures/tableV2NewDsl.json @@ -21,6 +21,7 @@ "children": [ { "widgetName": "Table1", + "inlineEditingSaveOption": "ROW_LEVEL", "columnOrder": [ "id", "email", diff --git a/app/client/cypress/integration/Smoke_TestSuite/ClientSideTests/Widgets/TableV2/TableV2_Color_spec.js b/app/client/cypress/integration/Smoke_TestSuite/ClientSideTests/Widgets/TableV2/TableV2_Color_spec.js index 2a61efeeb3..e36dd0db3f 100644 --- a/app/client/cypress/integration/Smoke_TestSuite/ClientSideTests/Widgets/TableV2/TableV2_Color_spec.js +++ b/app/client/cypress/integration/Smoke_TestSuite/ClientSideTests/Widgets/TableV2/TableV2_Color_spec.js @@ -1,9 +1,12 @@ +const ObjectsRegistry = require("../../../../../support/Objects/Registry") + .ObjectsRegistry; +let propPane = ObjectsRegistry.PropertyPane; const widgetsPage = require("../../../../../locators/Widgets.json"); const dsl = require("../../../../../fixtures/tableV2NewDsl.json"); const publish = require("../../../../../locators/publishWidgetspage.json"); describe("Table Widget V2 property pane feature validation", function() { - before(() => { + beforeEach(() => { cy.addDsl(dsl); }); @@ -65,4 +68,18 @@ describe("Table Widget V2 property pane feature validation", function() { ); cy.get(publish.backToEditor).click(); }); + + it("2. check background of the edit action column", function() { + cy.openPropertyPane("tablewidgetv2"); + cy.makeColumnEditable("id"); + cy.readTableV2dataValidateCSS(0, 5, "background-color", "rgba(0, 0, 0, 0)"); + cy.get(".t--property-control-cellbackgroundcolor") + .find(".t--js-toggle") + .click(); + propPane.UpdatePropertyFieldValue( + "Cell Background Color", + "rgb(255, 0, 0)", + ); + cy.readTableV2dataValidateCSS(0, 5, "background-color", "rgb(255, 0, 0)"); + }); }); diff --git a/app/client/src/components/ads/DraggableList.tsx b/app/client/src/components/ads/DraggableList.tsx index 34a0ca3516..577a3075ac 100644 --- a/app/client/src/components/ads/DraggableList.tsx +++ b/app/client/src/components/ads/DraggableList.tsx @@ -102,26 +102,31 @@ function DraggableList(props: any) { }, [items]); useEffect(() => { - if (focusedIndex && listRef && listRef.current) { - const container = listRef.current; + /* + * we need to wait for the ui to get rendered before scrolling to the focusedColumn + */ + requestAnimationFrame(() => { + if (focusedIndex && listRef && listRef.current) { + const container = listRef.current; - if (focusedIndex * itemHeight < container.scrollTop) { - listRef.current.scrollTo({ - top: (focusedIndex - 1) * itemHeight, - left: 0, - behavior: "smooth", - }); - } else if ( - (focusedIndex + 1) * itemHeight > - listRef.current.scrollTop + listRef.current.clientHeight - ) { - listRef.current.scrollTo({ - top: (focusedIndex + 1) * itemHeight - listRef.current.clientHeight, - left: 0, - behavior: "smooth", - }); + if (focusedIndex * itemHeight < container.scrollTop) { + listRef.current.scrollTo({ + top: (focusedIndex - 1) * itemHeight, + left: 0, + behavior: "smooth", + }); + } else if ( + (focusedIndex + 1) * itemHeight > + listRef.current.scrollTop + listRef.current.clientHeight + ) { + listRef.current.scrollTo({ + top: (focusedIndex + 1) * itemHeight - listRef.current.clientHeight, + left: 0, + behavior: "smooth", + }); + } } - } + }); }, [focusedIndex]); const [springs, setSprings] = useSprings( diff --git a/app/client/src/components/propertyControls/PrimaryColumnsControlV2.tsx b/app/client/src/components/propertyControls/PrimaryColumnsControlV2.tsx index df60c0ce9b..6e43d08045 100644 --- a/app/client/src/components/propertyControls/PrimaryColumnsControlV2.tsx +++ b/app/client/src/components/propertyControls/PrimaryColumnsControlV2.tsx @@ -94,7 +94,7 @@ type State = { hasScrollableList: boolean; }; -const LIST_CLASSNAME = "tablewidget-primarycolumn-list"; +const LIST_CLASSNAME = "tablewidgetv2-primarycolumn-list"; class PrimaryColumnsControlV2 extends BaseControl { constructor(props: ControlProps) { super(props); diff --git a/app/client/src/sagas/WidgetAdditionSagas.ts b/app/client/src/sagas/WidgetAdditionSagas.ts index a455ba85a7..3db53eb9f1 100644 --- a/app/client/src/sagas/WidgetAdditionSagas.ts +++ b/app/client/src/sagas/WidgetAdditionSagas.ts @@ -69,11 +69,21 @@ function* getEntityNames() { * @returns */ function* getThemeDefaultConfig(type: string) { + const fallbackStylesheet: Record = { + TABLE_WIDGET_V2: "TABLE_WIDGET", + }; + const stylesheet: Record = yield select( getSelectedAppThemeStylesheet, ); - return stylesheet[type] || themePropertiesDefaults; + if (stylesheet[type]) { + return stylesheet[type]; + } else if (fallbackStylesheet[type] && stylesheet[fallbackStylesheet[type]]) { + return stylesheet[fallbackStylesheet[type]]; + } else { + return themePropertiesDefaults; + } } function* getChildWidgetProps( diff --git a/app/client/src/widgets/TableWidgetV2/component/cellComponents/EditActionsCell.tsx b/app/client/src/widgets/TableWidgetV2/component/cellComponents/EditActionsCell.tsx index d606a4a8dd..e9f0782c42 100644 --- a/app/client/src/widgets/TableWidgetV2/component/cellComponents/EditActionsCell.tsx +++ b/app/client/src/widgets/TableWidgetV2/component/cellComponents/EditActionsCell.tsx @@ -58,6 +58,8 @@ export function EditActionCell(props: RenderEditActionsProps) { return ( Date: Wed, 20 Jul 2022 12:28:50 +0530 Subject: [PATCH 07/12] feat: Update the table widget to v2 in suggested widget list (#15277) Co-authored-by: balajisoundar (cherry picked from commit dbf0b94973a40103eacdf7e6867fc6bae164e750) --- .../cypress/fixtures/addWidgetTable-mock.json | 2 +- .../Bind_JSObject_Postgress_Table_spec.js | 4 ++-- .../QueryPane/AddWidgetTableAndBind_spec.js | 6 +++--- .../ServerSideTests/QueryPane/AddWidget_spec.js | 2 +- .../ServerSideTests/QueryPane/S3_spec.js | 2 +- app/client/cypress/locators/QueryEditor.json | 2 +- .../widget/propertyConfig/General.ts | 2 +- .../com/appsmith/external/models/WidgetType.java | 2 +- .../server/helpers/WidgetSuggestionHelper.java | 6 +++--- .../server/services/ce/ActionServiceCE_Test.java | 16 ++++++++-------- 10 files changed, 22 insertions(+), 22 deletions(-) diff --git a/app/client/cypress/fixtures/addWidgetTable-mock.json b/app/client/cypress/fixtures/addWidgetTable-mock.json index 7668852c78..38241194e1 100644 --- a/app/client/cypress/fixtures/addWidgetTable-mock.json +++ b/app/client/cypress/fixtures/addWidgetTable-mock.json @@ -45,7 +45,7 @@ ], "suggestedWidgets": [ { - "type": "TABLE_WIDGET", + "type": "TABLE_WIDGET_V2", "bindingQuery": "data" } ] diff --git a/app/client/cypress/integration/Smoke_TestSuite/ClientSideTests/Binding/Bind_JSObject_Postgress_Table_spec.js b/app/client/cypress/integration/Smoke_TestSuite/ClientSideTests/Binding/Bind_JSObject_Postgress_Table_spec.js index 4ec54dfb01..a8ea2ace1b 100644 --- a/app/client/cypress/integration/Smoke_TestSuite/ClientSideTests/Binding/Bind_JSObject_Postgress_Table_spec.js +++ b/app/client/cypress/integration/Smoke_TestSuite/ClientSideTests/Binding/Bind_JSObject_Postgress_Table_spec.js @@ -39,7 +39,7 @@ describe("Addwidget from Query and bind with other widgets", function() { .click({ force: true }); cy.testJsontext("tabledata", "{{JSObject1.myFun1()}}"); cy.isSelectRow(1); - cy.readTabledataPublish("1", "0").then((tabData) => { + cy.readTableV2dataPublish("1", "0").then((tabData) => { let tabValue = tabData; cy.log("the value is" + tabValue); expect(tabValue).to.be.equal("5"); @@ -86,7 +86,7 @@ describe("Addwidget from Query and bind with other widgets", function() { ).then(() => cy.wait(500)); cy.isSelectRow(1); - cy.readTabledataPublish("1", "0").then((tabData) => { + cy.readTableV2dataPublish("1", "0").then((tabData) => { let tabValue = tabData; cy.log("Value in public viewing: " + tabValue); expect(tabValue).to.be.equal("5"); diff --git a/app/client/cypress/integration/Smoke_TestSuite/ServerSideTests/QueryPane/AddWidgetTableAndBind_spec.js b/app/client/cypress/integration/Smoke_TestSuite/ServerSideTests/QueryPane/AddWidgetTableAndBind_spec.js index a5b64a4ea6..fe8e8ec3a0 100644 --- a/app/client/cypress/integration/Smoke_TestSuite/ServerSideTests/QueryPane/AddWidgetTableAndBind_spec.js +++ b/app/client/cypress/integration/Smoke_TestSuite/ServerSideTests/QueryPane/AddWidgetTableAndBind_spec.js @@ -46,7 +46,7 @@ describe("Addwidget from Query and bind with other widgets", function() { cy.get(queryEditor.suggestedTableWidget).click(); cy.SearchEntityandOpen("Table1"); cy.isSelectRow(1); - cy.readTabledataPublish("1", "0").then((tabData) => { + cy.readTableV2dataPublish("1", "0").then((tabData) => { const tabValue = tabData; cy.log("the value is" + tabValue); expect(tabValue).to.be.equal("5"); @@ -70,7 +70,7 @@ describe("Addwidget from Query and bind with other widgets", function() { it("4. validation of data displayed in input widget based on row data selected", function() { cy.isSelectRow(1); - cy.readTabledataPublish("1", "0").then((tabData) => { + cy.readTableV2dataPublish("1", "0").then((tabData) => { const tabValue = tabData; cy.log("the value is" + tabValue); expect(tabValue).to.be.equal("5"); @@ -83,7 +83,7 @@ describe("Addwidget from Query and bind with other widgets", function() { }); it("5. Input widget test with default value from table widget[Bug#4136]", () => { - cy.openPropertyPane("tablewidget"); + cy.openPropertyPane("tablewidgetv2"); cy.get(".t--property-pane-title").click({ force: true }); cy.get(".t--property-pane-title") .type("TableUpdated", { delay: 300 }) diff --git a/app/client/cypress/integration/Smoke_TestSuite/ServerSideTests/QueryPane/AddWidget_spec.js b/app/client/cypress/integration/Smoke_TestSuite/ServerSideTests/QueryPane/AddWidget_spec.js index 7c82c45422..121135d114 100644 --- a/app/client/cypress/integration/Smoke_TestSuite/ServerSideTests/QueryPane/AddWidget_spec.js +++ b/app/client/cypress/integration/Smoke_TestSuite/ServerSideTests/QueryPane/AddWidget_spec.js @@ -26,7 +26,7 @@ describe("Add widget - Postgress DataSource", function() { cy.CheckAndUnfoldEntityItem("WIDGETS"); cy.selectEntityByName("Table1"); cy.isSelectRow(1); - cy.readTabledataPublish("1", "0").then((tabData) => { + cy.readTableV2dataPublish("1", "0").then((tabData) => { cy.log("the value is " + tabData); expect(tabData).to.be.equal("5"); }); diff --git a/app/client/cypress/integration/Smoke_TestSuite/ServerSideTests/QueryPane/S3_spec.js b/app/client/cypress/integration/Smoke_TestSuite/ServerSideTests/QueryPane/S3_spec.js index 9ae09057ad..69f6bf961e 100644 --- a/app/client/cypress/integration/Smoke_TestSuite/ServerSideTests/QueryPane/S3_spec.js +++ b/app/client/cypress/integration/Smoke_TestSuite/ServerSideTests/QueryPane/S3_spec.js @@ -679,7 +679,7 @@ describe("Validate CRUD queries for Amazon S3 along with UI flow verifications", cy.get(queryLocators.suggestedTableWidget) .click() .wait(1000); - cy.get(commonlocators.TableRow).validateWidgetExists(); + cy.get(commonlocators.TableV2Row).validateWidgetExists(); cy.get("@entity").then((entityN) => cy.selectEntityByName(entityN)); cy.xpath(queryLocators.suggestedWidgetText) diff --git a/app/client/cypress/locators/QueryEditor.json b/app/client/cypress/locators/QueryEditor.json index 8b44bc351d..43cd511a53 100644 --- a/app/client/cypress/locators/QueryEditor.json +++ b/app/client/cypress/locators/QueryEditor.json @@ -15,7 +15,7 @@ "settings": "li:contains('Settings')", "query": "li:contains('Query')", "switch": ".t--form-control-SWITCH input", - "suggestedTableWidget": ".t--suggested-widget-TABLE_WIDGET", + "suggestedTableWidget": ".t--suggested-widget-TABLE_WIDGET_V2", "queryResponse": "(//div[@class='table']//div[@class='tr'])[3]//div[@class='td']", "querySelect": "//div[contains(@class, 't--template-menu')]//div[text()='Select']", "queryCreate": "//div[contains(@class, 't--template-menu')]//div[text()='Create']", diff --git a/app/client/src/widgets/TableWidgetV2/widget/propertyConfig/General.ts b/app/client/src/widgets/TableWidgetV2/widget/propertyConfig/General.ts index 44db00f120..7d7e42fe73 100644 --- a/app/client/src/widgets/TableWidgetV2/widget/propertyConfig/General.ts +++ b/app/client/src/widgets/TableWidgetV2/widget/propertyConfig/General.ts @@ -71,7 +71,7 @@ export default { }, { propertyName: "inlineEditingSaveOption", - helpText: "choose the save experience to save the edited cell", + helpText: "Choose the save experience to save the edited cell", label: "Update Mode", controlType: "DROP_DOWN", isBindProperty: true, diff --git a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/WidgetType.java b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/WidgetType.java index 3cdade7ff8..c33116aa44 100644 --- a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/WidgetType.java +++ b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/WidgetType.java @@ -7,7 +7,7 @@ public enum WidgetType { TEXT_WIDGET("data"), SELECT_WIDGET("data.map( (obj) =>{ return {'label': obj.%s, 'value': obj.%s } })"), CHART_WIDGET("data.map( (obj) =>{ return {'x': obj.%s, 'y': obj.%s } })"), - TABLE_WIDGET("data"), + TABLE_WIDGET_V2("data"), INPUT_WIDGET("data"); public final String query; diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/WidgetSuggestionHelper.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/WidgetSuggestionHelper.java index 0872ebfbaf..97acb907f8 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/WidgetSuggestionHelper.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/WidgetSuggestionHelper.java @@ -131,7 +131,7 @@ public class WidgetSuggestionHelper { } return getWidgetsForTypeArray(fields, numericFields); } - return List.of(getWidget(WidgetType.TABLE_WIDGET), getWidget(WidgetType.TEXT_WIDGET)); + return List.of(getWidget(WidgetType.TABLE_WIDGET_V2), getWidget(WidgetType.TEXT_WIDGET)); } /* @@ -186,7 +186,7 @@ public class WidgetSuggestionHelper { widgetTypeList.add(getWidget(WidgetType.CHART_WIDGET, fields.get(0), numericFields.get(0))); } } - widgetTypeList.add(getWidget(WidgetType.TABLE_WIDGET)); + widgetTypeList.add(getWidget(WidgetType.TABLE_WIDGET_V2)); widgetTypeList.add(getWidget(WidgetType.TEXT_WIDGET)); return widgetTypeList; } @@ -222,7 +222,7 @@ public class WidgetSuggestionHelper { widgetTypeList.add(getWidgetNestedData(WidgetType.CHART_WIDGET, nestedFieldName, fields.get(0), numericFields.get(0))); } } - widgetTypeList.add(getWidgetNestedData(WidgetType.TABLE_WIDGET, nestedFieldName)); + widgetTypeList.add(getWidgetNestedData(WidgetType.TABLE_WIDGET_V2, nestedFieldName)); widgetTypeList.add(getWidgetNestedData(WidgetType.TEXT_WIDGET, nestedFieldName)); return widgetTypeList; } diff --git a/app/server/appsmith-server/src/test/java/com/appsmith/server/services/ce/ActionServiceCE_Test.java b/app/server/appsmith-server/src/test/java/com/appsmith/server/services/ce/ActionServiceCE_Test.java index 9871e9e6e0..b2a7f321a7 100644 --- a/app/server/appsmith-server/src/test/java/com/appsmith/server/services/ce/ActionServiceCE_Test.java +++ b/app/server/appsmith-server/src/test/java/com/appsmith/server/services/ce/ActionServiceCE_Test.java @@ -1656,7 +1656,7 @@ public class ActionServiceCE_Test { List widgetTypeList = new ArrayList<>(); widgetTypeList.add(WidgetSuggestionHelper.getWidget(WidgetType.CHART_WIDGET, "x", "y")); widgetTypeList.add(WidgetSuggestionHelper.getWidget(WidgetType.SELECT_WIDGET, "x", "x")); - widgetTypeList.add(WidgetSuggestionHelper.getWidget(WidgetType.TABLE_WIDGET)); + widgetTypeList.add(WidgetSuggestionHelper.getWidget(WidgetType.TABLE_WIDGET_V2)); widgetTypeList.add(WidgetSuggestionHelper.getWidget(WidgetType.TEXT_WIDGET)); mockResult.setSuggestedWidgets(widgetTypeList); @@ -1769,7 +1769,7 @@ public class ActionServiceCE_Test { List widgetTypeList = new ArrayList<>(); widgetTypeList.add(WidgetSuggestionHelper.getWidget(WidgetType.CHART_WIDGET, "id", "ppu")); widgetTypeList.add(WidgetSuggestionHelper.getWidget(WidgetType.SELECT_WIDGET, "id", "type")); - widgetTypeList.add(WidgetSuggestionHelper.getWidget(WidgetType.TABLE_WIDGET)); + widgetTypeList.add(WidgetSuggestionHelper.getWidget(WidgetType.TABLE_WIDGET_V2)); widgetTypeList.add(WidgetSuggestionHelper.getWidget(WidgetType.TEXT_WIDGET)); mockResult.setSuggestedWidgets(widgetTypeList); @@ -1874,7 +1874,7 @@ public class ActionServiceCE_Test { List widgetTypeList = new ArrayList<>(); widgetTypeList.add(WidgetSuggestionHelper.getWidget(WidgetType.CHART_WIDGET, "url", "width")); widgetTypeList.add(WidgetSuggestionHelper.getWidget(WidgetType.SELECT_WIDGET, "url", "url")); - widgetTypeList.add(WidgetSuggestionHelper.getWidget(WidgetType.TABLE_WIDGET)); + widgetTypeList.add(WidgetSuggestionHelper.getWidget(WidgetType.TABLE_WIDGET_V2)); widgetTypeList.add(WidgetSuggestionHelper.getWidget(WidgetType.TEXT_WIDGET)); mockResult.setSuggestedWidgets(widgetTypeList); @@ -1934,7 +1934,7 @@ public class ActionServiceCE_Test { List widgetTypeList = new ArrayList<>(); widgetTypeList.add(WidgetSuggestionHelper.getWidget(WidgetType.SELECT_WIDGET, "CarType", "carID")); - widgetTypeList.add(WidgetSuggestionHelper.getWidget(WidgetType.TABLE_WIDGET)); + widgetTypeList.add(WidgetSuggestionHelper.getWidget(WidgetType.TABLE_WIDGET_V2)); widgetTypeList.add(WidgetSuggestionHelper.getWidget(WidgetType.TEXT_WIDGET)); mockResult.setSuggestedWidgets(widgetTypeList); @@ -2019,7 +2019,7 @@ public class ActionServiceCE_Test { mockResult.setDataTypes(List.of(new ParsedDataType(DisplayDataType.RAW))); List widgetTypeList = new ArrayList<>(); - widgetTypeList.add(WidgetSuggestionHelper.getWidget(WidgetType.TABLE_WIDGET)); + widgetTypeList.add(WidgetSuggestionHelper.getWidget(WidgetType.TABLE_WIDGET_V2)); widgetTypeList.add(WidgetSuggestionHelper.getWidget(WidgetType.TEXT_WIDGET)); mockResult.setSuggestedWidgets(widgetTypeList); @@ -2172,7 +2172,7 @@ public class ActionServiceCE_Test { List widgetTypeList = new ArrayList<>(); widgetTypeList.add(WidgetSuggestionHelper.getWidgetNestedData(WidgetType.TEXT_WIDGET,"users")); widgetTypeList.add(WidgetSuggestionHelper.getWidgetNestedData(WidgetType.CHART_WIDGET,"users","name","id")); - widgetTypeList.add(WidgetSuggestionHelper.getWidgetNestedData(WidgetType.TABLE_WIDGET,"users")); + widgetTypeList.add(WidgetSuggestionHelper.getWidgetNestedData(WidgetType.TABLE_WIDGET_V2,"users")); widgetTypeList.add(WidgetSuggestionHelper.getWidgetNestedData(WidgetType.SELECT_WIDGET,"users","name", "status")); mockResult.setSuggestedWidgets(widgetTypeList); @@ -2379,7 +2379,7 @@ public class ActionServiceCE_Test { List widgetTypeList = new ArrayList<>(); widgetTypeList.add(WidgetSuggestionHelper.getWidget(WidgetType.CHART_WIDGET, "url", "width")); widgetTypeList.add(WidgetSuggestionHelper.getWidget(WidgetType.SELECT_WIDGET, "url", "url")); - widgetTypeList.add(WidgetSuggestionHelper.getWidget(WidgetType.TABLE_WIDGET)); + widgetTypeList.add(WidgetSuggestionHelper.getWidget(WidgetType.TABLE_WIDGET_V2)); widgetTypeList.add(WidgetSuggestionHelper.getWidget(WidgetType.TEXT_WIDGET)); mockResult.setSuggestedWidgets(widgetTypeList); @@ -2427,7 +2427,7 @@ public class ActionServiceCE_Test { List widgetTypeList = new ArrayList<>(); widgetTypeList.add(WidgetSuggestionHelper.getWidget(WidgetType.SELECT_WIDGET, "url", "width")); - widgetTypeList.add(WidgetSuggestionHelper.getWidget(WidgetType.TABLE_WIDGET)); + widgetTypeList.add(WidgetSuggestionHelper.getWidget(WidgetType.TABLE_WIDGET_V2)); widgetTypeList.add(WidgetSuggestionHelper.getWidget(WidgetType.TEXT_WIDGET)); mockResult.setSuggestedWidgets(widgetTypeList); From b20a025a368784dc378f36459e74fcb82914baef Mon Sep 17 00:00:00 2001 From: Shrikant Sharat Kandula Date: Wed, 20 Jul 2022 12:48:55 +0530 Subject: [PATCH 08/12] Fix method name for Redis cleaning migration (#15322) (cherry picked from commit e8be0936e651ee19c0bb054306be5cd63991628e) --- .../com/appsmith/server/migrations/DatabaseChangelog2.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/DatabaseChangelog2.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/DatabaseChangelog2.java index 854f89874a..8d650aa748 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/DatabaseChangelog2.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/DatabaseChangelog2.java @@ -1369,8 +1369,8 @@ public class DatabaseChangelog2 { return newWhereClause; } - @ChangeSet(order = "021", id = "flush-spring-redis-keys-2", author = "") - public void migrateGoogleSheetsToUqi(ReactiveRedisOperations reactiveRedisOperations) { + @ChangeSet(order = "021", id = "flush-spring-redis-keys-2a", author = "") + public void clearRedisCache2(ReactiveRedisOperations reactiveRedisOperations) { DatabaseChangelog.doClearRedisKeys(reactiveRedisOperations); } From 806d144b7a6d2b286ec42f7882be17f96cbddc56 Mon Sep 17 00:00:00 2001 From: Shrikant Sharat Kandula Date: Wed, 20 Jul 2022 16:09:09 +0530 Subject: [PATCH 09/12] fix: Block the call to clear Redis sessions (#15330) (cherry picked from commit 5d166a46f1fc1c260e7956cd784ee3a52e92dbad) --- .../java/com/appsmith/server/migrations/DatabaseChangelog.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/DatabaseChangelog.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/DatabaseChangelog.java index 17eb214235..bcd72167bf 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/DatabaseChangelog.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/DatabaseChangelog.java @@ -4469,7 +4469,7 @@ public class DatabaseChangelog { "end"; final Flux flushdb = reactiveRedisOperations.execute(RedisScript.of(script)); - flushdb.subscribe(); + flushdb.blockLast(); } /* Map values from pluginSpecifiedTemplates to formData (UQI) */ From 505d3a331dc345532d97fcdf48d4476d895be2bc Mon Sep 17 00:00:00 2001 From: Anagh Hegde Date: Wed, 27 Jul 2022 17:14:17 +0530 Subject: [PATCH 10/12] chore: NPE issue while copying in git-theming migration (#15480) ## Description > Fix the NPE issue in theming while copying the properties. ## Type of change - Bug fix (non-breaking change which fixes an issue) ## How Has This Been Tested? > Locally ## Checklist: - [ ] My code follows the style guidelines of this project - [ ] I have performed a self-review of my own code - [ ] I have commented my code, particularly in hard-to-understand areas - [ ] I have made corresponding changes to the documentation - [ ] My changes generate no new warnings - [ ] I have added tests that prove my fix is effective or that my feature works - [ ] New and existing unit tests pass locally with my changes --- .../com/appsmith/server/migrations/DatabaseChangelog2.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/DatabaseChangelog2.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/DatabaseChangelog2.java index cf5ed7b019..a694ab9e72 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/DatabaseChangelog2.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/DatabaseChangelog2.java @@ -71,6 +71,7 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; +import static com.appsmith.external.helpers.AppsmithBeanUtils.copyNestedNonNullProperties; import static com.appsmith.external.helpers.AppsmithBeanUtils.copyNewFieldValuesIntoOldObject; import static com.appsmith.server.migrations.DatabaseChangelog.dropIndexIfExists; import static com.appsmith.server.migrations.DatabaseChangelog.ensureIndexes; @@ -1439,7 +1440,7 @@ public class DatabaseChangelog2 { Theme theme = mongockTemplate.findOne(themeQuery, Theme.class); for (Application application : applicationList) { Theme newTheme = new Theme(); - copyNewFieldValuesIntoOldObject(theme, newTheme); + copyNestedNonNullProperties(theme, newTheme); newTheme.setId(null); newTheme.setSystemTheme(false); newTheme = mongockTemplate.insert(newTheme); From fcaa6ae650342e8586055ef620789bb9f65df8dd Mon Sep 17 00:00:00 2001 From: Arpit Mohan Date: Sat, 6 Aug 2022 09:06:43 +0200 Subject: [PATCH 11/12] fix: Adding a check for invalid hosts on redirects as well (#15782) ## Description Fixes issue for checking for invalid hosts even when there are redirects in the Rest API plugin. ## Type of change - Bug fix (non-breaking change which fixes an issue) ## How Has This Been Tested? - Junit test ## Checklist: - [ ] My code follows the style guidelines of this project - [ ] I have performed a self-review of my own code - [ ] I have commented my code, particularly in hard-to-understand areas - [ ] I have made corresponding changes to the documentation - [ ] My changes generate no new warnings - [ ] I have added tests that prove my fix is effective or that my feature works - [ ] New and existing unit tests pass locally with my changes (cherry picked from commit 0967b516b69a6e9438a19f8f6953cf1d7d2d9aa6) --- .../com/external/plugins/RestApiPlugin.java | 6 +++- .../external/plugins/RestApiPluginTest.java | 34 +++++++++++++++++-- 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/app/server/appsmith-plugins/restApiPlugin/src/main/java/com/external/plugins/RestApiPlugin.java b/app/server/appsmith-plugins/restApiPlugin/src/main/java/com/external/plugins/RestApiPlugin.java index 4f8bf13801..8b4a8de6d0 100644 --- a/app/server/appsmith-plugins/restApiPlugin/src/main/java/com/external/plugins/RestApiPlugin.java +++ b/app/server/appsmith-plugins/restApiPlugin/src/main/java/com/external/plugins/RestApiPlugin.java @@ -585,7 +585,11 @@ public class RestApiPlugin extends BasePlugin { URI redirectUri = null; try { redirectUri = new URI(redirectUrl); - } catch (URISyntaxException e) { + if (DISALLOWED_HOSTS.contains(redirectUri.getHost()) + || DISALLOWED_HOSTS.contains(InetAddress.getByName(redirectUri.getHost()).getHostAddress())) { + return Mono.error(new AppsmithPluginException(AppsmithPluginError.PLUGIN_EXECUTE_ARGUMENT_ERROR, "Host not allowed.")); + } + } catch (URISyntaxException | UnknownHostException e) { return Mono.error(new AppsmithPluginException(AppsmithPluginError.PLUGIN_ERROR, e)); } return httpCall(webClient, httpMethod, redirectUri, finalRequestBody, iteration + 1, diff --git a/app/server/appsmith-plugins/restApiPlugin/src/test/java/com/external/plugins/RestApiPluginTest.java b/app/server/appsmith-plugins/restApiPlugin/src/test/java/com/external/plugins/RestApiPluginTest.java index cbf1801519..b7b30a0cd9 100644 --- a/app/server/appsmith-plugins/restApiPlugin/src/test/java/com/external/plugins/RestApiPluginTest.java +++ b/app/server/appsmith-plugins/restApiPlugin/src/test/java/com/external/plugins/RestApiPluginTest.java @@ -25,6 +25,9 @@ import io.jsonwebtoken.security.SignatureException; import net.minidev.json.JSONObject; import net.minidev.json.parser.JSONParser; import net.minidev.json.parser.ParseException; +import okhttp3.HttpUrl; +import okhttp3.mockwebserver.MockResponse; +import okhttp3.mockwebserver.MockWebServer; import org.junit.Before; import org.junit.Test; import org.springframework.http.HttpHeaders; @@ -34,6 +37,7 @@ import reactor.test.StepVerifier; import reactor.util.function.Tuple2; import javax.crypto.SecretKey; +import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HashMap; @@ -641,7 +645,7 @@ public class RestApiPluginTest { Map> duplicateHeadersWithDsConfigOnly = getAllDuplicateHeaders(null, dsConfig); // Header duplicates - Set expectedDuplicateHeaders = new HashSet<>(); + Set expectedDuplicateHeaders = new HashSet<>(); expectedDuplicateHeaders.add("myHeader1"); expectedDuplicateHeaders.add("myHeader2"); assertTrue(expectedDuplicateHeaders.equals(duplicateHeadersWithDsConfigOnly.get(DATASOURCE_CONFIG_ONLY))); @@ -651,7 +655,7 @@ public class RestApiPluginTest { dsConfig); // Query param duplicates - Set expectedDuplicateParams = new HashSet<>(); + Set expectedDuplicateParams = new HashSet<>(); expectedDuplicateParams.add("myParam1"); expectedDuplicateParams.add("myParam2"); assertTrue(expectedDuplicateParams.equals(duplicateParamsWithDsConfigOnly.get(DATASOURCE_CONFIG_ONLY))); @@ -1004,6 +1008,32 @@ public class RestApiPluginTest { .verifyComplete(); } + @Test + public void testDenyInstanceMetadataAwsWithRedirect() throws IOException { + // Generate a mock response which redirects to the invalid host + MockWebServer mockWebServer = new MockWebServer(); + MockResponse mockRedirectResponse = new MockResponse() + .setResponseCode(301) + .addHeader("Location", "http://169.254.169.254.nip.io/latest/meta-data"); + mockWebServer.enqueue(mockRedirectResponse); + mockWebServer.start(); + + HttpUrl mockHttpUrl = mockWebServer.url("/mock/redirect"); + DatasourceConfiguration dsConfig = new DatasourceConfiguration(); + dsConfig.setUrl(mockHttpUrl.toString()); + + ActionConfiguration actionConfig = new ActionConfiguration(); + actionConfig.setHttpMethod(HttpMethod.GET); + + Mono resultMono = pluginExecutor.executeParameterized(null, new ExecuteActionDTO(), dsConfig, actionConfig); + StepVerifier.create(resultMono) + .assertNext(result -> { + assertFalse(result.getIsExecutionSuccess()); + assertEquals("Host not allowed.", result.getBody()); + }) + .verifyComplete(); + } + @Test public void testGetApiWithBody() { DatasourceConfiguration dsConfig = new DatasourceConfiguration(); From 0b347b0a5b300ec0a999f0003174a9000a8c1878 Mon Sep 17 00:00:00 2001 From: Shrikant Sharat Kandula Date: Mon, 8 Aug 2022 21:07:15 +0530 Subject: [PATCH 12/12] fix: Adding checks to prevent disallowed hosts from connecting via Elasticsearch plugin (#15834) ## Description This PR fixes an issue where a potentially malicious user can connect to disallowed hosts from the Elasticsearch plugin within Appsmith. This is because Elasticsearch client SDK is a HTTP interface underneath the hood. ## Type of change - Bug fix (non-breaking change which fixes an issue) ## How Has This Been Tested? - Junits for the following: - create datasource with disallowed host - validate datasource with disallowed host - test datasource with disallowed host ## Checklist: - [x] My code follows the style guidelines of this project - [x] I have performed a self-review of my own code - [x] I have commented my code, particularly in hard-to-understand areas - [ ] I have made corresponding changes to the documentation - [x] My changes generate no new warnings - [x] I have added tests that prove my fix is effective or that my feature works - [x] New and existing unit tests pass locally with my changes (cherry picked from commit c1dbca67796f635711ff84ea9f8b4365039efa39) Signed-off-by: Shrikant Sharat Kandula --- .../external/plugins/ElasticSearchPlugin.java | 234 +++++++++++------- .../plugins/ElasticSearchPluginTest.java | 188 ++++++++++++-- 2 files changed, 315 insertions(+), 107 deletions(-) diff --git a/app/server/appsmith-plugins/elasticSearchPlugin/src/main/java/com/external/plugins/ElasticSearchPlugin.java b/app/server/appsmith-plugins/elasticSearchPlugin/src/main/java/com/external/plugins/ElasticSearchPlugin.java index 06ccc82594..777ec85944 100644 --- a/app/server/appsmith-plugins/elasticSearchPlugin/src/main/java/com/external/plugins/ElasticSearchPlugin.java +++ b/app/server/appsmith-plugins/elasticSearchPlugin/src/main/java/com/external/plugins/ElasticSearchPlugin.java @@ -37,14 +37,17 @@ import reactor.core.scheduler.Scheduler; import reactor.core.scheduler.Schedulers; import java.io.IOException; +import java.net.InetAddress; import java.net.MalformedURLException; import java.net.URL; +import java.net.UnknownHostException; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.regex.Pattern; import static com.appsmith.external.constants.ActionConstants.ACTION_CONFIGURATION_BODY; import static com.appsmith.external.constants.ActionConstants.ACTION_CONFIGURATION_PATH; @@ -59,7 +62,20 @@ public class ElasticSearchPlugin extends BasePlugin { @Extension public static class ElasticSearchPluginExecutor implements PluginExecutor { - private final Scheduler scheduler = Schedulers.elastic(); + private final Scheduler scheduler = Schedulers.boundedElastic(); + + public static final String esDatasourceNotFoundMessage = "Either your host URL is invalid or the page you are trying to access does not exist"; + + public static final String esDatasourceUnauthorizedMessage = "Your username or password is not correct"; + + public static final String esDatasourceUnauthorizedPattern = ".*unauthorized.*"; + + public static final String esDatasourceNotFoundPattern = ".*(?:not.?found)|(?:refused)|(?:not.?known)|(?:timed?\\s?out).*"; + + private static final Set DISALLOWED_HOSTS = Set.of( + "169.254.169.254", + "metadata.google.internal" + ); @Override public Mono execute(RestClient client, @@ -72,64 +88,64 @@ public class ElasticSearchPlugin extends BasePlugin { List requestParams = new ArrayList<>(); return Mono.fromCallable(() -> { - final ActionExecutionResult result = new ActionExecutionResult(); + final ActionExecutionResult result = new ActionExecutionResult(); - String body = query; + String body = query; - final String path = actionConfiguration.getPath(); - requestData.put("path", path); + final String path = actionConfiguration.getPath(); + requestData.put("path", path); - HttpMethod httpMethod = actionConfiguration.getHttpMethod(); - requestData.put("method", httpMethod.name()); - requestParams.add(new RequestParamDTO("actionConfiguration.httpMethod", httpMethod.name(), null, - null, null)); - requestParams.add(new RequestParamDTO(ACTION_CONFIGURATION_PATH, path, null, null, null)); - requestParams.add(new RequestParamDTO(ACTION_CONFIGURATION_BODY, query, null, null, null)); + HttpMethod httpMethod = actionConfiguration.getHttpMethod(); + requestData.put("method", httpMethod.name()); + requestParams.add(new RequestParamDTO("actionConfiguration.httpMethod", httpMethod.name(), null, + null, null)); + requestParams.add(new RequestParamDTO(ACTION_CONFIGURATION_PATH, path, null, null, null)); + requestParams.add(new RequestParamDTO(ACTION_CONFIGURATION_BODY, query, null, null, null)); - final Request request = new Request(httpMethod.toString(), path); - ContentType contentType = ContentType.APPLICATION_JSON; + final Request request = new Request(httpMethod.toString(), path); + ContentType contentType = ContentType.APPLICATION_JSON; - if (isBulkQuery(path)) { - contentType = ContentType.create("application/x-ndjson"); + if (isBulkQuery(path)) { + contentType = ContentType.create("application/x-ndjson"); - // If body is a JSON Array, convert it to an ND-JSON string. - if (body != null && body.trim().startsWith("[")) { - final StringBuilder ndJsonBuilder = new StringBuilder(); - try { - List commands = objectMapper.readValue(body, ArrayList.class); - for (Object object : commands) { - ndJsonBuilder.append(objectMapper.writeValueAsString(object)).append("\n"); + // If body is a JSON Array, convert it to an ND-JSON string. + if (body != null && body.trim().startsWith("[")) { + final StringBuilder ndJsonBuilder = new StringBuilder(); + try { + List commands = objectMapper.readValue(body, ArrayList.class); + for (Object object : commands) { + ndJsonBuilder.append(objectMapper.writeValueAsString(object)).append("\n"); + } + } catch (IOException e) { + final String message = "Error converting array to ND-JSON: " + e.getMessage(); + log.warn(message, e); + return Mono.error(new AppsmithPluginException(AppsmithPluginError.PLUGIN_EXECUTE_ARGUMENT_ERROR, message)); + } + body = ndJsonBuilder.toString(); } - } catch (IOException e) { - final String message = "Error converting array to ND-JSON: " + e.getMessage(); - log.warn(message, e); - return Mono.error(new AppsmithPluginException(AppsmithPluginError.PLUGIN_EXECUTE_ARGUMENT_ERROR, message)); } - body = ndJsonBuilder.toString(); - } - } - if (body != null) { - request.setEntity(new NStringEntity(body, contentType)); - } + if (body != null) { + request.setEntity(new NStringEntity(body, contentType)); + } - try { - final String responseBody = new String( - client.performRequest(request).getEntity().getContent().readAllBytes()); - result.setBody(objectMapper.readValue(responseBody, HashMap.class)); - } catch (IOException e) { - final String message = "Error performing request: " + e.getMessage(); - log.warn(message, e); - return Mono.error(new AppsmithPluginException(AppsmithPluginError.PLUGIN_ERROR, message)); - } + try { + final String responseBody = new String( + client.performRequest(request).getEntity().getContent().readAllBytes()); + result.setBody(objectMapper.readValue(responseBody, HashMap.class)); + } catch (IOException e) { + final String message = "Error performing request: " + e.getMessage(); + log.warn(message, e); + return Mono.error(new AppsmithPluginException(AppsmithPluginError.PLUGIN_ERROR, message)); + } - result.setIsExecutionSuccess(true); - log.debug("In the Elastic Search Plugin, got action execution result"); - return Mono.just(result); - }) + result.setIsExecutionSuccess(true); + log.debug("In the Elastic Search Plugin, got action execution result"); + return Mono.just(result); + }) .flatMap(obj -> obj) .map(obj -> (ActionExecutionResult) obj) - .onErrorResume(error -> { + .onErrorResume(error -> { ActionExecutionResult result = new ActionExecutionResult(); result.setIsExecutionSuccess(false); result.setErrorInfo(error); @@ -155,55 +171,60 @@ public class ElasticSearchPlugin extends BasePlugin { @Override public Mono datasourceCreate(DatasourceConfiguration datasourceConfiguration) { - return (Mono) Mono.fromCallable(() -> { - final List hosts = new ArrayList<>(); + final List hosts = new ArrayList<>(); - for (Endpoint endpoint : datasourceConfiguration.getEndpoints()) { - URL url; - try { - url = new URL(endpoint.getHost()); - } catch (MalformedURLException e) { - return Mono.error(new AppsmithPluginException(AppsmithPluginError.PLUGIN_DATASOURCE_ARGUMENT_ERROR, - "Invalid host provided. It should be of the form http(s)://your-es-url.com")); + for (Endpoint endpoint : datasourceConfiguration.getEndpoints()) { + URL url; + try { + url = new URL(endpoint.getHost()); + if (DISALLOWED_HOSTS.contains(url.getHost()) + || DISALLOWED_HOSTS.contains(InetAddress.getByName(url.getHost()).getHostAddress())) { + return Mono.error(new AppsmithPluginException(AppsmithPluginError.PLUGIN_EXECUTE_ARGUMENT_ERROR, "Invalid host provided.")); } - String scheme = "http"; - if (url.getProtocol() != null) { - scheme = url.getProtocol(); - } - - hosts.add(new HttpHost(url.getHost(), endpoint.getPort().intValue(), scheme)); + } catch (MalformedURLException e) { + return Mono.error(new AppsmithPluginException(AppsmithPluginError.PLUGIN_DATASOURCE_ARGUMENT_ERROR, + "Invalid host provided. It should be of the form http(s)://your-es-url.com")); + } catch (UnknownHostException e) { + return Mono.error(new AppsmithPluginException(AppsmithPluginError.PLUGIN_DATASOURCE_ARGUMENT_ERROR, + esDatasourceNotFoundMessage)); + } + String scheme = "http"; + if (url.getProtocol() != null) { + scheme = url.getProtocol(); } - final RestClientBuilder clientBuilder = RestClient.builder(hosts.toArray(new HttpHost[]{})); + hosts.add(new HttpHost(url.getHost(), endpoint.getPort().intValue(), scheme)); + } - final DBAuth authentication = (DBAuth) datasourceConfiguration.getAuthentication(); - if (authentication != null - && !StringUtils.isEmpty(authentication.getUsername()) - && !StringUtils.isEmpty(authentication.getPassword())) { - final CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); - credentialsProvider.setCredentials( - AuthScope.ANY, - new UsernamePasswordCredentials(authentication.getUsername(), authentication.getPassword()) - ); + final RestClientBuilder clientBuilder = RestClient.builder(hosts.toArray(new HttpHost[]{})); - clientBuilder - .setHttpClientConfigCallback( - httpClientBuilder -> httpClientBuilder - .setDefaultCredentialsProvider(credentialsProvider) - ); - } + final DBAuth authentication = (DBAuth) datasourceConfiguration.getAuthentication(); + if (authentication != null + && !StringUtils.isEmpty(authentication.getUsername()) + && !StringUtils.isEmpty(authentication.getPassword())) { + final CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials( + AuthScope.ANY, + new UsernamePasswordCredentials(authentication.getUsername(), authentication.getPassword()) + ); - if (!CollectionUtils.isEmpty(datasourceConfiguration.getHeaders())) { - clientBuilder.setDefaultHeaders( - (Header[]) datasourceConfiguration.getHeaders() - .stream() - .map(h -> new BasicHeader(h.getKey(), (String) h.getValue())) - .toArray() - ); - } + clientBuilder + .setHttpClientConfigCallback( + httpClientBuilder -> httpClientBuilder + .setDefaultCredentialsProvider(credentialsProvider) + ); + } - return Mono.just(clientBuilder.build()); - }) + if (!CollectionUtils.isEmpty(datasourceConfiguration.getHeaders())) { + clientBuilder.setDefaultHeaders( + (Header[]) datasourceConfiguration.getHeaders() + .stream() + .map(h -> new BasicHeader(h.getKey(), (String) h.getValue())) + .toArray() + ); + } + + return Mono.fromCallable(() -> Mono.just(clientBuilder.build())) .flatMap(obj -> obj) .subscribeOn(scheduler); } @@ -224,14 +245,21 @@ public class ElasticSearchPlugin extends BasePlugin { if (CollectionUtils.isEmpty(datasourceConfiguration.getEndpoints())) { invalids.add("No endpoint provided. Please provide a host:port where ElasticSearch is reachable."); } else { - for(Endpoint endpoint : datasourceConfiguration.getEndpoints()) { + for (Endpoint endpoint : datasourceConfiguration.getEndpoints()) { + if (endpoint.getHost() == null) { invalids.add("Missing host for endpoint"); } else { try { URL url = new URL(endpoint.getHost()); + if (DISALLOWED_HOSTS.contains(url.getHost()) + || DISALLOWED_HOSTS.contains(InetAddress.getByName(url.getHost()).getHostAddress())) { + invalids.add("Invalid host provided."); + } } catch (MalformedURLException e) { invalids.add("Invalid host provided. It should be of the form http(s)://your-es-url.com"); + } catch (UnknownHostException e) { + invalids.add(esDatasourceNotFoundMessage); } } @@ -239,6 +267,7 @@ public class ElasticSearchPlugin extends BasePlugin { invalids.add("Missing port for endpoint"); } } + } return invalids; @@ -251,16 +280,32 @@ public class ElasticSearchPlugin extends BasePlugin { if (client == null) { return new DatasourceTestResult("Null client object to ElasticSearch."); } - - // This HEAD request is to check if an index exists. It response with 200 if the index exists, + // This HEAD request is to check if the base of datasource exists. It responds with 200 if the index exists, // 404 if it doesn't. We just check for either of these two. // Ref: https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-exists.html - Request request = new Request("HEAD", "/potentially-missing-index?local=true"); + Request request = new Request("HEAD", "/"); final Response response; try { response = client.performRequest(request); } catch (IOException e) { + + /* since the 401, and 403 are registered as IOException, but for the given connection it + * in the current rest-client. We will figure out with matching patterns with regexes. + */ + + Pattern patternForUnauthorized = Pattern.compile(esDatasourceUnauthorizedPattern, Pattern.CASE_INSENSITIVE); + Pattern patternForNotFound = Pattern.compile(esDatasourceNotFoundPattern, Pattern.CASE_INSENSITIVE); + + if (patternForUnauthorized.matcher(e.getMessage()).find()) { + return new DatasourceTestResult(esDatasourceUnauthorizedMessage); + } + + if (patternForNotFound.matcher(e.getMessage()).find()) { + return new DatasourceTestResult(esDatasourceNotFoundMessage); + } + + return new DatasourceTestResult("Error running HEAD request: " + e.getMessage()); } @@ -271,8 +316,13 @@ public class ElasticSearchPlugin extends BasePlugin { } catch (IOException e) { log.warn("Error closing ElasticSearch client that was made for testing.", e); } + // earlier it was 404 and 200, now it has been changed to just expect 200 status code + // here it checks if it is anything else than 200, even 404 is not allowed! + if (statusLine.getStatusCode() == 404) { + return new DatasourceTestResult(esDatasourceNotFoundMessage); + } - if (statusLine.getStatusCode() != 404 && statusLine.getStatusCode() != 200) { + if (statusLine.getStatusCode() != 200) { return new DatasourceTestResult( "Unexpected response from ElasticSearch: " + statusLine); } diff --git a/app/server/appsmith-plugins/elasticSearchPlugin/src/test/java/com/external/plugins/ElasticSearchPluginTest.java b/app/server/appsmith-plugins/elasticSearchPlugin/src/test/java/com/external/plugins/ElasticSearchPluginTest.java index a2dadc3552..60bdda079e 100755 --- a/app/server/appsmith-plugins/elasticSearchPlugin/src/test/java/com/external/plugins/ElasticSearchPluginTest.java +++ b/app/server/appsmith-plugins/elasticSearchPlugin/src/test/java/com/external/plugins/ElasticSearchPluginTest.java @@ -1,21 +1,29 @@ package com.external.plugins; +import com.appsmith.external.constants.Authentication; +import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginException; import com.appsmith.external.models.ActionConfiguration; import com.appsmith.external.models.ActionExecutionResult; +import com.appsmith.external.models.DBAuth; import com.appsmith.external.models.DatasourceConfiguration; import com.appsmith.external.models.Endpoint; import com.appsmith.external.models.RequestParamDTO; import lombok.extern.slf4j.Slf4j; import org.apache.http.HttpHost; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.impl.nio.client.HttpAsyncClientBuilder; import org.elasticsearch.client.Request; import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestClientBuilder; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.Test; import org.springframework.http.HttpMethod; import org.testcontainers.elasticsearch.ElasticsearchContainer; -import org.testcontainers.utility.DockerImageName; import reactor.core.publisher.Mono; import reactor.test.StepVerifier; @@ -39,20 +47,36 @@ public class ElasticSearchPluginTest { @ClassRule public static final ElasticsearchContainer container = new ElasticsearchContainer("docker.elastic.co/elasticsearch/elasticsearch:7.12.1") - .withEnv("discovery.type", "single-node"); - + .withEnv("discovery.type", "single-node") + .withPassword("esPassword"); + private static String username = "elastic"; + private static String password = "esPassword"; private static final DatasourceConfiguration dsConfig = new DatasourceConfiguration(); + private static DBAuth elasticInstanceCredentials = new DBAuth(DBAuth.Type.USERNAME_PASSWORD, username, password, null); private static String host; private static Integer port; + @BeforeClass public static void setUp() throws IOException { port = container.getMappedPort(9200); host = "http://" + container.getContainerIpAddress(); - final RestClient client = RestClient.builder( - new HttpHost(container.getContainerIpAddress(), port, "http") - ).build(); + final CredentialsProvider credentialsProvider = + new BasicCredentialsProvider(); + credentialsProvider.setCredentials(AuthScope.ANY, + new UsernamePasswordCredentials(username, password)); + + RestClient client = RestClient.builder( + new HttpHost(container.getContainerIpAddress(), port, "http")) + .setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() { + @Override + public HttpAsyncClientBuilder customizeHttpClient( + HttpAsyncClientBuilder httpClientBuilder) { + return httpClientBuilder + .setDefaultCredentialsProvider(credentialsProvider); + } + }).build(); Request request; @@ -69,8 +93,11 @@ public class ElasticSearchPluginTest { client.performRequest(request); client.close(); - + elasticInstanceCredentials.setAuthenticationType(Authentication.BASIC); + elasticInstanceCredentials.setUsername(username); + elasticInstanceCredentials.setPassword(password); dsConfig.setEndpoints(List.of(new Endpoint(host, port.longValue()))); + dsConfig.setAuthentication(elasticInstanceCredentials); } private Mono execute(HttpMethod method, String path, String body) { @@ -130,7 +157,7 @@ public class ElasticSearchPluginTest { expectedRequestParams.add(new RequestParamDTO("actionConfiguration.httpMethod", HttpMethod.GET.toString(), null, null, null)); expectedRequestParams.add(new RequestParamDTO(ACTION_CONFIGURATION_PATH, "/planets/_mget", null, null, null)); - expectedRequestParams.add(new RequestParamDTO(ACTION_CONFIGURATION_BODY, contentJson, null, null, null)); + expectedRequestParams.add(new RequestParamDTO(ACTION_CONFIGURATION_BODY, contentJson, null, null, null)); assertEquals(result.getRequest().getRequestParams().toString(), expectedRequestParams.toString()); }) .verifyComplete(); @@ -208,12 +235,12 @@ public class ElasticSearchPluginTest { public void testBulkWithDirectBody() { final String contentJson = "{ \"index\" : { \"_index\" : \"test2\", \"_type\": \"doc\", \"_id\" : \"1\" } }\n" + - "{ \"field1\" : \"value1\" }\n" + - "{ \"delete\" : { \"_index\" : \"test2\", \"_type\": \"doc\", \"_id\" : \"2\" } }\n" + - "{ \"create\" : { \"_index\" : \"test2\", \"_type\": \"doc\", \"_id\" : \"3\" } }\n" + - "{ \"field1\" : \"value3\" }\n" + - "{ \"update\" : {\"_id\" : \"1\", \"_type\": \"doc\", \"_index\" : \"test2\"} }\n" + - "{ \"doc\" : {\"field2\" : \"value2\"} }\n"; + "{ \"field1\" : \"value1\" }\n" + + "{ \"delete\" : { \"_index\" : \"test2\", \"_type\": \"doc\", \"_id\" : \"2\" } }\n" + + "{ \"create\" : { \"_index\" : \"test2\", \"_type\": \"doc\", \"_id\" : \"3\" } }\n" + + "{ \"field1\" : \"value3\" }\n" + + "{ \"update\" : {\"_id\" : \"1\", \"_type\": \"doc\", \"_index\" : \"test2\"} }\n" + + "{ \"doc\" : {\"field2\" : \"value2\"} }\n"; StepVerifier.create(execute(HttpMethod.POST, "/_bulk", contentJson)) .assertNext(result -> { @@ -230,6 +257,7 @@ public class ElasticSearchPluginTest { @Test public void itShouldValidateDatasourceWithNoEndpoints() { DatasourceConfiguration invalidDatasourceConfiguration = new DatasourceConfiguration(); + invalidDatasourceConfiguration.setAuthentication(elasticInstanceCredentials); Assert.assertEquals(Set.of("No endpoint provided. Please provide a host:port where ElasticSearch is reachable."), pluginExecutor.validateDatasource(invalidDatasourceConfiguration)); @@ -238,6 +266,7 @@ public class ElasticSearchPluginTest { @Test public void itShouldValidateDatasourceWithEmptyPort() { DatasourceConfiguration datasourceConfiguration = new DatasourceConfiguration(); + datasourceConfiguration.setAuthentication(elasticInstanceCredentials); Endpoint endpoint = new Endpoint(); endpoint.setHost(host); datasourceConfiguration.setEndpoints(Collections.singletonList(endpoint)); @@ -249,6 +278,7 @@ public class ElasticSearchPluginTest { @Test public void itShouldValidateDatasourceWithEmptyHost() { DatasourceConfiguration datasourceConfiguration = new DatasourceConfiguration(); + datasourceConfiguration.setAuthentication(elasticInstanceCredentials); Endpoint endpoint = new Endpoint(); endpoint.setPort(Long.valueOf(port)); datasourceConfiguration.setEndpoints(Collections.singletonList(endpoint)); @@ -260,7 +290,7 @@ public class ElasticSearchPluginTest { @Test public void itShouldValidateDatasourceWithMissingEndpoint() { DatasourceConfiguration datasourceConfiguration = new DatasourceConfiguration(); - + datasourceConfiguration.setAuthentication(elasticInstanceCredentials); Endpoint endpoint = new Endpoint(); datasourceConfiguration.setEndpoints(Collections.singletonList(endpoint)); @@ -272,6 +302,7 @@ public class ElasticSearchPluginTest { public void itShouldValidateDatasourceWithEndpointNoProtocol() { DatasourceConfiguration datasourceConfiguration = new DatasourceConfiguration(); Endpoint endpoint = new Endpoint(); + datasourceConfiguration.setAuthentication(elasticInstanceCredentials); endpoint.setHost("localhost"); endpoint.setPort(Long.valueOf(port)); datasourceConfiguration.setEndpoints(Collections.singletonList(endpoint)); @@ -284,6 +315,7 @@ public class ElasticSearchPluginTest { @Test public void itShouldTestDatasourceWithInvalidEndpoint() { DatasourceConfiguration datasourceConfiguration = new DatasourceConfiguration(); + datasourceConfiguration.setAuthentication(elasticInstanceCredentials); Endpoint endpoint = new Endpoint(); endpoint.setHost("localhost"); endpoint.setPort(Long.valueOf(port)); @@ -304,4 +336,130 @@ public class ElasticSearchPluginTest { }) .verifyComplete(); } + + @Test + public void shouldVerifyUnauthorized() { + final Integer secureHostPort = container.getMappedPort(9200); + final String secureHostEndpoint = "http://" + container.getHttpHostAddress(); + DatasourceConfiguration datasourceConfiguration = new DatasourceConfiguration(); + Endpoint endpoint = new Endpoint(secureHostEndpoint, Long.valueOf(secureHostPort)); + datasourceConfiguration.setEndpoints(Collections.singletonList(endpoint)); + + + StepVerifier.create(pluginExecutor.testDatasource(datasourceConfiguration) + .map(result -> { + return (Set) result.getInvalids(); + })) + .expectNext(Set.of(ElasticSearchPlugin.ElasticSearchPluginExecutor.esDatasourceUnauthorizedMessage)) + .verifyComplete(); + + } + + + @Test + public void shouldVerifyNotFound() { + final Integer secureHostPort = container.getMappedPort(9200); + final String secureHostEndpoint = "http://esdatabasenotfound.co"; + DatasourceConfiguration datasourceConfiguration = new DatasourceConfiguration(); + Endpoint endpoint = new Endpoint(secureHostEndpoint, Long.valueOf(secureHostPort)); + datasourceConfiguration.setEndpoints(Collections.singletonList(endpoint)); + + StepVerifier.create(pluginExecutor.testDatasource(datasourceConfiguration) + .map(result -> { + return (Set) result.getInvalids(); + })) + .expectNext(Set.of(ElasticSearchPlugin.ElasticSearchPluginExecutor.esDatasourceNotFoundMessage)) + .verifyComplete(); + + } + + @Test + public void itShouldDenyTestDatasourceWithInstanceMetadataAws() { + DatasourceConfiguration datasourceConfiguration = new DatasourceConfiguration(); + datasourceConfiguration.setAuthentication(elasticInstanceCredentials); + Endpoint endpoint = new Endpoint(); + endpoint.setHost("http://169.254.169.254"); + endpoint.setPort(Long.valueOf(port)); + datasourceConfiguration.setEndpoints(Collections.singletonList(endpoint)); + + StepVerifier.create(pluginExecutor.testDatasource(datasourceConfiguration)) + .assertNext(result -> { + assertFalse(result.getInvalids().isEmpty()); + assertTrue(result.getInvalids().contains("Invalid host provided.")); + }) + .verifyComplete(); + } + + @Test + public void itShouldDenyTestDatasourceWithInstanceMetadataGcp() { + DatasourceConfiguration datasourceConfiguration = new DatasourceConfiguration(); + datasourceConfiguration.setAuthentication(elasticInstanceCredentials); + Endpoint endpoint = new Endpoint(); + endpoint.setHost("http://metadata.google.internal"); + endpoint.setPort(Long.valueOf(port)); + datasourceConfiguration.setEndpoints(Collections.singletonList(endpoint)); + + StepVerifier.create(pluginExecutor.testDatasource(datasourceConfiguration)) + .assertNext(result -> { + assertFalse(result.getInvalids().isEmpty()); + assertTrue(result.getInvalids().contains("Invalid host provided.")); + }) + .verifyComplete(); + } + + @Test + public void itShouldDenyCreateDatasourceWithInstanceMetadataAws() { + DatasourceConfiguration datasourceConfiguration = new DatasourceConfiguration(); + datasourceConfiguration.setAuthentication(elasticInstanceCredentials); + Endpoint endpoint = new Endpoint(); + endpoint.setHost("http://169.254.169.254"); + endpoint.setPort(Long.valueOf(port)); + datasourceConfiguration.setEndpoints(Collections.singletonList(endpoint)); + + StepVerifier.create(pluginExecutor.datasourceCreate(datasourceConfiguration)) + .verifyErrorSatisfies(e -> { + assertTrue(e instanceof AppsmithPluginException); + assertEquals("Invalid host provided.", e.getMessage()); + }); + } + + @Test + public void itShouldDenyCreateDatasourceWithInstanceMetadataGcp() { + DatasourceConfiguration datasourceConfiguration = new DatasourceConfiguration(); + datasourceConfiguration.setAuthentication(elasticInstanceCredentials); + Endpoint endpoint = new Endpoint(); + endpoint.setHost("https://metadata.google.internal"); + endpoint.setPort(Long.valueOf(port)); + datasourceConfiguration.setEndpoints(Collections.singletonList(endpoint)); + + StepVerifier.create(pluginExecutor.datasourceCreate(datasourceConfiguration)) + .verifyErrorSatisfies(e -> { + assertTrue(e instanceof AppsmithPluginException); + assertEquals("Invalid host provided.", e.getMessage()); + }); + } + + @Test + public void itShouldValidateDatasourceWithInstanceMetadataAws() { + DatasourceConfiguration datasourceConfiguration = new DatasourceConfiguration(); + datasourceConfiguration.setAuthentication(elasticInstanceCredentials); + Endpoint endpoint = new Endpoint(); + endpoint.setHost("http://169.254.169.254"); + endpoint.setPort(Long.valueOf(port)); + datasourceConfiguration.setEndpoints(Collections.singletonList(endpoint)); + + Assert.assertEquals(Set.of("Invalid host provided."), pluginExecutor.validateDatasource(datasourceConfiguration)); + } + + @Test + public void itShouldValidateDatasourceWithInstanceMetadataGcp() { + DatasourceConfiguration datasourceConfiguration = new DatasourceConfiguration(); + datasourceConfiguration.setAuthentication(elasticInstanceCredentials); + Endpoint endpoint = new Endpoint(); + endpoint.setHost("https://metadata.google.internal"); + endpoint.setPort(Long.valueOf(port)); + datasourceConfiguration.setEndpoints(Collections.singletonList(endpoint)); + + Assert.assertEquals(Set.of("Invalid host provided."), pluginExecutor.validateDatasource(datasourceConfiguration)); + } }