From 70df93a37cfafec720be011b5c31d7788bf55ec5 Mon Sep 17 00:00:00 2001 From: Manish Kumar <107841575+sondermanish@users.noreply.github.com> Date: Mon, 3 Jul 2023 18:36:05 +0530 Subject: [PATCH] feat: updating datasource endpoints contract (#23920) --- .../cypress/locators/DatasourcesEditor.json | 28 +- .../cypress/support/Pages/DataSources.ts | 12 +- app/client/src/api/ActionAPI.tsx | 5 +- app/client/src/api/DatasourcesApi.ts | 38 +- .../src/ce/constants/ReduxActionConstants.tsx | 1 + app/client/src/ce/sagas/ApplicationSagas.tsx | 21 +- .../src/ce/selectors/environmentSelectors.tsx | 5 + .../src/ce/utils/Environments/index.tsx | 52 +++ .../editorComponents/Debugger/index.tsx | 27 +- .../GlobalSearch/GlobalSearchHooks.test.ts | 96 +++-- .../DatasourceDropdown/useDatasource.tsx | 3 +- .../fields/EmbeddedDatasourcePathField.tsx | 92 +++-- .../src/components/formControls/utils.test.ts | 131 +++++-- app/client/src/ee/api/ApiUtils.ts | 5 + .../src/ee/selectors/environmentSelectors.tsx | 1 + .../src/ee/utils/Environments/index.tsx | 1 + app/client/src/entities/Datasource/index.ts | 21 +- .../Editor/APIEditor/ApiAuthentication.tsx | 18 +- .../pages/Editor/APIEditor/ApiRightPane.tsx | 9 +- .../pages/Editor/DataSourceEditor/DBForm.tsx | 1 + .../DataSourceEditor/DatasourceSection.tsx | 21 +- .../Editor/DataSourceEditor/JSONtoForm.tsx | 32 +- .../DataSourceEditor/NewActionButton.tsx | 8 +- .../pages/Editor/DataSourceEditor/index.tsx | 21 +- .../src/pages/Editor/Explorer/ContextMenu.tsx | 3 +- app/client/src/pages/Editor/FormControl.tsx | 29 +- .../components/GeneratePageForm/hooks.ts | 6 +- .../IntegrationEditor/DatasourceCard.tsx | 35 +- .../IntegrationEditor/mockData/index.ts | 46 ++- .../Editor/SaaSEditor/DatasourceCard.tsx | 2 + .../Editor/SaaSEditor/DatasourceForm.tsx | 22 +- .../src/pages/Editor/SaaSEditor/errorUtils.ts | 23 +- .../gitSync/ReconnectDatasourceModal.tsx | 55 ++- .../gitSync/components/DatasourceListItem.tsx | 3 +- .../src/pages/common/datasourceAuth/index.tsx | 13 +- .../entityReducers/datasourceReducer.ts | 38 ++ app/client/src/sagas/DatasourcesSagas.ts | 119 ++++-- app/client/src/sagas/ErrorSagas.tsx | 2 +- app/client/src/sagas/QueryPaneSagas.ts | 5 +- .../src/selectors/datasourceSelectors.tsx | 4 - .../RestAPIDatasourceFormTransformer.ts | 104 ++++-- app/client/src/utils/editorContextUtils.ts | 23 +- .../appsmith/external/models/Datasource.java | 40 +- .../external/models/DatasourceStorage.java | 2 + .../external/models/DatasourceStorageDTO.java | 25 ++ .../external/models/OAuth2ResponseDTO.java | 2 +- .../src/main/resources/form.json | 2 +- .../src/main/resources/form.json | 2 +- .../src/main/resources/form.json | 2 +- .../server/constants/ce/FieldNameCE.java | 1 + .../ce/ApplicationControllerCE.java | 6 +- .../ce/DatasourceControllerCE.java | 39 +- .../domains/DatasourceContextIdentifier.java | 46 ++- .../ce/DatasourceContextIdentifierCE.java | 57 --- .../helpers/DatasourceAnalyticsUtils.java | 18 +- .../services/ce/DatasourceServiceCE.java | 23 +- .../services/ce/DatasourceServiceCEImpl.java | 345 +++++++++++------- .../ce/DatasourceStorageServiceCE.java | 4 +- .../ce/DatasourceStorageServiceCEImpl.java | 33 +- .../server/services/ce/MockDataServiceCE.java | 4 +- .../services/ce/MockDataServiceCEImpl.java | 14 +- .../services/ce/NewActionServiceCEImpl.java | 9 +- .../ce/AuthenticationServiceCEImpl.java | 26 +- .../ce/CreateDBTablePageSolutionCEImpl.java | 2 +- .../ForkExamplesWorkspaceServiceCEImpl.java | 6 +- .../ce/ImportExportApplicationServiceCE.java | 3 - .../ImportExportApplicationServiceCEImpl.java | 15 +- .../DatasourceContextIdentifierTest.java | 59 +++ .../DatasourceContextServiceTest.java | 91 +++-- .../services/DatasourceServiceTest.java | 306 ++++++++++++---- .../server/services/MockDataServiceTest.java | 38 +- .../ce/ActionExecutionSolutionCETest.java | 5 +- 72 files changed, 1620 insertions(+), 786 deletions(-) create mode 100644 app/client/src/ce/selectors/environmentSelectors.tsx create mode 100644 app/client/src/ce/utils/Environments/index.tsx create mode 100644 app/client/src/ee/selectors/environmentSelectors.tsx create mode 100644 app/client/src/ee/utils/Environments/index.tsx delete mode 100644 app/client/src/selectors/datasourceSelectors.tsx delete mode 100644 app/server/appsmith-server/src/main/java/com/appsmith/server/domains/ce/DatasourceContextIdentifierCE.java create mode 100644 app/server/appsmith-server/src/test/java/com/appsmith/server/domains/DatasourceContextIdentifierTest.java diff --git a/app/client/cypress/locators/DatasourcesEditor.json b/app/client/cypress/locators/DatasourcesEditor.json index 16d7ef023c..4b755dfc6c 100644 --- a/app/client/cypress/locators/DatasourcesEditor.json +++ b/app/client/cypress/locators/DatasourcesEditor.json @@ -1,13 +1,13 @@ { "datasourceEditorIcon": ".t--entity-name:contains('DataSources)", "datasourcesCard": "t--datasource", - "host": "input[name='datasourceConfiguration.endpoints[0].host']", - "port": "input[name='datasourceConfiguration.endpoints[0].port']", - "databaseName": "input[name='datasourceConfiguration.authentication.databaseName']", - "username": "input[name='datasourceConfiguration.authentication.username']", - "password": "input[name='datasourceConfiguration.authentication.password']", + "host": "input[name$='.datasourceConfiguration.endpoints[0].host']", + "port": "input[name$='.datasourceConfiguration.endpoints[0].port']", + "databaseName": "input[name$='.datasourceConfiguration.authentication.databaseName']", + "username": "input[name$='.datasourceConfiguration.authentication.username']", + "password": "input[name$='.datasourceConfiguration.authentication.password']", "headers": "input[placeholder='Authorization Header']", - "authenticationAuthtype": "[data-testid=datasourceConfiguration\\.authentication\\.authType]", + "authenticationAuthtype": "[data-testid$=.datasourceConfiguration\\.authentication\\.authType]", "url": "input[name='url']", "MongoDB": ".t--plugin-name:contains('MongoDB')", "RESTAPI": ".t--plugin-name:contains('REST API')", @@ -29,10 +29,10 @@ "datasourceTitle": ".t--edit-datasource-name .bp3-editable-text-content", "datasourceTitleLocator": ".t--edit-datasource-name", "datasourceTitleInputLocator": ".t--edit-datasource-name input", - "defaultDatabaseName": "input[name='datasourceConfiguration.connection.defaultDatabaseName']", - "datasourceConfigurationProperty": "input[name='datasourceConfiguration.properties[0]']", + "defaultDatabaseName": "input[name$='.datasourceConfiguration.connection.defaultDatabaseName']", + "datasourceConfigurationProperty": "input[name$='.datasourceConfiguration.properties[0]']", "googleSheets": ".t--plugin-name:contains('Google Sheets')", - "selConnectionType": "[data-testid='datasourceConfiguration.connection.type']", + "selConnectionType": "[data-testid$='.datasourceConfiguration.connection.type']", "scope": "[data-testid='authentication.scopeString']", "Mysql": ".t--plugin-name:contains('Mysql')", "ElasticSearch": ".t--plugin-name:contains('Elasticsearch')", @@ -48,16 +48,16 @@ "accessTokenUrl": "[data-testid='authentication.accessTokenUrl'] input", "clienID": "[data-testid='authentication.clientId'] input", "clientSecret": "[data-testid='authentication.clientSecret'] input", - "datasourceConfigUrl": "[data-testid='datasourceConfiguration.url'] input", - "projectID": "[data-testid='datasourceConfiguration.authentication.username'] input", - "serviceAccCredential": "[data-testid='datasourceConfiguration.authentication.password'] input", + "datasourceConfigUrl": "[data-testid$='.datasourceConfiguration.url'] input", + "projectID": "[data-testid$='.datasourceConfiguration.authentication.username'] input", + "serviceAccCredential": "[data-testid$='.datasourceConfiguration.authentication.password'] input", "grantType": "[data-testid='authentication.grantType']", "authorizationURL": "[data-testid='authentication.authorizationUrl'] input", "authorizationCode": ".rc-select-item-option-content:contains('Authorization Code')", "clientCredentials": ".rc-select-item-option-content:contains('Client Credentials')", "clientAuthentication": "[data-testid='authentication.isAuthorizationHeader']", "sendClientCredentialsInBody": ".rc-select-item-option-content:contains('Send client credentials in body')", - "scopeString": "[data-testid='datasourceConfiguration.authentication.scopeString']", + "scopeString": "[data-testid$='.datasourceConfiguration.authentication.scopeString']", "GS_readFiles": "[data-testid='t--dropdown-option-Read Files']", "GS_readAndEditFiles": "[data-testid='t--dropdown-option-Read, Edit and Create Files']", "GS_readEditCreateAndDeleteFiles": "[data-testid='t--dropdown-option-Read, Edit, Create and Delete Files']", @@ -83,4 +83,4 @@ "connectionSettingsSection": "[data-testid='section-Connection'] .t--collapse-section-container", "authenticationSettingsSection": "[data-testid='section-Authentication'] .t--collapse-section-container", "sslSettingsSection": "[data-testid='section-SSL (optional)'] .t--collapse-section-container" -} +} \ No newline at end of file diff --git a/app/client/cypress/support/Pages/DataSources.ts b/app/client/cypress/support/Pages/DataSources.ts index cc1df6769d..890ec70e5e 100644 --- a/app/client/cypress/support/Pages/DataSources.ts +++ b/app/client/cypress/support/Pages/DataSources.ts @@ -53,19 +53,19 @@ export class DataSources { private _collapseContainer = ".t--collapse-section-container"; private _collapseSettings = "[data-testid='t--dropdown-connection.ssl.authType']"; - public _host = "input[name='datasourceConfiguration.endpoints[0].host']"; - public _port = "input[name='datasourceConfiguration.endpoints[0].port']"; + public _host = "input[name$='.datasourceConfiguration.endpoints[0].host']"; + public _port = "input[name$='.datasourceConfiguration.endpoints[0].port']"; _databaseName = - "input[name='datasourceConfiguration.authentication.databaseName']"; + "input[name$='.datasourceConfiguration.authentication.databaseName']"; private _username = - "input[name='datasourceConfiguration.authentication.username']"; + "input[name$='.datasourceConfiguration.authentication.username']"; private _section = (name: string) => "//div[text()='" + name + "']/parent::div"; private _sectionState = (name: string) => this._section(name) + "/following-sibling::div/div[@class ='bp3-collapse-body']"; private _password = - "input[name = 'datasourceConfiguration.authentication.password']"; + "input[name$='.datasourceConfiguration.authentication.password']"; private _testDs = ".t--test-datasource"; _saveAndAuthorizeDS = ".t--save-and-authorize-datasource"; _saveDs = ".t--save-datasource"; @@ -177,7 +177,7 @@ export class DataSources { _globalSearchInput = (inputText: string) => "//input[@id='global-search'][@value='" + inputText + "']"; _gsScopeDropdown = - "[data-testid='datasourceConfiguration.authentication.scopeString']"; + "[data-testid^='datasourceStorages.'][data-testid$='.datasourceConfiguration.authentication.scopeString']"; _gsScopeOptions = ".ads-v2-select__dropdown .rc-select-item-option"; private _queryTimeout = "//input[@name='actionConfiguration.timeoutInMillisecond']"; diff --git a/app/client/src/api/ActionAPI.tsx b/app/client/src/api/ActionAPI.tsx index 9144b29eca..6f7d9c6df0 100644 --- a/app/client/src/api/ActionAPI.tsx +++ b/app/client/src/api/ActionAPI.tsx @@ -7,6 +7,7 @@ import axios from "axios"; import type { Action, ActionViewMode } from "entities/Action"; import type { APIRequest } from "constants/AppsmithActionConstants/ActionConstants"; import type { WidgetType } from "constants/WidgetConstants"; +import { omit } from "lodash"; export interface CreateActionRequest extends APIRequest { datasourceId: string; @@ -181,9 +182,11 @@ class ActionAPI extends API { ActionAPI.apiUpdateCancelTokenSource.cancel(); } ActionAPI.apiUpdateCancelTokenSource = axios.CancelToken.source(); - const action = Object.assign({}, apiConfig); + let action = Object.assign({}, apiConfig); // While this line is not required, name can not be changed from this endpoint delete action.name; + // Removing datasource storages from the action object since embedded datasources don't have storages + action = omit(action, ["datasource.datasourceStorages"]); return API.put(`${ActionAPI.url}/${action.id}`, action, undefined, { cancelToken: ActionAPI.apiUpdateCancelTokenSource.token, }); diff --git a/app/client/src/api/DatasourcesApi.ts b/app/client/src/api/DatasourcesApi.ts index 83e9936e48..3f6e938237 100644 --- a/app/client/src/api/DatasourcesApi.ts +++ b/app/client/src/api/DatasourcesApi.ts @@ -3,16 +3,13 @@ import API from "api/Api"; import type { ApiResponse } from "./ApiResponses"; import type { AxiosPromise } from "axios"; -import type { DatasourceAuthentication, Datasource } from "entities/Datasource"; +import type { Datasource, DatasourceStorage } from "entities/Datasource"; export interface CreateDatasourceConfig { name: string; pluginId: string; type?: string; - datasourceConfiguration: { - url: string; - databaseName?: string; - authentication?: DatasourceAuthentication; - }; + // key in the map representation of environment id of type string + datasourceStorages: Record; //Passed for logging purposes. appName?: string; } @@ -47,12 +44,23 @@ class DatasourcesApi extends API { return API.post(DatasourcesApi.url, datasourceConfig); } - static testDatasource(datasourceConfig: Partial): Promise { - return API.post(`${DatasourcesApi.url}/test`, datasourceConfig, undefined, { - timeout: DEFAULT_TEST_DATA_SOURCE_TIMEOUT_MS, - }); + // Api to test current environment datasource + static testDatasource( + datasourceConfig: Partial, + pluginId: string, + workspaceId: string, + ): Promise { + return API.post( + `${DatasourcesApi.url}/test`, + { ...datasourceConfig, pluginId, workspaceId }, + undefined, + { + timeout: DEFAULT_TEST_DATA_SOURCE_TIMEOUT_MS, + }, + ); } + // Api to update datasource name. static updateDatasource( datasourceConfig: Partial, id: string, @@ -60,6 +68,16 @@ class DatasourcesApi extends API { return API.put(DatasourcesApi.url + `/${id}`, datasourceConfig); } + // Api to update specific datasource storage/environment configuration + static updateDatasourceStorage( + datasourceConfig: Partial, + ): Promise { + return API.put( + DatasourcesApi.url + `/datasource-storages`, + datasourceConfig, + ); + } + static deleteDatasource(id: string): Promise { return API.delete(DatasourcesApi.url + `/${id}`); } diff --git a/app/client/src/ce/constants/ReduxActionConstants.tsx b/app/client/src/ce/constants/ReduxActionConstants.tsx index a124d30c11..20eb9cdf2b 100644 --- a/app/client/src/ce/constants/ReduxActionConstants.tsx +++ b/app/client/src/ce/constants/ReduxActionConstants.tsx @@ -273,6 +273,7 @@ const ActionTypes = { "CREATE_TEMP_DATASOURCE_FROM_FORM_SUCCESS", UPDATE_DATASOURCE_INIT: "UPDATE_DATASOURCE_INIT", UPDATE_DATASOURCE_SUCCESS: "UPDATE_DATASOURCE_SUCCESS", + UPDATE_DATASOURCE_STORAGE_SUCCESS: "UPDATE_DATASOURCE_STORAGE_SUCCESS", CHANGE_DATASOURCE: "CHANGE_DATASOURCE", FETCH_DATASOURCE_STRUCTURE_INIT: "FETCH_DATASOURCE_STRUCTURE_INIT", ADD_AND_FETCH_MOCK_DATASOURCE_STRUCTURE_INIT: diff --git a/app/client/src/ce/sagas/ApplicationSagas.tsx b/app/client/src/ce/sagas/ApplicationSagas.tsx index e5c9d5c918..4a422685c1 100644 --- a/app/client/src/ce/sagas/ApplicationSagas.tsx +++ b/app/client/src/ce/sagas/ApplicationSagas.tsx @@ -122,6 +122,7 @@ import { keysOfNavigationSetting, } from "constants/AppConstants"; import { setAllEntityCollapsibleStates } from "../../actions/editorContextActions"; +import { getCurrentEnvironment } from "@appsmith/utils/Environments"; export const getDefaultPageId = ( pages?: ApplicationPagePayload[], @@ -851,9 +852,19 @@ export function* fetchUnconfiguredDatasourceList( } export function* initializeDatasourceWithDefaultValues(datasource: Datasource) { + let currentEnvironment = getCurrentEnvironment(); + if (!datasource.datasourceStorages.hasOwnProperty(currentEnvironment)) { + // if the currentEnvironemnt is not present for use here, take the first key from datasourceStorages + currentEnvironment = Object.keys(datasource.datasourceStorages)[0]; + } // Added isEmpty instead of ! condition as ! does not account for // datasourceConfiguration being empty - if (isEmpty(datasource.datasourceConfiguration)) { + if ( + isEmpty( + datasource.datasourceStorages[currentEnvironment] + ?.datasourceConfiguration, + ) + ) { yield call(checkAndGetPluginFormConfigsSaga, datasource.pluginId); const formConfig: Record[] = yield select( getPluginForm, @@ -863,11 +874,13 @@ export function* initializeDatasourceWithDefaultValues(datasource: Datasource) { getConfigInitialValues, formConfig, ); - const payload = merge(initialValues, datasource); + const payload = merge( + initialValues, + datasource.datasourceStorages[currentEnvironment], + ); payload.isConfigured = false; // imported datasource as not configured yet - const response: ApiResponse = yield DatasourcesApi.updateDatasource( + const response: ApiResponse = yield DatasourcesApi.updateDatasourceStorage( payload, - datasource.id, ); const isValidResponse: boolean = yield validateResponse(response); if (isValidResponse) { diff --git a/app/client/src/ce/selectors/environmentSelectors.tsx b/app/client/src/ce/selectors/environmentSelectors.tsx new file mode 100644 index 0000000000..a8ba79fee5 --- /dev/null +++ b/app/client/src/ce/selectors/environmentSelectors.tsx @@ -0,0 +1,5 @@ +import type { AppState } from "@appsmith/reducers"; + +// eslint-disable-next-line @typescript-eslint/no-unused-vars +export const areEnvironmentsFetched = (state: AppState, workspaceId: string) => + true; diff --git a/app/client/src/ce/utils/Environments/index.tsx b/app/client/src/ce/utils/Environments/index.tsx new file mode 100644 index 0000000000..f66df173f5 --- /dev/null +++ b/app/client/src/ce/utils/Environments/index.tsx @@ -0,0 +1,52 @@ +import type { Datasource } from "entities/Datasource"; + +export const ENVIRONMENT_QUERY_KEY = "environment"; +export const ENVIRONMENT_LOCAL_STORAGE_KEY = "currentEnvironment"; +export const ENVIRONMENT_ID_LOCAL_STORAGE_KEY = "currentEnvironmentId"; + +export const updateLocalStorage = (name: string, id: string) => { + // Set the values of currentEnv and currentEnvId in localStorage also + localStorage.setItem(ENVIRONMENT_LOCAL_STORAGE_KEY, name.toLowerCase()); + localStorage.setItem(ENVIRONMENT_ID_LOCAL_STORAGE_KEY, id); +}; + +// function to get the current environment from the URL +export const getCurrentEnvironment = () => { + const localStorageEnv = localStorage.getItem(ENVIRONMENT_LOCAL_STORAGE_KEY); + //compare currentEnv with local storage and get currentEnvId from localstorage if true + + if (localStorageEnv && localStorageEnv.length > 0) { + const localStorageEnvId = localStorage.getItem( + ENVIRONMENT_ID_LOCAL_STORAGE_KEY, + ); + if (!!localStorageEnvId && localStorageEnvId?.length > 0) + return localStorageEnvId; + } + return "unused_env"; +}; + +// function to check if the datasource is configured for the current environment +export const isEnvironmentConfigured = ( + datasource: Datasource | null, + environment?: string, +) => { + !environment && (environment = getCurrentEnvironment()); + const isConfigured = + !!datasource && + !!datasource.datasourceStorages && + datasource.datasourceStorages[environment]?.isConfigured; + return !!isConfigured ? isConfigured : false; +}; + +// function to check if the datasource is valid for the current environment +export const isEnvironmentValid = ( + datasource: Datasource | null, + environment?: string, +) => { + !environment && (environment = getCurrentEnvironment()); + const isValid = + datasource && + datasource.datasourceStorages && + datasource.datasourceStorages[environment]?.isValid; + return isValid ? isValid : false; +}; diff --git a/app/client/src/components/editorComponents/Debugger/index.tsx b/app/client/src/components/editorComponents/Debugger/index.tsx index 65f7736e9f..ca0e9071f0 100644 --- a/app/client/src/components/editorComponents/Debugger/index.tsx +++ b/app/client/src/components/editorComponents/Debugger/index.tsx @@ -1,7 +1,6 @@ -import React from "react"; +import React, { useEffect } from "react"; import { useDispatch, useSelector } from "react-redux"; import DebuggerTabs from "./DebuggerTabs"; -import type { AppState } from "@appsmith/reducers"; import { setDebuggerSelectedTab, setErrorCount, @@ -27,14 +26,14 @@ function Debugger() { export function DebuggerTrigger() { const dispatch = useDispatch(); - const showDebugger = useSelector( - (state: AppState) => state.ui.debugger.isOpen, - ); + const showDebugger = useSelector(showDebuggerFlag); const selectedTab = useSelector(getDebuggerSelectedTab); const messageCounters = useSelector(getMessageCount); - const totalMessageCount = messageCounters.errors + messageCounters.warnings; const hideDebuggerIcon = useSelector(hideDebuggerIconSelector); - dispatch(setErrorCount(totalMessageCount)); + + useEffect(() => { + dispatch(setErrorCount(messageCounters.errors)); + }); const onClick = (e: any) => { // If debugger is already open and selected tab is error tab then we will close debugger. @@ -58,9 +57,9 @@ export function DebuggerTrigger() { //tooltip will always show error count as we are opening error tab on click of debugger. const tooltipContent = - totalMessageCount !== 0 - ? `View details for ${totalMessageCount} ${ - totalMessageCount > 1 ? "errors" : "error" + messageCounters.errors !== 0 + ? `View details for ${messageCounters.errors} ${ + messageCounters.errors > 1 ? "errors" : "error" }` : `No errors`; @@ -70,12 +69,14 @@ export function DebuggerTrigger() { ); diff --git a/app/client/src/components/editorComponents/GlobalSearch/GlobalSearchHooks.test.ts b/app/client/src/components/editorComponents/GlobalSearch/GlobalSearchHooks.test.ts index 7bb0642f6f..94400e0501 100644 --- a/app/client/src/components/editorComponents/GlobalSearch/GlobalSearchHooks.test.ts +++ b/app/client/src/components/editorComponents/GlobalSearch/GlobalSearchHooks.test.ts @@ -56,22 +56,34 @@ describe("getFilteredAndSortedFileOperations", () => { it("shows app datasources before other datasources", () => { const appDatasource: Datasource = { - datasourceConfiguration: { - url: "", + datasourceStorages: { + unused_env: { + datasourceId: "", + environmentId: "", + datasourceConfiguration: { + url: "", + }, + isValid: true, + }, }, id: "", - isValid: true, pluginId: "", workspaceId: "", name: "App datasource", }; const otherDatasource: Datasource = { - datasourceConfiguration: { - url: "", + datasourceStorages: { + unused_env: { + datasourceId: "", + environmentId: "", + datasourceConfiguration: { + url: "", + }, + isValid: false, + }, }, id: "", - isValid: false, pluginId: "", workspaceId: "", name: "Other datasource", @@ -118,22 +130,34 @@ describe("getFilteredAndSortedFileOperations", () => { it("sorts datasources based on recency", () => { const appDatasource: Datasource = { - datasourceConfiguration: { - url: "", + datasourceStorages: { + unused_env: { + datasourceId: "", + environmentId: "", + datasourceConfiguration: { + url: "", + }, + isValid: true, + }, }, id: "123", - isValid: true, pluginId: "", workspaceId: "", name: "App datasource", }; const otherDatasource: Datasource = { - datasourceConfiguration: { - url: "", + datasourceStorages: { + unused_env: { + datasourceId: "", + environmentId: "", + datasourceConfiguration: { + url: "", + }, + isValid: false, + }, }, id: "abc", - isValid: false, pluginId: "", workspaceId: "", name: "Other datasource", @@ -180,22 +204,34 @@ describe("getFilteredAndSortedFileOperations", () => { it("filters with a query", () => { const appDatasource: Datasource = { - datasourceConfiguration: { - url: "", + datasourceStorages: { + unused_env: { + datasourceId: "", + environmentId: "", + datasourceConfiguration: { + url: "", + }, + isValid: true, + }, }, id: "", - isValid: true, pluginId: "", workspaceId: "", name: "App datasource", }; const otherDatasource: Datasource = { - datasourceConfiguration: { - url: "", + datasourceStorages: { + unused_env: { + datasourceId: "", + environmentId: "", + datasourceConfiguration: { + url: "", + }, + isValid: false, + }, }, id: "", - isValid: false, pluginId: "", workspaceId: "", name: "Other datasource", @@ -223,22 +259,34 @@ describe("getFilteredAndSortedFileOperations", () => { it("Non matching query shows on datasource creation", () => { const appDatasource: Datasource = { - datasourceConfiguration: { - url: "", + datasourceStorages: { + unused_env: { + datasourceId: "", + environmentId: "", + datasourceConfiguration: { + url: "", + }, + isValid: true, + }, }, id: "", - isValid: true, pluginId: "", workspaceId: "", name: "App datasource", }; const otherDatasource: Datasource = { - datasourceConfiguration: { - url: "", + datasourceStorages: { + unused_env: { + datasourceId: "", + environmentId: "", + datasourceConfiguration: { + url: "", + }, + isValid: false, + }, }, id: "", - isValid: false, pluginId: "", workspaceId: "", name: "Other datasource", diff --git a/app/client/src/components/editorComponents/WidgetQueryGeneratorForm/CommonControls/DatasourceDropdown/useDatasource.tsx b/app/client/src/components/editorComponents/WidgetQueryGeneratorForm/CommonControls/DatasourceDropdown/useDatasource.tsx index 3e86353f6e..2852ae15f7 100644 --- a/app/client/src/components/editorComponents/WidgetQueryGeneratorForm/CommonControls/DatasourceDropdown/useDatasource.tsx +++ b/app/client/src/components/editorComponents/WidgetQueryGeneratorForm/CommonControls/DatasourceDropdown/useDatasource.tsx @@ -35,6 +35,7 @@ import { getWidget } from "sagas/selectors"; import type { AppState } from "@appsmith/reducers"; import { DatasourceCreateEntryPoints } from "constants/Datasource"; import { getCurrentWorkspaceId } from "@appsmith/selectors/workspaceSelectors"; +import { isEnvironmentValid } from "@appsmith/utils/Environments"; import type { ActionDataState } from "reducers/entityReducers/actionsReducer"; import { getDatatype } from "utils/AppsmithUtils"; @@ -140,7 +141,7 @@ export function useDatasource(searchText: string) { value: datasource.name, data: { pluginId: datasource.pluginId, - isValid: datasource.isValid, + isValid: isEnvironmentValid(datasource), pluginPackageName: pluginsPackageNamesMap[datasource.pluginId], isSample: false, }, diff --git a/app/client/src/components/editorComponents/form/fields/EmbeddedDatasourcePathField.tsx b/app/client/src/components/editorComponents/form/fields/EmbeddedDatasourcePathField.tsx index bb83fff32d..60b0fe72de 100644 --- a/app/client/src/components/editorComponents/form/fields/EmbeddedDatasourcePathField.tsx +++ b/app/client/src/components/editorComponents/form/fields/EmbeddedDatasourcePathField.tsx @@ -7,8 +7,7 @@ import type { EditorProps } from "components/editorComponents/CodeEditor"; import { CodeEditorBorder } from "components/editorComponents/CodeEditor/EditorConfig"; import type { AppState } from "@appsmith/reducers"; import { connect } from "react-redux"; -import get from "lodash/get"; -import merge from "lodash/merge"; +import { get, merge } from "lodash"; import type { EmbeddedRestDatasource, Datasource } from "entities/Datasource"; import { DEFAULT_DATASOURCE } from "entities/Datasource"; import type CodeMirror from "codemirror"; @@ -56,13 +55,19 @@ import { TEMP_DATASOURCE_ID } from "constants/Datasource"; import LazyCodeEditor from "components/editorComponents/LazyCodeEditor"; import { getCodeMirrorNamespaceFromEditor } from "utils/getCodeMirrorNamespace"; import { isDynamicValue } from "utils/DynamicBindingUtils"; +import { + getCurrentEnvironment, + isEnvironmentValid, +} from "@appsmith/utils/Environments"; import { DEFAULT_DATASOURCE_NAME } from "constants/ApiEditorConstants/ApiEditorConstants"; import { isString } from "lodash"; type ReduxStateProps = { workspaceId: string; - datasource: Datasource | EmbeddedRestDatasource; + currentEnvironment: string; + datasource: EmbeddedRestDatasource; datasourceList: Datasource[]; + datasourceObject?: Datasource; applicationId?: string; dataTree: DataTree; actionName: string; @@ -71,7 +76,7 @@ type ReduxStateProps = { }; type ReduxDispatchProps = { - updateDatasource: (datasource: Datasource | EmbeddedRestDatasource) => void; + updateDatasource: (datasource: EmbeddedRestDatasource) => void; }; type Props = EditorProps & @@ -180,7 +185,10 @@ const StyledTooltip = styled.span<{ width?: number }>` `; //Avoiding styled components since ReactDOM.render cannot directly work with it -function CustomHint(props: { datasource: Datasource }) { +function CustomHint(props: { + currentEnvironment: string; + datasource: Datasource; +}) { return (
@@ -190,7 +198,10 @@ function CustomHint(props: { datasource: Datasource }) {
- {get(props.datasource, "datasourceConfiguration.url")} + {get( + props.datasource, + `datasourceStorages.${props.currentEnvironment}.datasourceConfiguration.url`, + )}
); @@ -245,6 +256,7 @@ class EmbeddedDatasourcePathComponent extends React.Component< }; } if (datasource && datasource.hasOwnProperty("id")) { + // We are not using datasourceStorages here since EmbeddedDatasources will not have environments const datasourceUrl = get(datasource, "datasourceConfiguration.url", ""); if (value.includes(datasourceUrl)) { return { @@ -330,7 +342,7 @@ class EmbeddedDatasourcePathComponent extends React.Component< }; handleDatasourceHint = (): HintHelper => { - const { datasourceList } = this.props; + const { currentEnvironment, datasourceList } = this.props; return () => { return { showHint: (editor: CodeMirror.Editor) => { @@ -346,19 +358,24 @@ class EmbeddedDatasourcePathComponent extends React.Component< hint: () => { const list = datasourceList .filter((datasource: Datasource) => - (datasource.datasourceConfiguration?.url || "").includes( - parsed.datasourceUrl, - ), + ( + datasource.datasourceStorages[currentEnvironment] + ?.datasourceConfiguration?.url || "" + ).includes(parsed.datasourceUrl), ) .map((datasource: Datasource) => ({ - text: datasource.datasourceConfiguration?.url, + text: datasource.datasourceStorages[currentEnvironment] + ?.datasourceConfiguration?.url, data: datasource, - className: !datasource.isValid + className: !isEnvironmentValid(datasource) ? "datasource-hint custom invalid" : "datasource-hint custom", render: (element: HTMLElement, self: any, data: any) => { ReactDOM.render( - , + , element, ); }, @@ -374,7 +391,15 @@ class EmbeddedDatasourcePathComponent extends React.Component< hints, "pick", (selected: { text: string; data: Datasource }) => { - this.props.updateDatasource(selected.data); + this.props.updateDatasource({ + ...selected.data.datasourceStorages[currentEnvironment], + id: selected.data.id, + invalids: selected.data.invalids || [], + messages: selected.data.messages || [], + pluginId: selected.data.pluginId, + name: selected.data.name, + workspaceId: selected.data.workspaceId, + }); }, ); return hints; @@ -469,6 +494,7 @@ class EmbeddedDatasourcePathComponent extends React.Component< const { codeEditorVisibleOverflow, datasource, + datasourceObject, input: { value }, userWorkspacePermissions, } = this.props; @@ -486,7 +512,7 @@ class EmbeddedDatasourcePathComponent extends React.Component< userWorkspacePermissions, ); - const datasourcePermissions = datasource?.userPermissions || []; + const datasourcePermissions = datasourceObject?.userPermissions || []; const canManageDatasource = hasManageDatasourcePermission( datasourcePermissions, @@ -524,20 +550,26 @@ class EmbeddedDatasourcePathComponent extends React.Component< evaluatedValue={this.handleEvaluatedValue()} focusElementName={`${this.props.actionName}.url`} /> - {datasource && datasource.name !== DEFAULT_DATASOURCE_NAME && ( - - - {`Datasource ${datasource?.name}`} - - - )} + {datasourceObject && + datasourceObject.name !== DEFAULT_DATASOURCE_NAME && ( + + + {`Datasource ${datasourceObject?.name}`} + + + )} {displayValue && ( { const hiddenTrueInputs: any = [ { values: { name: "Name" }, hidden: true }, { - values: { name: "Name", number: 2, email: "temp@temp.com" }, + values: { + datasourceStorages: { + unused_env: { name: "Name", number: 2, email: "temp@temp.com" }, + }, + }, hidden: { conditionType: "AND", conditions: [ { - path: "name", + path: "datasourceStorages.unused_env.name", value: "Name", comparison: "EQUALS", }, @@ -35,12 +39,12 @@ describe("isHidden test", () => { conditionType: "AND", conditions: [ { - path: "number", + path: "datasourceStorages.unused_env.number", value: 2, comparison: "EQUALS", }, { - path: "email", + path: "datasourceStorages.unused_env.email", value: "temp@temp.com", comparison: "EQUALS", }, @@ -50,17 +54,25 @@ describe("isHidden test", () => { }, }, { - values: { name: "Name" }, + values: { + datasourceStorages: { + unused_env: { name: "Name" }, + }, + }, hidden: { - path: "name", + path: "datasourceStorages.unused_env.name", value: "Name", comparison: "EQUALS", }, }, { - values: { name: "Name", config: { type: "EMAIL" } }, + values: { + datasourceStorages: { + unused_env: { name: "Name", config: { type: "EMAIL" } }, + }, + }, hidden: { - path: "name.config.type", + path: "datasourceStorages.unused_env.name.config.type", value: "USER_ID", comparison: "NOT_EQUALS", }, @@ -84,33 +96,49 @@ describe("isHidden test", () => { const hiddenFalseInputs: any = [ { values: { name: "Name" }, hidden: false }, { - values: { name: "Name" }, + values: { + datasourceStorages: { + unused_env: { name: "Name" }, + }, + }, hidden: { - path: "name", + path: "datasourceStorages.unused_env.name", value: "Different Name", comparison: "EQUALS", }, }, { - values: { name: "Name", config: { type: "EMAIL" } }, + values: { + datasourceStorages: { + unused_env: { name: "Name", config: { type: "EMAIL" } }, + }, + }, hidden: { - path: "config.type", + path: "datasourceStorages.unused_env.config.type", value: "EMAIL", comparison: "NOT_EQUALS", }, }, { - values: { name: "Name", config: { type: "Different BODY" } }, + values: { + datasourceStorages: { + unused_env: { name: "Name", config: { type: "Different BODY" } }, + }, + }, hidden: { - path: "config.type", + path: "datasourceStorages.unused_env.config.type", value: ["EMAIL", "BODY"], comparison: "IN", }, }, { - values: { name: "Name", config: { type: "BODY" } }, + values: { + datasourceStorages: { + unused_env: { name: "Name", config: { type: "BODY" } }, + }, + }, hidden: { - path: "config.type", + path: "datasourceStorages.unused_env.config.type", value: ["EMAIL", "BODY"], comparison: "NOT_IN", }, @@ -127,19 +155,27 @@ describe("isHidden test", () => { values: undefined, }, { - values: { name: "Name" }, + values: { + datasourceStorages: { + unused_env: { name: "Name" }, + }, + }, }, { values: { - name: "Name", - config: { type: "EMAIL", name: "TEMP" }, - contact: { number: 1234, address: "abcd" }, + datasourceStorages: { + unused_env: { + name: "Name", + config: { type: "EMAIL", name: "TEMP" }, + contact: { number: 1234, address: "abcd" }, + }, + }, }, hidden: { conditionType: "AND", conditions: [ { - path: "contact.number", + path: "datasourceStorages.unused_env.contact.number", value: 1234, comparison: "NOT_EQUALS", }, @@ -150,19 +186,19 @@ describe("isHidden test", () => { conditionType: "AND", conditions: [ { - path: "config.name", + path: "datasourceStorages.unused_env.config.name", value: "TEMP", comparison: "EQUALS", }, { - path: "config.name", + path: "datasourceStorages.unused_env.config.name", value: "HELLO", comparison: "EQUALS", }, ], }, { - path: "config.type", + path: "datasourceStorages.unused_env.config.type", value: "EMAIL", comparison: "NOT_EQUALS", }, @@ -190,7 +226,7 @@ describe("getConfigInitialValues test", () => { { label: "Region", configProperty: - "datasourceConfiguration.authentication.databaseName", + "datasourceStorages.unused_env.datasourceConfiguration.authentication.databaseName", controlType: "DROP_DOWN", initialValue: "ap-south-1", options: [ @@ -208,8 +244,12 @@ describe("getConfigInitialValues test", () => { }, ], output: { - datasourceConfiguration: { - authentication: { databaseName: "ap-south-1" }, + datasourceStorages: { + unused_env: { + datasourceConfiguration: { + authentication: { databaseName: "ap-south-1" }, + }, + }, }, }, }, @@ -221,7 +261,7 @@ describe("getConfigInitialValues test", () => { { label: "Region", configProperty: - "datasourceConfiguration.authentication.databaseName", + "datasourceStorages.unused_env.datasourceConfiguration.authentication.databaseName", controlType: "INPUT_TEXT", }, ], @@ -236,13 +276,15 @@ describe("getConfigInitialValues test", () => { children: [ { label: "Host address (for overriding endpoint only)", - configProperty: "datasourceConfiguration.endpoints[*].host", + configProperty: + "datasourceStorages.unused_env.datasourceConfiguration.endpoints[*].host", controlType: "KEYVALUE_ARRAY", initialValue: ["jsonplaceholder.typicode.com"], }, { label: "Port", - configProperty: "datasourceConfiguration.endpoints[*].port", + configProperty: + "datasourceStorages.unused_env.datasourceConfiguration.endpoints[*].port", dataType: "NUMBER", controlType: "KEYVALUE_ARRAY", }, @@ -250,8 +292,12 @@ describe("getConfigInitialValues test", () => { }, ], output: { - datasourceConfiguration: { - endpoints: [{ host: "jsonplaceholder.typicode.com" }], + datasourceStorages: { + unused_env: { + datasourceConfiguration: { + endpoints: [{ host: "jsonplaceholder.typicode.com" }], + }, + }, }, }, }, @@ -262,7 +308,8 @@ describe("getConfigInitialValues test", () => { children: [ { label: "Smart substitution", - configProperty: "datasourceConfiguration.isSmart", + configProperty: + "datasourceStorages.unused_env.datasourceConfiguration.isSmart", controlType: "SWITCH", initialValue: false, }, @@ -270,8 +317,12 @@ describe("getConfigInitialValues test", () => { }, ], output: { - datasourceConfiguration: { - isSmart: false, + datasourceStorages: { + unused_env: { + datasourceConfiguration: { + isSmart: false, + }, + }, }, }, }, @@ -285,15 +336,19 @@ describe("getConfigInitialValues test", () => { describe("caculateIsHidden test", () => { it("calcualte hidden field value", () => { - const values = { name: "Name" }; + const values = { + datasourceStorages: { + unused_env: { name: "Name" }, + }, + }; const hiddenTruthy: HiddenType = { - path: "name", + path: "datasourceStorages.unused_env.name", comparison: "EQUALS", value: "Name", flagValue: FEATURE_FLAG.TEST_FLAG, }; const hiddenFalsy: HiddenType = { - path: "name", + path: "datasourceStorages.unused_env.name", comparison: "EQUALS", value: "Different Name", flagValue: FEATURE_FLAG.TEST_FLAG, diff --git a/app/client/src/ee/api/ApiUtils.ts b/app/client/src/ee/api/ApiUtils.ts index 080489b3e0..565bca05b1 100644 --- a/app/client/src/ee/api/ApiUtils.ts +++ b/app/client/src/ee/api/ApiUtils.ts @@ -5,3 +5,8 @@ export const DEFAULT_ENV_ID = "unused_env"; export const getEnvironmentIdForHeader = (): string => { return DEFAULT_ENV_ID; }; + +// function to get the default environment +export const getDefaultEnvId = () => { + return DEFAULT_ENV_ID; +}; diff --git a/app/client/src/ee/selectors/environmentSelectors.tsx b/app/client/src/ee/selectors/environmentSelectors.tsx new file mode 100644 index 0000000000..d8ea701c29 --- /dev/null +++ b/app/client/src/ee/selectors/environmentSelectors.tsx @@ -0,0 +1 @@ +export * from "ce/selectors/environmentSelectors"; diff --git a/app/client/src/ee/utils/Environments/index.tsx b/app/client/src/ee/utils/Environments/index.tsx new file mode 100644 index 0000000000..e30de4b63f --- /dev/null +++ b/app/client/src/ee/utils/Environments/index.tsx @@ -0,0 +1 @@ +export * from "ce/utils/Environments"; diff --git a/app/client/src/entities/Datasource/index.ts b/app/client/src/entities/Datasource/index.ts index bd08d46c64..a7605a0347 100644 --- a/app/client/src/entities/Datasource/index.ts +++ b/app/client/src/entities/Datasource/index.ts @@ -79,8 +79,6 @@ interface BaseDatasource { name: string; type?: string; workspaceId: string; - isValid: boolean; - isConfigured?: boolean; userPermissions?: string[]; isDeleting?: boolean; isMock?: boolean; @@ -100,9 +98,11 @@ export const isEmbeddedRestDatasource = ( }; export interface EmbeddedRestDatasource extends BaseDatasource { + id?: string; datasourceConfiguration: { url: string }; invalids: Array; messages: Array; + isValid: boolean; } export interface DatasourceConfiguration { @@ -116,12 +116,21 @@ export interface DatasourceConfiguration { export interface Datasource extends BaseDatasource { id: string; - datasourceConfiguration: DatasourceConfiguration; - invalids?: string[]; - structure?: DatasourceStructure; - messages?: string[]; + // key in the map representation of environment id of type string + datasourceStorages: Record; success?: boolean; isMock?: boolean; + invalids?: string[]; + messages?: string[]; +} + +export interface DatasourceStorage { + datasourceId: string; + environmentId: string; + datasourceConfiguration: DatasourceConfiguration; + isValid: boolean; + structure?: DatasourceStructure; + isConfigured?: boolean; } export interface TokenResponse { diff --git a/app/client/src/pages/Editor/APIEditor/ApiAuthentication.tsx b/app/client/src/pages/Editor/APIEditor/ApiAuthentication.tsx index 12de117b2c..c5408f9812 100644 --- a/app/client/src/pages/Editor/APIEditor/ApiAuthentication.tsx +++ b/app/client/src/pages/Editor/APIEditor/ApiAuthentication.tsx @@ -1,5 +1,5 @@ import React from "react"; -import type { Datasource, EmbeddedRestDatasource } from "entities/Datasource"; +import type { EmbeddedRestDatasource } from "entities/Datasource"; import { get, merge } from "lodash"; import styled from "styled-components"; import { connect, useSelector } from "react-redux"; @@ -20,8 +20,9 @@ import { hasManageDatasourcePermission, } from "@appsmith/utils/permissionHelpers"; import { Icon, Text } from "design-system"; +import { getCurrentEnvironment } from "@appsmith/utils/Environments"; interface ReduxStateProps { - datasource: EmbeddedRestDatasource | Datasource; + datasource: EmbeddedRestDatasource; } const AuthContainer = styled.div` @@ -128,7 +129,8 @@ function ApiAuthentication(props: Props): JSX.Element { const mapStateToProps = (state: AppState, ownProps: any): ReduxStateProps => { const apiFormValueSelector = formValueSelector(ownProps.formName); const datasourceFromAction = apiFormValueSelector(state, "datasource"); - let datasourceMerged = datasourceFromAction; + const currentEnvironment = getCurrentEnvironment(); + let datasourceMerged: EmbeddedRestDatasource = datasourceFromAction; if (datasourceFromAction && "id" in datasourceFromAction) { const datasourceFromDataSourceList = state.entities.datasources.list.find( (d) => d.id === datasourceFromAction.id, @@ -137,8 +139,16 @@ const mapStateToProps = (state: AppState, ownProps: any): ReduxStateProps => { datasourceMerged = merge( {}, datasourceFromAction, - datasourceFromDataSourceList, + // datasourceFromDataSourceList, + datasourceFromDataSourceList.datasourceStorages[currentEnvironment], ); + + // update the id in object to datasourceId, this is because the value in id post merge is the id of the datasource storage + // and not of the datasource. + datasourceMerged.id = + datasourceFromDataSourceList.datasourceStorages[ + currentEnvironment + ].datasourceId; } } diff --git a/app/client/src/pages/Editor/APIEditor/ApiRightPane.tsx b/app/client/src/pages/Editor/APIEditor/ApiRightPane.tsx index d267a842ec..b4ca5ad9fd 100644 --- a/app/client/src/pages/Editor/APIEditor/ApiRightPane.tsx +++ b/app/client/src/pages/Editor/APIEditor/ApiRightPane.tsx @@ -15,6 +15,8 @@ import { useDispatch, useSelector } from "react-redux"; import { getApiRightPaneSelectedTab } from "selectors/apiPaneSelectors"; import isUndefined from "lodash/isUndefined"; import { Button, Tab, TabPanel, Tabs, TabsList, Tag } from "design-system"; +import type { Datasource } from "entities/Datasource"; +import { getCurrentEnvironment } from "@appsmith/utils/Environments"; const EmptyDatasourceContainer = styled.div` display: flex; @@ -245,7 +247,7 @@ function ApiRightPane(props: any) { selectedTab === API_RIGHT_PANE_TABS.DATASOURCES ? "show" : "" } > - {(sortedDatasources || []).map((d: any, idx: number) => { + {(sortedDatasources || []).map((d: Datasource, idx: number) => { const dataSourceInfo: string = getDatasourceInfo(d); return ( props.onClick(d)}> @@ -276,7 +278,10 @@ function ApiRightPane(props: any) { /> - {d.datasourceConfiguration?.url} + { + d.datasourceStorages[getCurrentEnvironment()] + .datasourceConfiguration?.url + } {dataSourceInfo && ( diff --git a/app/client/src/pages/Editor/DataSourceEditor/DBForm.tsx b/app/client/src/pages/Editor/DataSourceEditor/DBForm.tsx index 8b4f40ef5b..79c083396f 100644 --- a/app/client/src/pages/Editor/DataSourceEditor/DBForm.tsx +++ b/app/client/src/pages/Editor/DataSourceEditor/DBForm.tsx @@ -133,6 +133,7 @@ class DatasourceDBEditor extends JSONtoForm { {!_.isNil(formConfig) && !_.isNil(datasource) ? ( diff --git a/app/client/src/pages/Editor/DataSourceEditor/DatasourceSection.tsx b/app/client/src/pages/Editor/DataSourceEditor/DatasourceSection.tsx index 0677d8a13c..428c94cc93 100644 --- a/app/client/src/pages/Editor/DataSourceEditor/DatasourceSection.tsx +++ b/app/client/src/pages/Editor/DataSourceEditor/DatasourceSection.tsx @@ -5,6 +5,7 @@ import styled from "styled-components"; import { isHidden, isKVArray } from "components/formControls/utils"; import log from "loglevel"; import { ComparisonOperationsEnum } from "components/formControls/BaseControl"; +import { getCurrentEnvironment } from "@appsmith/utils/Environments"; const Key = styled.div` color: var(--ads-v2-color-fg-muted); @@ -37,11 +38,14 @@ export default class RenderDatasourceInformation extends React.Component<{ config: any; datasource: Datasource; viewMode?: boolean; + currentEnvironment: string; }> { renderKVArray = (children: Array) => { try { // setup config for each child - const firstConfigProperty = children[0].configProperty; + const firstConfigProperty = + `datasourceStorages.${this.props.currentEnvironment}.` + + children[0].configProperty || children[0].configProperty; const configPropertyInfo = firstConfigProperty.split("[*]."); const values = get(this.props.datasource, configPropertyInfo[0], null); const renderValues: Array< @@ -88,10 +92,18 @@ export default class RenderDatasourceInformation extends React.Component<{ renderDatasourceSection(section: any) { const { datasource, viewMode } = this.props; + const currentEnvironment = getCurrentEnvironment(); return ( {map(section.children, (section) => { - if (isHidden(datasource, section.hidden, undefined, viewMode)) + if ( + isHidden( + datasource.datasourceStorages[currentEnvironment], + section.hidden, + undefined, + viewMode, + ) + ) return null; if ("children" in section) { if (isKVArray(section.children)) { @@ -102,8 +114,9 @@ export default class RenderDatasourceInformation extends React.Component<{ } else { try { const { configProperty, controlType, label } = section; + const customConfigProperty = + `datasourceStorages.${currentEnvironment}.` + configProperty; const reactKey = datasource.id + "_" + label; - if (controlType === "FIXED_KEY_INPUT") { return ( @@ -113,7 +126,7 @@ export default class RenderDatasourceInformation extends React.Component<{ ); } - let value = get(datasource, configProperty); + let value = get(datasource, customConfigProperty); if (controlType === "DROP_DOWN") { if (Array.isArray(section.options)) { diff --git a/app/client/src/pages/Editor/DataSourceEditor/JSONtoForm.tsx b/app/client/src/pages/Editor/DataSourceEditor/JSONtoForm.tsx index 987c293eb7..773c1d3b35 100644 --- a/app/client/src/pages/Editor/DataSourceEditor/JSONtoForm.tsx +++ b/app/client/src/pages/Editor/DataSourceEditor/JSONtoForm.tsx @@ -39,6 +39,7 @@ export interface JSONtoFormProps { datasourceId: string; featureFlags?: FeatureFlags; setupConfig: (config: ControlProps) => void; + currentEnvironment: string; } export class JSONtoForm< @@ -60,12 +61,24 @@ export class JSONtoForm< }; renderMainSection = (section: any, index: number) => { + if ( + !this.props.formData || + !this.props.formData.hasOwnProperty("datasourceStorages") || + !this.props.hasOwnProperty("currentEnvironment") || + !this.props.currentEnvironment || + !this.props.formData.datasourceStorages.hasOwnProperty( + this.props.currentEnvironment, + ) + ) { + return null; + } + // hides features/configs that are hidden behind feature flag // TODO: remove hidden config property as well as this param, // when feature flag is removed if ( isHidden( - this.props.formData, + this.props.formData.datasourceStorages[this.props.currentEnvironment], section.hidden, this.props?.featureFlags, false, // viewMode is false here. @@ -90,13 +103,18 @@ export class JSONtoForm< multipleConfig?: ControlProps[], ) => { multipleConfig = multipleConfig || []; - + const customConfig = { + ...config, + configProperty: + `datasourceStorages.${this.props.currentEnvironment}.` + + config.configProperty, + }; try { - this.props.setupConfig(config); + this.props.setupConfig(customConfig); return ( -
+
@@ -128,7 +146,9 @@ export class JSONtoForm< // when feature flag is removed if ( isHidden( - this.props.formData, + this.props.formData.datasourceStorages[ + this.props.currentEnvironment + ], propertyControlOrSection.hidden, this.props?.featureFlags, false, diff --git a/app/client/src/pages/Editor/DataSourceEditor/NewActionButton.tsx b/app/client/src/pages/Editor/DataSourceEditor/NewActionButton.tsx index 34690bb667..7716b86415 100644 --- a/app/client/src/pages/Editor/DataSourceEditor/NewActionButton.tsx +++ b/app/client/src/pages/Editor/DataSourceEditor/NewActionButton.tsx @@ -14,6 +14,7 @@ import { getCurrentPageId } from "selectors/editorSelectors"; import type { Datasource } from "entities/Datasource"; import type { EventLocation } from "utils/AnalyticsUtil"; import { noop } from "utils/AppsmithUtils"; +import { getCurrentEnvironment } from "@appsmith/utils/Environments"; type NewActionButtonProps = { datasource?: Datasource; @@ -31,6 +32,7 @@ function NewActionButton(props: NewActionButtonProps) { const dispatch = useDispatch(); const actions = useSelector((state: AppState) => state.entities.actions); const currentPageId = useSelector(getCurrentPageId); + const currentEnvironment = getCurrentEnvironment(); const createQueryAction = useCallback( (e) => { @@ -38,8 +40,10 @@ function NewActionButton(props: NewActionButtonProps) { if ( pluginType === PluginType.API && (!datasource || - !datasource.datasourceConfiguration || - !datasource.datasourceConfiguration.url) + !datasource.datasourceStorages[currentEnvironment] + .datasourceConfiguration || + !datasource.datasourceStorages[currentEnvironment] + .datasourceConfiguration.url) ) { toast.show(ERROR_ADD_API_INVALID_URL(), { kind: "error", diff --git a/app/client/src/pages/Editor/DataSourceEditor/index.tsx b/app/client/src/pages/Editor/DataSourceEditor/index.tsx index 9858e84a7a..a2f5403e6a 100644 --- a/app/client/src/pages/Editor/DataSourceEditor/index.tsx +++ b/app/client/src/pages/Editor/DataSourceEditor/index.tsx @@ -80,6 +80,7 @@ import { formValuesToDatasource } from "transformers/RestAPIDatasourceFormTransf import { DSFormHeader } from "./DSFormHeader"; import type { PluginType } from "entities/Action"; import DSDataFilter from "@appsmith/components/DSDataFilter"; +import { DEFAULT_ENV_ID } from "@appsmith/api/ApiUtils"; interface ReduxStateProps { canCreateDatasourceActions: boolean; @@ -192,7 +193,7 @@ class DatasourceEditorRouter extends React.Component { requiredFields: {}, configDetails: {}, filterParams: { - id: "", + id: DEFAULT_ENV_ID, name: "", userPermissions: [], showFilterPane: false, @@ -249,13 +250,25 @@ class DatasourceEditorRouter extends React.Component { // if the datasource id changes, we need to reset the required fields and configDetails if (this.props.datasourceId !== prevProps.datasourceId) { this.setState({ - ...this.state, requiredFields: {}, configDetails: {}, }); } } + getEnvironmentId = () => { + if ( + this.props.isInsideReconnectModal && + this.state.filterParams.id.length === 0 && + !!this.props.datasource + ) { + return Object.keys( + (this.props.datasource as Datasource).datasourceStorages, + )[0]; + } + return this.state.filterParams.id; + }; + componentDidMount() { const urlObject = new URL(window.location.href); const pluginId = urlObject?.searchParams.get("pluginId"); @@ -343,7 +356,6 @@ class DatasourceEditorRouter extends React.Component { configDetails[configProperty] = controlType; if (isRequired) requiredFields[configProperty] = config; this.setState({ - ...this.state, configDetails, requiredFields, }); @@ -440,7 +452,6 @@ class DatasourceEditorRouter extends React.Component { showFilterPane: boolean, ) => { this.setState({ - ...this.state, filterParams: { id, name, @@ -516,6 +527,7 @@ class DatasourceEditorRouter extends React.Component { <> { {/* Render datasource form call-to-actions */} {datasource && ( + {option.label} {option.children.map(renderTreeOption)} @@ -90,6 +90,7 @@ export default function TreeDropdown(props: TreeDropdownProps) { option.className }`} disabled={option.disabled} + key={option.value} onClick={(e) => { handleSelect(option); e.stopPropagation(); diff --git a/app/client/src/pages/Editor/FormControl.tsx b/app/client/src/pages/Editor/FormControl.tsx index 794b4d2495..888399404f 100644 --- a/app/client/src/pages/Editor/FormControl.tsx +++ b/app/client/src/pages/Editor/FormControl.tsx @@ -32,6 +32,8 @@ import { } from "constants/Datasource"; import TemplateMenu from "./QueryEditor/TemplateMenu"; import { SQL_DATASOURCES } from "../../constants/QueryEditorConstants"; +import { getCurrentEnvironment } from "@appsmith/utils/Environments"; +import type { Datasource } from "entities/Datasource"; export interface FormControlProps { config: ControlProps; @@ -40,8 +42,8 @@ export interface FormControlProps { } function FormControl(props: FormControlProps) { - const formValues: Partial = useSelector((state: AppState) => - getFormValues(props.formName)(state), + const formValues: Partial = useSelector( + (state: AppState) => getFormValues(props.formName)(state), ); const actionValues = useSelector((state: AppState) => getAction(state, formValues?.id || ""), @@ -53,7 +55,12 @@ function FormControl(props: FormControlProps) { const [convertFormToRaw, setConvertFormToRaw] = useState(false); const viewType = getViewType(formValues, props.config.configProperty); - const hidden = isHidden(formValues, props.config.hidden); + let formValueForEvaluatingHiddenObj = formValues; + if (!!formValues && formValues.hasOwnProperty("datasourceStorages")) { + formValueForEvaluatingHiddenObj = (formValues as Datasource) + .datasourceStorages[getCurrentEnvironment()]; + } + const hidden = isHidden(formValueForEvaluatingHiddenObj, props.config.hidden); const configErrors: EvaluationError[] = useSelector( (state: AppState) => getConfigErrors(state, { @@ -62,16 +69,18 @@ function FormControl(props: FormControlProps) { }), shallowEqual, ); + const dsId = + ((formValues as Action)?.datasource as any)?.id || + (formValues as Datasource)?.id; const datasourceTableName: string = useSelector((state: AppState) => - getDatasourceFirstTableName(state, (formValues?.datasource as any)?.id), + getDatasourceFirstTableName(state, dsId), ); const pluginTemplates: Record = useSelector((state: AppState) => getPluginTemplates(state), ); - const pluginTemplate = !!formValues?.datasource?.pluginId - ? pluginTemplates[formValues?.datasource?.pluginId] - : undefined; + const pluginId: string = formValues?.pluginId || ""; + const pluginTemplate = !!pluginId ? pluginTemplates[pluginId] : undefined; const pluginName: string = useSelector((state: AppState) => getPluginNameFromId(state, pluginId), ); @@ -84,7 +93,9 @@ function FormControl(props: FormControlProps) { ); const showTemplate = - isNewQuery && formValues?.datasource?.pluginId && isQueryBodyField; + isNewQuery && + (formValues as Action)?.datasource?.pluginId && + isQueryBodyField; const updateQueryParams = () => { const params = getQueryParams(); @@ -203,7 +214,7 @@ function FormControl(props: FormControlProps) { props?.config?.configProperty, ) } - pluginId={formValues?.datasource?.pluginId || ""} + pluginId={(formValues as Action)?.datasource?.pluginId || ""} /> ) : viewTypes.length > 0 && viewTypes.includes(ViewTypes.JSON) ? ( ( [], ); + const currentEnvironment = getCurrentEnvironment(); useEffect(() => { // On mount of component and on change of datasources, Update the list. @@ -42,7 +44,7 @@ export const useDatasourceOptions = ({ FAKE_DATASOURCE_OPTION.CONNECT_NEW_DATASOURCE_OPTION, ); } - datasources.forEach(({ id, isValid, name, pluginId }) => { + datasources.forEach(({ datasourceStorages, id, name, pluginId }) => { const datasourceObject = { id, label: name, @@ -50,7 +52,7 @@ export const useDatasourceOptions = ({ data: { pluginId, isSupportedForTemplate: !!generateCRUDSupportedPlugin[pluginId], - isValid, + isValid: datasourceStorages[currentEnvironment]?.isValid, }, }; if (generateCRUDSupportedPlugin[pluginId]) diff --git a/app/client/src/pages/Editor/IntegrationEditor/DatasourceCard.tsx b/app/client/src/pages/Editor/IntegrationEditor/DatasourceCard.tsx index d39f7a4235..246653c424 100644 --- a/app/client/src/pages/Editor/IntegrationEditor/DatasourceCard.tsx +++ b/app/client/src/pages/Editor/IntegrationEditor/DatasourceCard.tsx @@ -45,6 +45,10 @@ import { import { getAssetUrl } from "@appsmith/utils/airgapHelpers"; import { MenuWrapper, StyledMenu } from "components/utils/formComponents"; import { DatasourceEditEntryPoints } from "constants/Datasource"; +import { + getCurrentEnvironment, + isEnvironmentConfigured, +} from "@appsmith/utils/Environments"; const Wrapper = styled.div` padding: 15px; @@ -275,11 +279,12 @@ function DatasourceCard(props: DatasourceCardProps) {
- {(!datasource.isConfigured || supportTemplateGeneration) && + {(!isEnvironmentConfigured(datasource) || + supportTemplateGeneration) && isDatasourceAuthorizedForQueryCreation(datasource, plugin) && ( )} - + {isEnvironmentConfigured(datasource) && ( + + )} {(canDeleteDatasource || canEditDatasource) && ( diff --git a/app/client/src/pages/Editor/IntegrationEditor/mockData/index.ts b/app/client/src/pages/Editor/IntegrationEditor/mockData/index.ts index c3e7f0cad7..bfcae1224c 100644 --- a/app/client/src/pages/Editor/IntegrationEditor/mockData/index.ts +++ b/app/client/src/pages/Editor/IntegrationEditor/mockData/index.ts @@ -1,3 +1,4 @@ +import { getDefaultEnvId } from "@appsmith/api/ApiUtils"; import { getAssetUrl } from "@appsmith/utils/airgapHelpers"; import { ASSETS_CDN_URL } from "constants/ThirdPartyConstants"; import { PluginPackageName } from "entities/Action"; @@ -38,6 +39,7 @@ export const mockPlugins = [ }, ]; +const defaultEnvId = getDefaultEnvId(); export const mockDatasources = [ { id: "623ab2519b867130d3ed1c27", @@ -50,15 +52,19 @@ export const mockDatasources = [ name: "Mock Database", pluginId: "623a809913b3311bd5e77228", workspaceId: "623a80d613b3311bd5e77308", - datasourceConfiguration: { - connection: { mode: "READ_WRITE", ssl: { authType: "DEFAULT" } }, - endpoints: [ - { - host: "fake-api.cvuydmurdlas.us-east-1.rds.amazonaws.com", - port: 5432, + datasourceStorages: { + [defaultEnvId]: { + datasourceConfiguration: { + connection: { mode: "READ_WRITE", ssl: { authType: "DEFAULT" } }, + endpoints: [ + { + host: "fake-api.cvuydmurdlas.us-east-1.rds.amazonaws.com", + port: 5432, + }, + ], + sshProxyEnabled: false, }, - ], - sshProxyEnabled: false, + }, }, invalids: [], messages: [], @@ -77,16 +83,20 @@ export const mockDatasources = [ name: "Test", pluginId: "623a809913b3311bd5e77229", workspaceId: "623a80d613b3311bd5e77308", - datasourceConfiguration: { - connection: { ssl: { authType: "DEFAULT" } }, - sshProxyEnabled: false, - properties: [ - { key: "isSendSessionEnabled", value: "N" }, - { key: "sessionSignatureKey", value: "" }, - ], - url: "Test", - headers: [], - queryParameters: [], + datasourceStorages: { + [defaultEnvId]: { + datasourceConfiguration: { + connection: { ssl: { authType: "DEFAULT" } }, + sshProxyEnabled: false, + properties: [ + { key: "isSendSessionEnabled", value: "N" }, + { key: "sessionSignatureKey", value: "" }, + ], + url: "Test", + headers: [], + queryParameters: [], + }, + }, }, invalids: [], messages: [], diff --git a/app/client/src/pages/Editor/SaaSEditor/DatasourceCard.tsx b/app/client/src/pages/Editor/SaaSEditor/DatasourceCard.tsx index f0695bbdac..f310e32058 100644 --- a/app/client/src/pages/Editor/SaaSEditor/DatasourceCard.tsx +++ b/app/client/src/pages/Editor/SaaSEditor/DatasourceCard.tsx @@ -19,6 +19,7 @@ import { BaseButton } from "components/designSystems/appsmith/BaseButton"; import { saasEditorDatasourceIdURL } from "RouteBuilder"; import { getAssetUrl } from "@appsmith/utils/airgapHelpers"; import { Button } from "design-system"; +import { getCurrentEnvironment } from "@appsmith/utils/Environments"; const Wrapper = styled.div` border: 2px solid #d6d6d6; @@ -156,6 +157,7 @@ function DatasourceCard(props: DatasourceCardProps) { {!isEmpty(currentFormConfig) ? ( ) : undefined} diff --git a/app/client/src/pages/Editor/SaaSEditor/DatasourceForm.tsx b/app/client/src/pages/Editor/SaaSEditor/DatasourceForm.tsx index b1065e24dc..3482b3f17b 100644 --- a/app/client/src/pages/Editor/SaaSEditor/DatasourceForm.tsx +++ b/app/client/src/pages/Editor/SaaSEditor/DatasourceForm.tsx @@ -1,5 +1,5 @@ import React from "react"; -import _, { isEqual, omit } from "lodash"; +import { get, isEqual, isNil, map, memoize, omit } from "lodash"; import { DATASOURCE_SAAS_FORM } from "@appsmith/constants/forms"; import type { Datasource } from "entities/Datasource"; import { ActionType } from "entities/Datasource"; @@ -76,6 +76,7 @@ import Debugger, { } from "../DataSourceEditor/Debugger"; import { showDebuggerFlag } from "selectors/debuggerSelectors"; import { Form, ViewModeWrapper } from "../DataSourceEditor/DBForm"; +import { getCurrentEnvironment } from "@appsmith/utils/Environments"; interface StateProps extends JSONtoFormProps { applicationId: string; @@ -155,6 +156,7 @@ type RouteProps = { type SaasEditorWrappperState = { requiredFields: Record; configDetails: Record; + currentEditingEnvironment: string; }; class SaasEditorWrapper extends React.Component< SaasEditorWrappperProps, @@ -165,6 +167,7 @@ class SaasEditorWrapper extends React.Component< this.state = { requiredFields: {}, configDetails: {}, + currentEditingEnvironment: getCurrentEnvironment(), }; } @@ -172,7 +175,6 @@ class SaasEditorWrapper extends React.Component< // if the datasource id changes, we need to reset the required fields and configDetails if (this.props.datasourceId !== prevProps.datasourceId) { this.setState({ - ...this.state, requiredFields: {}, configDetails: {}, }); @@ -187,7 +189,6 @@ class SaasEditorWrapper extends React.Component< configDetails[configProperty] = controlType; if (isRequired) requiredFields[configProperty] = config; this.setState({ - ...this.state, configDetails, requiredFields, }); @@ -198,6 +199,7 @@ class SaasEditorWrapper extends React.Component< @@ -460,8 +462,8 @@ class DatasourceSaaSEditor extends JSONtoForm { pageId={pageId} /> ) : null} - {!_.isNil(sections) - ? _.map(sections, this.renderMainSection) + {!isNil(sections) + ? map(sections, this.renderMainSection) : null} {""} @@ -477,11 +479,12 @@ class DatasourceSaaSEditor extends JSONtoForm { pageId={pageId} /> ) : null} - {!_.isNil(formConfig) && - !_.isNil(datasource) && + {!isNil(formConfig) && + !isNil(datasource) && !hideDatasourceSection ? ( @@ -492,10 +495,11 @@ class DatasourceSaaSEditor extends JSONtoForm { {/* Render datasource form call-to-actions */} {datasource && ( { const datasource = getDatasource(state, datasourceId); const { formConfigs } = plugins; const formData = getFormValues(DATASOURCE_SAAS_FORM)(state) as Datasource; - const pluginId = _.get(datasource, "pluginId", ""); + const pluginId = get(datasource, "pluginId", ""); const plugin = getPlugin(state, pluginId); const formConfig = formConfigs[pluginId]; const initialValues = getFormInitialValues(DATASOURCE_SAAS_FORM)( diff --git a/app/client/src/pages/Editor/SaaSEditor/errorUtils.ts b/app/client/src/pages/Editor/SaaSEditor/errorUtils.ts index 7e87fb29c1..a7dca37bb7 100644 --- a/app/client/src/pages/Editor/SaaSEditor/errorUtils.ts +++ b/app/client/src/pages/Editor/SaaSEditor/errorUtils.ts @@ -10,6 +10,8 @@ import { import { getDatasourcePropertyValue } from "utils/editorContextUtils"; import { GOOGLE_SHEET_SPECIFIC_SHEETS_SCOPE } from "constants/Datasource"; import { PluginPackageName } from "entities/Action"; +import { getCurrentEnvironment } from "@appsmith/utils/Environments"; +import { get } from "lodash"; /** * Returns true if : @@ -22,13 +24,22 @@ export function isAuthorisedFilesEmptyGsheet( datasource: Datasource, propertyKey: string, ): boolean { - const scopeValue: string = ( - datasource?.datasourceConfiguration?.authentication as any - )?.scopeString; + const currentEnvironment = getCurrentEnvironment(); + const value = get( + datasource, + `datasourceStorages.${currentEnvironment}.datasourceConfiguration.authentication.scopeString`, + ); + const scopeValue: string = value ? value : ""; - const authorisedFileIds = getDatasourcePropertyValue(datasource, propertyKey); - const authStatus = - datasource?.datasourceConfiguration?.authentication?.authenticationStatus; + const authorisedFileIds = getDatasourcePropertyValue( + datasource, + propertyKey, + currentEnvironment, + ); + const authStatus = get( + datasource, + `datasourceStorages.${currentEnvironment}.datasourceConfiguration.authentication.authenticationStatus`, + ); const isAuthFailure = !!authStatus && authStatus === AuthenticationStatus.FAILURE_FILE_NOT_SELECTED; diff --git a/app/client/src/pages/Editor/gitSync/ReconnectDatasourceModal.tsx b/app/client/src/pages/Editor/gitSync/ReconnectDatasourceModal.tsx index 0aef85f183..c5a0208eab 100644 --- a/app/client/src/pages/Editor/gitSync/ReconnectDatasourceModal.tsx +++ b/app/client/src/pages/Editor/gitSync/ReconnectDatasourceModal.tsx @@ -63,7 +63,15 @@ import { Button, Text, } from "design-system"; +import { isEnvironmentConfigured } from "@appsmith/utils/Environments"; import { keyBy } from "lodash"; +import type { Plugin } from "api/PluginApi"; +import { + isDatasourceAuthorizedForQueryCreation, + isGoogleSheetPluginDS, +} from "utils/editorContextUtils"; +import { areEnvironmentsFetched } from "@appsmith/selectors/environmentSelectors"; +import type { AppState } from "@appsmith/reducers"; const Section = styled.div` display: flex; @@ -248,6 +256,9 @@ function ReconnectDatasourceModal() { const isModalOpen = useSelector(getIsReconnectingDatasourcesModalOpen); const workspaceId = useSelector(getWorkspaceIdForImport); const pageIdForImport = useSelector(getPageIdForImport); + const environmentsFetched = useSelector((state: AppState) => + areEnvironmentsFetched(state, workspaceId), + ); const unconfiguredDatasources = useSelector(getUnconfiguredDatasources); const unconfiguredDatasourceIds = unconfiguredDatasources.map( (ds: Datasource) => ds.id, @@ -293,10 +304,17 @@ function ReconnectDatasourceModal() { const queryDS = datasources.find((ds) => ds.id === queryDatasourceId); const dsName = queryDS?.name; const orgId = queryDS?.workspaceId; - let pluginName = ""; - if (!!queryDS?.pluginId) { - pluginName = plugins[queryDS.pluginId]?.name; - } + + const checkIfDatasourceIsConfigured = (ds: Datasource | null) => { + if (!ds) return false; + const plugin = plugins[ds.pluginId]; + const output = isGoogleSheetPluginDS(plugin?.packageName) + ? isDatasourceAuthorizedForQueryCreation(ds, plugin as Plugin) + : ds.datasourceStorages + ? isEnvironmentConfigured(ds) + : false; + return output; + }; // when redirecting from oauth, processing the status if (isImport) { @@ -316,7 +334,7 @@ function ReconnectDatasourceModal() { oAuthPassOrFailVerdict: status, workspaceId: orgId, datasourceName: dsName, - pluginName: pluginName, + pluginName: plugins[datasource?.pluginId || ""]?.name, }); } else if (queryDatasourceId) { dispatch(loadFilePickerAction()); @@ -362,12 +380,12 @@ function ReconnectDatasourceModal() { // todo uncomment this to fetch datasource config useEffect(() => { - if (isModalOpen && workspaceId) { + if (isModalOpen && workspaceId && environmentsFetched) { dispatch( initDatasourceConnectionDuringImportRequest(workspaceId as string), ); } - }, [workspaceId, isModalOpen]); + }, [workspaceId, isModalOpen, environmentsFetched]); useEffect(() => { if (isModalOpen) { @@ -428,7 +446,7 @@ function ReconnectDatasourceModal() { id: ds.id, name: ds.name, pluginName: plugins[ds.id]?.name, - isConfigured: ds.isConfigured, + isConfigured: checkIfDatasourceIsConfigured(ds), }); }, []); @@ -474,19 +492,20 @@ function ReconnectDatasourceModal() { useEffect(() => { if (isModalOpen && !isTesting) { const id = selectedDatasourceId; - const pending = datasources.filter((ds: Datasource) => !ds.isConfigured); + const pending = datasources.filter( + (ds: Datasource) => !checkIfDatasourceIsConfigured(ds), + ); if (pending.length > 0) { - let next: Datasource | undefined = undefined; if (id) { - const index = datasources.findIndex((ds: Datasource) => ds.id === id); - if (index > -1 && !datasources[index].isConfigured) { + // checking if the current datasource is still pending + const index = pending.findIndex((ds: Datasource) => ds.id === id); + if (index > -1) { + // don't do anything if the current datasource is still pending return; } - next = datasources - .slice(index + 1) - .find((ds: Datasource) => !ds.isConfigured); } - next = next || pending[0]; + // goto next pending datasource + const next: Datasource = pending[0]; if (next && next.id) { setSelectedDatasourceId(next.id); setDatasource(next); @@ -525,7 +544,7 @@ function ReconnectDatasourceModal() { }); const shouldShowDBForm = - isConfigFetched && !isLoading && !datasource?.isConfigured; + isConfigFetched && !isLoading && !checkIfDatasourceIsConfigured(datasource); return ( @@ -562,7 +581,7 @@ function ReconnectDatasourceModal() { pageId={pageId} /> )} - {datasource?.isConfigured && SuccessMessages()} + {checkIfDatasourceIsConfigured(datasource) && SuccessMessages()} diff --git a/app/client/src/pages/Editor/gitSync/components/DatasourceListItem.tsx b/app/client/src/pages/Editor/gitSync/components/DatasourceListItem.tsx index 5cfea401db..84bce08231 100644 --- a/app/client/src/pages/Editor/gitSync/components/DatasourceListItem.tsx +++ b/app/client/src/pages/Editor/gitSync/components/DatasourceListItem.tsx @@ -5,6 +5,7 @@ import type { Datasource } from "entities/Datasource"; import styled from "styled-components"; import { getAssetUrl } from "@appsmith/utils/airgapHelpers"; import { PluginImage } from "pages/Editor/DataSourceEditor/DSFormHeader"; +import { isEnvironmentConfigured } from "@appsmith/utils/Environments"; import type { Plugin } from "api/PluginApi"; import { isDatasourceAuthorizedForQueryCreation, @@ -62,7 +63,7 @@ function ListItemWrapper(props: { const { ds, onClick, plugin, selected } = props; const isPluginAuthorized = isGoogleSheetPluginDS(plugin?.packageName) ? isDatasourceAuthorizedForQueryCreation(ds, plugin ?? {}) - : ds.isConfigured; + : isEnvironmentConfigured(ds); return ( Datasource; + currentEnvironment: string; isInvalid: boolean; pageId?: string; viewMode?: boolean; @@ -120,6 +121,7 @@ const StyledAuthMessage = styled.div` `; function DatasourceAuth({ + currentEnvironment, datasource, datasourceButtonConfiguration = [ DatasourceButtonTypeEnum.CANCEL, @@ -147,7 +149,9 @@ function DatasourceAuth({ const authType = formData && "authType" in formData ? formData?.authType - : formData?.datasourceConfiguration?.authentication?.authenticationType; + : formData?.datasourceStorages && + formData?.datasourceStorages[currentEnvironment] + ?.datasourceConfiguration?.authentication?.authenticationType; const { id: datasourceId } = datasource; const applicationId = useSelector(getCurrentApplicationId); @@ -223,10 +227,10 @@ function DatasourceAuth({ } } }, [triggerSave]); - const isAuthorized = - datasource?.datasourceConfiguration?.authentication - ?.authenticationStatus === AuthenticationStatus.SUCCESS; + datasource?.datasourceStorages && + datasource?.datasourceStorages[currentEnvironment]?.datasourceConfiguration + ?.authentication?.authenticationStatus === AuthenticationStatus.SUCCESS; // Button Operations for respective buttons. @@ -297,7 +301,6 @@ function DatasourceAuth({ }; const createMode = datasourceId === TEMP_DATASOURCE_ID; - const datasourceButtonsComponentMap = (buttonType: string): JSX.Element => { return { [DatasourceButtonType.TEST]: ( diff --git a/app/client/src/reducers/entityReducers/datasourceReducer.ts b/app/client/src/reducers/entityReducers/datasourceReducer.ts index afeff907df..a1ab5cb1a2 100644 --- a/app/client/src/reducers/entityReducers/datasourceReducer.ts +++ b/app/client/src/reducers/entityReducers/datasourceReducer.ts @@ -6,6 +6,7 @@ import { } from "@appsmith/constants/ReduxActionConstants"; import type { Datasource, + DatasourceStorage, DatasourceStructure, MockDatasource, } from "entities/Datasource"; @@ -300,6 +301,43 @@ const datasourceReducer = createReducer(initialState, { ], }; }, + [ReduxActionTypes.UPDATE_DATASOURCE_STORAGE_SUCCESS]: ( + state: DatasourceDataState, + action: ReduxAction, + ): DatasourceDataState => { + return { + ...state, + loading: false, + list: state.list.map((datasource) => { + if (datasource.id === action.payload.datasourceId) + return { + ...datasource, + datasourceStorages: { + [`${action.payload.environmentId}`]: action.payload, + }, + }; + + return datasource; + }), + unconfiguredList: state.unconfiguredList.map((datasource) => { + if (datasource.id === action.payload.datasourceId) + return { + ...datasource, + datasourceStorages: { + [`${action.payload.environmentId}`]: action.payload, + }, + }; + + return datasource; + }), + recentDatasources: [ + action.payload.datasourceId, + ...state.recentDatasources.filter( + (ds) => ds !== action.payload.datasourceId, + ), + ], + }; + }, [ReduxActionTypes.UPDATE_DATASOURCE_IMPORT_SUCCESS]: ( state: DatasourceDataState, action: ReduxAction, diff --git a/app/client/src/sagas/DatasourcesSagas.ts b/app/client/src/sagas/DatasourcesSagas.ts index ee3f47098f..8b9a2000f9 100644 --- a/app/client/src/sagas/DatasourcesSagas.ts +++ b/app/client/src/sagas/DatasourcesSagas.ts @@ -143,11 +143,13 @@ import { toast } from "design-system"; import { fetchPluginFormConfig } from "actions/pluginActions"; import { addClassToDocumentRoot } from "pages/utils"; import { AuthorizationStatus } from "pages/common/datasourceAuth"; +import { getCurrentEnvironment } from "@appsmith/utils/Environments"; import { getFormDiffPaths, getFormName, isGoogleSheetPluginDS, } from "utils/editorContextUtils"; +import { getDefaultEnvId } from "@appsmith/api/ApiUtils"; function* fetchDatasourcesSaga( action: ReduxAction<{ workspaceId?: string } | undefined>, @@ -417,29 +419,37 @@ function* updateDatasourceSaga( ) { try { const queryParams = getQueryParams(); + const currentEnvironment = getCurrentEnvironment(); const datasourcePayload = omit(actionPayload.payload, "name"); const pluginPackageName: PluginPackageName = yield select( getPluginPackageFromDatasourceId, datasourcePayload?.id, ); - datasourcePayload.isConfigured = true; // when clicking save button, it should be changed as configured + // when clicking save button, it should be changed as configured + set( + datasourcePayload, + `datasourceStorages.${currentEnvironment}.isConfigured`, + true, + ); // When importing app with google sheets with specific sheets scope // We do not want to set isConfigured to true immediately on save // instead we want to wait for authorisation as well as file selection to be complete if (isGoogleSheetPluginDS(pluginPackageName)) { - const scopeString: string = - (datasourcePayload?.datasourceConfiguration?.authentication as any) - ?.scopeString || ""; + const value = get( + datasourcePayload, + `datasourceStorages.${currentEnvironment}.datasourceConfiguration.authentication.scopeString`, + ); + const scopeString: string = value ? value : ""; if (scopeString.includes(GOOGLE_SHEET_SPECIFIC_SHEETS_SCOPE)) { - datasourcePayload.isConfigured = false; + datasourcePayload.datasourceStorages[currentEnvironment].isConfigured = + false; } } const response: ApiResponse = - yield DatasourcesApi.updateDatasource( - datasourcePayload, - datasourcePayload.id, + yield DatasourcesApi.updateDatasourceStorage( + datasourcePayload.datasourceStorages[currentEnvironment], ); const isValidResponse: boolean = yield validateResponse(response); if (isValidResponse) { @@ -495,7 +505,9 @@ function* updateDatasourceSaga( type: ENTITY_TYPE.DATASOURCE, }, state: { - datasourceConfiguration: response.data.datasourceConfiguration, + datasourceConfiguration: + response.data.datasourceStorages[currentEnvironment] + .datasourceConfiguration, }, }); @@ -588,10 +600,10 @@ function* getOAuthAccessTokenSaga( response.data.datasource?.pluginId, ); if (validateResponse(response)) { - // Update the datasource object + // Update the datasource storage object only since the token call only returns the storage object yield put({ - type: ReduxActionTypes.UPDATE_DATASOURCE_SUCCESS, - payload: response.data.datasource, + type: ReduxActionTypes.UPDATE_DATASOURCE_STORAGE_SUCCESS, + payload: response.data.datasource, // This is the datasourceStorage object }); if (!!response.data.token) { @@ -681,6 +693,7 @@ function* testDatasourceSaga(actionPayload: ReduxAction) { yield select(getDatasource, actionPayload.payload.id), `Datasource not found for id - ${actionPayload.payload.id}`, ); + const currentEnvironment = getCurrentEnvironment(); const payload = { ...actionPayload.payload, id: actionPayload.payload.id as any, @@ -695,10 +708,11 @@ function* testDatasourceSaga(actionPayload: ReduxAction) { try { const response: ApiResponse = - yield DatasourcesApi.testDatasource({ - ...payload, + yield DatasourcesApi.testDatasource( + payload.datasourceStorages[currentEnvironment], + plugin.id, workspaceId, - }); + ); const isValidResponse: boolean = yield validateResponse(response); let messages: Array = []; if (isValidResponse) { @@ -816,19 +830,27 @@ function* createTempDatasourceFromFormSaga( datasourceType = plugin?.type; } + const defaultEnvId = getDefaultEnvId(); + const initialPayload = { id: TEMP_DATASOURCE_ID, name: DATASOURCE_NAME_DEFAULT_PREFIX + sequence, type: datasourceType, pluginId: actionPayload.payload.pluginId, new: false, - datasourceConfiguration: { - properties: [], + datasourceStorages: { + [defaultEnvId]: { + datasourceId: "", + environmentId: defaultEnvId, + datasourceConfiguration: { + properties: [], + }, + }, }, }; - - const payload = merge( - merge(initialPayload, actionPayload.payload), + const payload = merge(initialPayload, actionPayload.payload); + payload.datasourceStorages[defaultEnvId] = merge( + payload.datasourceStorages[defaultEnvId], initialValues, ); @@ -861,19 +883,21 @@ function* createDatasourceFromFormSaga( getPluginForm, actionPayload.payload.pluginId, ); + const currentEnvironment = getCurrentEnvironment(); const initialValues: unknown = yield call( getConfigInitialValues, formConfig, ); + actionPayload.payload.datasourceStorages[currentEnvironment] = merge( + initialValues, + actionPayload.payload.datasourceStorages[currentEnvironment], + ); - const payload = omit(merge(initialValues, actionPayload.payload), [ - "id", - "new", - "type", - ]); + const payload = omit(actionPayload.payload, ["id", "new", "type"]); - payload.isConfigured = true; + if (payload.datasourceStorages) + payload.datasourceStorages[currentEnvironment].isConfigured = true; const response: ApiResponse = yield DatasourcesApi.createDatasource({ @@ -1050,6 +1074,7 @@ function* storeAsDatasourceSaga() { let datasource = get(values, "datasource"); datasource = omit(datasource, ["name"]); const originalHeaders = get(values, "actionConfiguration.headers", []); + const currentEnvironment = getCurrentEnvironment(); const [datasourceHeaders, actionHeaders] = partition( originalHeaders, ({ key, value }: { key: string; value: string }) => { @@ -1069,15 +1094,23 @@ function* storeAsDatasourceSaga() { (d) => !(d.key === "" && d.key === ""), ); - set(datasource, "datasourceConfiguration.headers", filteredDatasourceHeaders); - yield put(createTempDatasourceFromForm(datasource)); const createDatasourceSuccessAction: unknown = yield take( ReduxActionTypes.CREATE_DATASOURCE_SUCCESS, ); // @ts-expect-error: createDatasourceSuccessAction is of type unknown - const createdDatasource = createDatasourceSuccessAction.payload; - + let createdDatasource = createDatasourceSuccessAction.payload; + set( + createdDatasource, + `datasourceStorages.${currentEnvironment}.datasourceConfiguration.headers`, + filteredDatasourceHeaders, + ); + set( + createdDatasource, + `datasourceStorages.${currentEnvironment}.datasourceConfiguration.url`, + datasource.datasourceConfiguration.url, + ); + createdDatasource = omit(createdDatasource, ["datasourceConfiguration"]); // Set datasource page to edit mode yield put(setDatasourceViewMode(false)); @@ -1440,6 +1473,7 @@ function* filePickerActionCallbackSaga( const plugin: Plugin = yield select(getPlugin, datasource?.pluginId); const applicationId: string = yield select(getCurrentApplicationId); const pageId: string = yield select(getCurrentPageId); + const currentEnvironment = getCurrentEnvironment(); // update authentication status based on whether files were picked or not const authStatus = @@ -1448,7 +1482,11 @@ function* filePickerActionCallbackSaga( : AuthenticationStatus.FAILURE_FILE_NOT_SELECTED; // Once files are selected in case of import, set this flag - set(datasource, "isConfigured", true); + set( + datasource, + `datasourceStorages.${currentEnvironment}.datasourceConfiguration.authentication.authenticationStatus`, + true, + ); // auth complete event once the files are selected/not selected AnalyticsUtil.logEvent("DATASOURCE_AUTH_COMPLETE", { @@ -1468,10 +1506,14 @@ function* filePickerActionCallbackSaga( // Sending sheet ids selected as part of datasource // config properties in order to save it in database // using the second index specifically for file ids. - set(datasource, "datasourceConfiguration.properties[1]", { - key: createMessage(GSHEET_AUTHORISED_FILE_IDS_KEY), - value: fileIds, - }); + set( + datasource, + `datasourceStorages.${currentEnvironment}.datasourceConfiguration.properties[1]`, + { + key: createMessage(GSHEET_AUTHORISED_FILE_IDS_KEY), + value: fileIds, + }, + ); yield put(updateDatasourceAuthState(datasource, authStatus)); } catch (error) { yield put({ @@ -1703,13 +1745,16 @@ function* updateDatasourceAuthStateSaga( ) { try { const { authStatus, datasource } = actionPayload.payload; + const currentEnvironment = getCurrentEnvironment(); set( datasource, - "datasourceConfiguration.authentication.authenticationStatus", + `datasourceStorages.${currentEnvironment}.datasourceConfiguration.authentication.authenticationStatus`, authStatus, ); const response: ApiResponse = - yield DatasourcesApi.updateDatasource(datasource, datasource.id); + yield DatasourcesApi.updateDatasourceStorage( + datasource.datasourceStorages[currentEnvironment], + ); const isValidResponse: boolean = yield validateResponse(response); if (isValidResponse) { yield put({ diff --git a/app/client/src/sagas/ErrorSagas.tsx b/app/client/src/sagas/ErrorSagas.tsx index fe504bbf5d..5ac8964d5e 100644 --- a/app/client/src/sagas/ErrorSagas.tsx +++ b/app/client/src/sagas/ErrorSagas.tsx @@ -250,7 +250,7 @@ export function* errorSaga(errorAction: ReduxAction) { function logErrorSaga(action: ReduxAction<{ error: ErrorPayloadType }>) { log.debug(`Error in action ${action.type}`); - if (action.payload) log.error(action.payload.error); + if (action.payload) log.error(action.payload.error, action); } /** diff --git a/app/client/src/sagas/QueryPaneSagas.ts b/app/client/src/sagas/QueryPaneSagas.ts index 5b19bb7fd6..5e4cc790e0 100644 --- a/app/client/src/sagas/QueryPaneSagas.ts +++ b/app/client/src/sagas/QueryPaneSagas.ts @@ -81,6 +81,7 @@ import { getIsGeneratePageInitiator } from "utils/GenerateCrudUtil"; import { toast } from "design-system"; import type { CreateDatasourceSuccessAction } from "actions/datasourceActions"; import { createDefaultActionPayload } from "./ActionSagas"; +import { getCurrentEnvironment } from "@appsmith/utils/Environments"; // Called whenever the query being edited is changed via the URL or query pane function* changeQuerySaga(actionPayload: ReduxAction<{ id: string }>) { @@ -260,6 +261,7 @@ function* formValueChangeSaga( // Editing form fields triggers evaluations. // We pass the action to run form evaluations when the dataTree evaluation is complete + const currentEnvironment = getCurrentEnvironment(); const postEvalActions = uiComponent === UIComponentTypes.UQIDbEditorForm ? [ @@ -270,7 +272,8 @@ function* formValueChangeSaga( values.pluginId, field, hasRouteChanged, - datasource?.datasourceConfiguration, + datasource?.datasourceStorages[currentEnvironment] + .datasourceConfiguration, ), ] : []; diff --git a/app/client/src/selectors/datasourceSelectors.tsx b/app/client/src/selectors/datasourceSelectors.tsx deleted file mode 100644 index 2ea92f1f08..0000000000 --- a/app/client/src/selectors/datasourceSelectors.tsx +++ /dev/null @@ -1,4 +0,0 @@ -import type { AppState } from "@appsmith/reducers"; - -export const getDatasourceResponsePaneHeight = (state: AppState) => - state.ui.datasourcePane.responseTabHeight; diff --git a/app/client/src/transformers/RestAPIDatasourceFormTransformer.ts b/app/client/src/transformers/RestAPIDatasourceFormTransformer.ts index 6392999644..5d9c39a39e 100644 --- a/app/client/src/transformers/RestAPIDatasourceFormTransformer.ts +++ b/app/client/src/transformers/RestAPIDatasourceFormTransformer.ts @@ -1,3 +1,7 @@ +import { + getCurrentEnvironment, + isEnvironmentValid, +} from "@appsmith/utils/Environments"; import type { Property } from "entities/Action"; import type { Datasource } from "entities/Datasource"; import type { @@ -12,38 +16,54 @@ import type { SSL, } from "entities/Datasource/RestAPIForm"; import { AuthType, GrantType, SSLType } from "entities/Datasource/RestAPIForm"; -import _ from "lodash"; +import { get, set } from "lodash"; export const datasourceToFormValues = ( datasource: Datasource, ): ApiDatasourceForm => { - const authType = _.get( + const currentEnvironment = getCurrentEnvironment(); + const authType = get( datasource, - "datasourceConfiguration.authentication.authenticationType", + `datasourceStorages.${currentEnvironment}.datasourceConfiguration.authentication.authenticationType`, AuthType.NONE, ) as AuthType; - const connection = _.get(datasource, "datasourceConfiguration.connection", { - ssl: { - authType: SSLType.DEFAULT, - } as SSL, - }); + const connection = get( + datasource, + `datasourceStorages.${currentEnvironment}.datasourceConfiguration.connection`, + { + ssl: { + authType: SSLType.DEFAULT, + } as SSL, + }, + ); const authentication = datasourceToFormAuthentication(authType, datasource); const isSendSessionEnabled = - _.get(datasource, "datasourceConfiguration.properties[0].value", "N") === - "Y"; + get( + datasource, + `datasourceStorages.${currentEnvironment}.datasourceConfiguration.properties[0].value`, + "N", + ) === "Y"; const sessionSignatureKey = isSendSessionEnabled ? // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - _.get(datasource, "datasourceConfiguration.properties[1].value")! + get( + datasource, + `datasourceStorages.${currentEnvironment}.datasourceConfiguration.properties[1].value`, + )! : ""; return { datasourceId: datasource.id, workspaceId: datasource.workspaceId, pluginId: datasource.pluginId, - isValid: datasource.isValid, - url: datasource.datasourceConfiguration?.url, - headers: cleanupProperties(datasource.datasourceConfiguration?.headers), + isValid: isEnvironmentValid(datasource, currentEnvironment), + url: datasource.datasourceStorages[currentEnvironment] + ?.datasourceConfiguration?.url, + headers: cleanupProperties( + datasource.datasourceStorages[currentEnvironment]?.datasourceConfiguration + ?.headers, + ), queryParameters: cleanupProperties( - datasource.datasourceConfiguration?.queryParameters, + datasource.datasourceStorages[currentEnvironment]?.datasourceConfiguration + ?.queryParameters, ), isSendSessionEnabled: isSendSessionEnabled, sessionSignatureKey: sessionSignatureKey, @@ -57,28 +77,31 @@ export const formValuesToDatasource = ( datasource: Datasource, form: ApiDatasourceForm, ): Datasource => { + const currentEnvironment = getCurrentEnvironment(); const authentication = formToDatasourceAuthentication( form.authType, form.authentication, ); - - return { - ...datasource, - datasourceConfiguration: { - url: form.url, - headers: cleanupProperties(form.headers), - queryParameters: cleanupProperties(form.queryParameters), - properties: [ - { - key: "isSendSessionEnabled", - value: form.isSendSessionEnabled ? "Y" : "N", - }, - { key: "sessionSignatureKey", value: form.sessionSignatureKey }, - ], - authentication: authentication, - connection: form.connection, - }, - } as Datasource; + const conf = { + url: form.url, + headers: cleanupProperties(form.headers), + queryParameters: cleanupProperties(form.queryParameters), + properties: [ + { + key: "isSendSessionEnabled", + value: form.isSendSessionEnabled ? "Y" : "N", + }, + { key: "sessionSignatureKey", value: form.sessionSignatureKey }, + ], + authentication: authentication, + connection: form.connection, + }; + set( + datasource, + `datasourceStorages.${currentEnvironment}.datasourceConfiguration`, + conf, + ); + return datasource; }; const formToDatasourceAuthentication = ( @@ -166,15 +189,20 @@ const datasourceToFormAuthentication = ( authType: AuthType, datasource: Datasource, ): Authentication | undefined => { + const currentEnvironment = getCurrentEnvironment(); if ( !datasource || - !datasource.datasourceConfiguration || - !datasource.datasourceConfiguration.authentication + !datasource.datasourceStorages[currentEnvironment] + ?.datasourceConfiguration || + !datasource.datasourceStorages[currentEnvironment]?.datasourceConfiguration + .authentication ) { return; } - const authentication = datasource.datasourceConfiguration.authentication; + const authentication = + datasource.datasourceStorages[currentEnvironment].datasourceConfiguration + .authentication || {}; if ( isClientCredentials(authType, authentication) || isAuthorizationCode(authType, authentication) @@ -255,7 +283,7 @@ const isClientCredentials = ( if (authType !== AuthType.OAuth2) return false; // If there's no authentication object at all and it is oauth2, it is client credentials by default if (!val) return true; - return _.get(val, "grantType") === GrantType.ClientCredentials; + return get(val, "grantType") === GrantType.ClientCredentials; }; const isAuthorizationCode = ( @@ -263,7 +291,7 @@ const isAuthorizationCode = ( val: any, ): val is AuthorizationCode => { if (authType !== AuthType.OAuth2) return false; - return _.get(val, "grantType") === GrantType.AuthorizationCode; + return get(val, "grantType") === GrantType.AuthorizationCode; }; const cleanupProperties = (values: Property[] | undefined): Property[] => { diff --git a/app/client/src/utils/editorContextUtils.ts b/app/client/src/utils/editorContextUtils.ts index cfda50e4a2..97ae8a9b6d 100644 --- a/app/client/src/utils/editorContextUtils.ts +++ b/app/client/src/utils/editorContextUtils.ts @@ -4,11 +4,12 @@ import { DATASOURCE_REST_API_FORM, DATASOURCE_SAAS_FORM, } from "@appsmith/constants/forms"; +import { getCurrentEnvironment } from "@appsmith/utils/Environments"; import { diff } from "deep-diff"; import { PluginPackageName, PluginType } from "entities/Action"; import type { Datasource } from "entities/Datasource"; import { AuthenticationStatus, AuthType } from "entities/Datasource"; -import { isArray } from "lodash"; +import { get, isArray } from "lodash"; export function isCurrentFocusOnInput() { return ( ["input", "textarea"].indexOf( @@ -83,17 +84,21 @@ export function isDatasourceAuthorizedForQueryCreation( datasource: Datasource, plugin: Plugin, ): boolean { + const currentEnvironment = getCurrentEnvironment(); if (!datasource) return false; - const authType = - datasource && - datasource?.datasourceConfiguration?.authentication?.authenticationType; + const authType = get( + datasource, + `datasourceStorages.${currentEnvironment}.datasourceConfiguration.authentication.authenticationType`, + ); const isGoogleSheetPlugin = isGoogleSheetPluginDS(plugin?.packageName); if (isGoogleSheetPlugin) { const isAuthorized = authType === AuthType.OAUTH2 && - datasource?.datasourceConfiguration?.authentication - ?.authenticationStatus === AuthenticationStatus.SUCCESS; + get( + datasource, + `datasourceStorages.${currentEnvironment}.datasourceConfiguration.authentication.authenticationStatus`, + ) === AuthenticationStatus.SUCCESS; return isAuthorized; } @@ -118,12 +123,16 @@ export function isGoogleSheetPluginDS(pluginPackageName?: string) { export function getDatasourcePropertyValue( datasource: Datasource, propertyKey: string, + currentEnvironment: string, ): string | null { if (!datasource) { return null; } - const properties = datasource?.datasourceConfiguration?.properties; + const properties = get( + datasource, + `datasourceStorages.${currentEnvironment}.datasourceConfiguration.properties`, + ); if (!!properties && properties.length > 0) { const propertyObj = properties.find((prop) => prop.key === propertyKey); if (!!propertyObj) { diff --git a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/Datasource.java b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/Datasource.java index 30a89648a3..2622c6e660 100644 --- a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/Datasource.java +++ b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/Datasource.java @@ -52,11 +52,12 @@ public class Datasource extends BranchAwareDomain implements Forkable datasourceStorages = new HashMap<>(); @@ -86,7 +87,7 @@ public class Datasource extends BranchAwareDomain implements Forkable storages = new HashMap<>(); - this.datasourceStorages = storages; - if (datasourceStorage.getEnvironmentId() != null) { - storages.put(datasourceStorage.getEnvironmentId(), new DatasourceStorageDTO(datasourceStorage)); - } - this.hasDatasourceStorage = true; - } - /** + * * This method is here so that the JSON version of this class' instances have a `isValid` field, for backwards * compatibility. It may be removed, when sure that no API received is relying on this field. * @@ -183,4 +162,15 @@ public class Datasource extends BranchAwareDomain implements Forkable { Set invalids; Set messages; + String pluginId; + String workspaceId; + public DatasourceStorageDTO(DatasourceStorage datasourceStorage) { this.id = datasourceStorage.getId(); this.datasourceId = datasourceStorage.getDatasourceId(); @@ -39,6 +45,25 @@ public class DatasourceStorageDTO implements Forkable { this.messages = datasource.getMessages(); } + /** + * This constructor is used when we have datasource config readily available for creation of datasource. + * or, for updating the datasource storages. + * @param datasourceId + * @param environmentId + * @param datasourceConfiguration + */ + public DatasourceStorageDTO(String datasourceId, String environmentId, DatasourceConfiguration datasourceConfiguration) { + this.datasourceId = datasourceId; + this.environmentId = environmentId; + this.datasourceConfiguration = datasourceConfiguration; + this.isConfigured = Boolean.TRUE; + } + + @JsonView(Views.Public.class) + public boolean getIsValid() { + return CollectionUtils.isEmpty(invalids); + } + /** * Intended to function like `.equals`, but only semantically significant fields, except for the ID. Semantically * significant just means that if two datasource have same values for these fields, actions against them will behave diff --git a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/OAuth2ResponseDTO.java b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/OAuth2ResponseDTO.java index f895423a3f..b4caf8b63c 100644 --- a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/OAuth2ResponseDTO.java +++ b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/OAuth2ResponseDTO.java @@ -10,7 +10,7 @@ import lombok.ToString; @ToString @NoArgsConstructor public class OAuth2ResponseDTO { - DatasourceDTO datasource; + DatasourceStorage datasource; String token; String projectID; } diff --git a/app/server/appsmith-plugins/elasticSearchPlugin/src/main/resources/form.json b/app/server/appsmith-plugins/elasticSearchPlugin/src/main/resources/form.json index ab311b52cf..d6dd3a2b12 100644 --- a/app/server/appsmith-plugins/elasticSearchPlugin/src/main/resources/form.json +++ b/app/server/appsmith-plugins/elasticSearchPlugin/src/main/resources/form.json @@ -27,7 +27,7 @@ }, { "sectionName": "Authentication", - "children": [ + "children": [ { "label": "Username for Basic Auth", "configProperty": "datasourceConfiguration.authentication.username", diff --git a/app/server/appsmith-plugins/googleSheetsPlugin/src/main/resources/form.json b/app/server/appsmith-plugins/googleSheetsPlugin/src/main/resources/form.json index d9f10d78d2..66447b9f56 100644 --- a/app/server/appsmith-plugins/googleSheetsPlugin/src/main/resources/form.json +++ b/app/server/appsmith-plugins/googleSheetsPlugin/src/main/resources/form.json @@ -60,4 +60,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/app/server/appsmith-plugins/graphqlPlugin/src/main/resources/form.json b/app/server/appsmith-plugins/graphqlPlugin/src/main/resources/form.json index 7a8202acf5..f5dd0c7681 100644 --- a/app/server/appsmith-plugins/graphqlPlugin/src/main/resources/form.json +++ b/app/server/appsmith-plugins/graphqlPlugin/src/main/resources/form.json @@ -206,7 +206,7 @@ "placeholderText": "Client secret", "controlType": "INPUT_TEXT", "isRequired": false, - "encrypted":true, + "encrypted": true, "hidden": { "path": "datasourceConfiguration.authentication.authenticationType", "comparison": "NOT_EQUALS", diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/constants/ce/FieldNameCE.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/constants/ce/FieldNameCE.java index 468878ea6f..0e5e140676 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/constants/ce/FieldNameCE.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/constants/ce/FieldNameCE.java @@ -6,6 +6,7 @@ public class FieldNameCE { @Deprecated public static final String ORGANIZATION_ID = "organizationId"; public static final String WORKSPACE_ID = "workspaceId"; + public static final String DATASOURCE_ID = "datasourceId"; public static final String DELETED = "deleted"; public static final String CREATED_AT = "createdAt"; public static final String DELETED_AT = "deletedAt"; diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/controllers/ce/ApplicationControllerCE.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/controllers/ce/ApplicationControllerCE.java index f0492fe3c5..d9d9b9fac6 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/controllers/ce/ApplicationControllerCE.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/controllers/ce/ApplicationControllerCE.java @@ -1,6 +1,6 @@ package com.appsmith.server.controllers.ce; -import com.appsmith.external.models.DatasourceDTO; +import com.appsmith.external.models.Datasource; import com.appsmith.external.views.Views; import com.appsmith.server.constants.FieldName; import com.appsmith.server.constants.Url; @@ -284,8 +284,8 @@ public class ApplicationControllerCE extends BaseController>> getUnConfiguredDatasource(@PathVariable String workspaceId, @RequestParam String defaultApplicationId) { - return importExportApplicationService.findDatasourceDTOByApplicationId(defaultApplicationId, workspaceId) + public Mono>> getUnConfiguredDatasource(@PathVariable String workspaceId, @RequestParam String defaultApplicationId) { + return importExportApplicationService.findDatasourceByApplicationId(defaultApplicationId, workspaceId) .map(result -> new ResponseDTO<>(HttpStatus.OK.value(), result, null)); } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/controllers/ce/DatasourceControllerCE.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/controllers/ce/DatasourceControllerCE.java index d9d8dd54c9..250889175f 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/controllers/ce/DatasourceControllerCE.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/controllers/ce/DatasourceControllerCE.java @@ -1,7 +1,7 @@ package com.appsmith.server.controllers.ce; import com.appsmith.external.models.Datasource; -import com.appsmith.external.models.DatasourceDTO; +import com.appsmith.external.models.DatasourceStorageDTO; import com.appsmith.external.models.DatasourceStructure; import com.appsmith.external.models.DatasourceTestResult; import com.appsmith.external.models.TriggerRequestDTO; @@ -66,7 +66,7 @@ public class DatasourceControllerCE { @JsonView(Views.Public.class) @GetMapping("") - public Mono>> getAll(@RequestParam MultiValueMap params) { + public Mono>> getAll(@RequestParam MultiValueMap params) { log.debug("Going to get all resources from datasource controller {}", params); return datasourceService.getAllWithStorages(params).collectList() .map(resources -> new ResponseDTO<>(HttpStatus.OK.value(), resources, null)); @@ -75,20 +75,31 @@ public class DatasourceControllerCE { @JsonView(Views.Public.class) @PostMapping @ResponseStatus(HttpStatus.CREATED) - public Mono> create(@Valid @RequestBody DatasourceDTO resource, - @RequestHeader(name = FieldName.ENVIRONMENT_ID, required = false) String environmentId) { + public Mono> create(@Valid @RequestBody Datasource resource, + @RequestHeader(name = FieldName.ENVIRONMENT_ID, required = false) String activeEnvironmentId) { log.debug("Going to create resource from datasource controller"); - return datasourceService.create(resource, environmentId) + return datasourceService.create(resource) .map(created -> new ResponseDTO<>(HttpStatus.CREATED.value(), created, null)); } @JsonView(Views.Public.class) @PutMapping("/{id}") - public Mono> update(@PathVariable String id, - @RequestBody DatasourceDTO datasourceDTO, + public Mono> update(@PathVariable String id, + @RequestBody Datasource datasource, @RequestHeader(name = FieldName.ENVIRONMENT_ID, required = false) String environmentId) { log.debug("Going to update resource from datasource controller with id: {}", id); - return datasourceService.update(id, datasourceDTO, environmentId, Boolean.TRUE) + return datasourceService.updateDatasource(id, datasource, environmentId, Boolean.TRUE) + .map(updatedResource -> new ResponseDTO<>(HttpStatus.OK.value(), updatedResource, null)); + } + + @JsonView(Views.Public.class) + @PutMapping("/datasource-storages") + public Mono> updateDatasourceStorages(@RequestBody DatasourceStorageDTO datasourceStorageDTO, + @RequestHeader(name = FieldName.ENVIRONMENT_ID, required = false) String activeEnvironmentId) { + log.debug("Going to update datasource from datasource controller with id: {} and environmentId: {}", + datasourceStorageDTO.getDatasourceId(), datasourceStorageDTO.getEnvironmentId()); + + return datasourceService.updateDatasourceStorage(datasourceStorageDTO, activeEnvironmentId , Boolean.TRUE) .map(updatedResource -> new ResponseDTO<>(HttpStatus.OK.value(), updatedResource, null)); } @@ -102,11 +113,11 @@ public class DatasourceControllerCE { @JsonView(Views.Public.class) @PostMapping("/test") - public Mono> testDatasource(@RequestBody DatasourceDTO datasourceDTO, - @RequestHeader(name = FieldName.ENVIRONMENT_ID, required = false) String environmentId) { + public Mono> testDatasource(@RequestBody DatasourceStorageDTO datasourceStorageDTO, + @RequestHeader(name = FieldName.ENVIRONMENT_ID, required = false) String activeEnvironmentId) { - log.debug("Going to test the datasource with name: {} and id: {}", datasourceDTO.getName(), datasourceDTO.getId()); - return datasourceService.testDatasource(datasourceDTO, environmentId) + log.debug("Going to test the datasource with id: {}", datasourceStorageDTO.getDatasourceId()); + return datasourceService.testDatasource(datasourceStorageDTO, activeEnvironmentId) .map(testResult -> new ResponseDTO<>(HttpStatus.OK.value(), testResult, null)); } @@ -154,8 +165,8 @@ public class DatasourceControllerCE { @JsonView(Views.Public.class) @PostMapping(Url.MOCKS) - public Mono> createMockDataSet(@RequestBody MockDataSource mockDataSource, - @RequestHeader(name = FieldName.ENVIRONMENT_ID, required = false) String environmentId) { + public Mono> createMockDataSet(@RequestBody MockDataSource mockDataSource, + @RequestHeader(name = FieldName.ENVIRONMENT_ID, required = false) String environmentId) { return mockDataService.createMockDataSet(mockDataSource, environmentId) .map(datasource -> new ResponseDTO<>(HttpStatus.OK.value(), datasource, null)); } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/domains/DatasourceContextIdentifier.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/domains/DatasourceContextIdentifier.java index 18e97173fc..8659efe11a 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/domains/DatasourceContextIdentifier.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/domains/DatasourceContextIdentifier.java @@ -1,17 +1,55 @@ package com.appsmith.server.domains; -import com.appsmith.server.domains.ce.DatasourceContextIdentifierCE; +import lombok.AllArgsConstructor; +import lombok.Getter; import lombok.NoArgsConstructor; +import lombok.Setter; + +import static org.springframework.util.StringUtils.hasText; /** * This class is for generating keys for dsContext. * The object of this class will be used as keys for dsContext */ +@Getter +@Setter @NoArgsConstructor -public class DatasourceContextIdentifier extends DatasourceContextIdentifierCE { +@AllArgsConstructor +public class DatasourceContextIdentifier { - public DatasourceContextIdentifier(String datasourceId, String environmentId) { - super(datasourceId, environmentId); + private String datasourceId; + private String environmentId; + + @Override + public boolean equals(Object obj) { + if (obj == this) { + return true; + } + + if (!(obj instanceof DatasourceContextIdentifier keyObj)) { + return false; + } + + // if datasourceId is null for either of the objects then the keys can't be equal + return hasText(this.getDatasourceId()) + && this.getDatasourceId().equals(keyObj.getDatasourceId()) + && isEnvironmentIdEqual(keyObj.getEnvironmentId()); + } + + private boolean isEnvironmentIdEqual(String otherEnvironmentId) { + return hasText(this.getEnvironmentId()) && this.getEnvironmentId().equals(otherEnvironmentId); + } + + @Override + public int hashCode() { + int result = 0; + result = hasText(this.getDatasourceId()) ? this.getDatasourceId().hashCode() : result; + result = hasText(this.getEnvironmentId()) ? result * 31 + this.getEnvironmentId().hashCode() : result; + return result; + } + + public boolean isKeyValid() { + return hasText(this.getDatasourceId()) && hasText(this.getEnvironmentId()); } } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/domains/ce/DatasourceContextIdentifierCE.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/domains/ce/DatasourceContextIdentifierCE.java deleted file mode 100644 index 5411ab141f..0000000000 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/domains/ce/DatasourceContextIdentifierCE.java +++ /dev/null @@ -1,57 +0,0 @@ -package com.appsmith.server.domains.ce; - -import com.appsmith.server.domains.DatasourceContextIdentifier; -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; - -import static org.springframework.util.StringUtils.hasLength; - -/** - * This class is for generating keys for dsContext. - * The object of this class will be used as keys for dsContext - */ -@Getter -@Setter -@NoArgsConstructor -@AllArgsConstructor -public class DatasourceContextIdentifierCE { - - protected String datasourceId; - protected String environmentId; - - @Override - public boolean equals(Object obj) { - if (obj == this) { - return true; - } - - if (!(obj instanceof DatasourceContextIdentifier keyObj)) { - return false; - } - - // if datasourceId is null for either of the objects then the keys can't be equal - return hasLength(this.getDatasourceId()) - && this.getDatasourceId().equals(keyObj.getDatasourceId()) - && isEnvironmentIdEqual(keyObj.getEnvironmentId()); - } - - protected boolean isEnvironmentIdEqual(String otherEnvironmentId) { - return true; - } - - @Override - public int hashCode() { - int result = 0; - result = hasLength(this.getDatasourceId()) ? this.getDatasourceId().hashCode() : result; - result = hasLength(this.getDatasourceId()) ? result * 31 + this.getDatasourceId().hashCode() : result; - return result; - } - - public boolean isKeyValid() { - return hasLength(this.getDatasourceId()); - } - - -} diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/DatasourceAnalyticsUtils.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/DatasourceAnalyticsUtils.java index 2194302355..84f31de4c7 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/DatasourceAnalyticsUtils.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/DatasourceAnalyticsUtils.java @@ -26,6 +26,19 @@ public class DatasourceAnalyticsUtils { return analyticsProperties; } + public static Map getAnalyticsProperties(DatasourceStorage datasourceStroge) { + Map analyticsProperties = new HashMap<>(); + analyticsProperties.put("orgId", datasourceStroge.getWorkspaceId()); + analyticsProperties.put("pluginName", datasourceStroge.getPluginName()); + analyticsProperties.put("pluginId", datasourceStroge.getPluginId()); + analyticsProperties.put("dsName", datasourceStroge.getName()); + analyticsProperties.put("workspaceId", datasourceStroge.getWorkspaceId()); + analyticsProperties.put("isAutoGenerated", datasourceStroge.getIsAutoGenerated()); + analyticsProperties.put("dsIsTemplate", ObjectUtils.defaultIfNull(datasourceStroge.getIsTemplate(), "")); + analyticsProperties.put("dsIsMock", ObjectUtils.defaultIfNull(datasourceStroge.getIsMock(), "")); + return analyticsProperties; + } + public static Map getAnalyticsPropertiesForTestEventStatus (DatasourceStorage datasourceStorage, boolean status, Throwable e) { Map analyticsProperties = getAnalyticsPropertiesWithStorage(datasourceStorage); @@ -59,11 +72,8 @@ public class DatasourceAnalyticsUtils { } public static Map getAnalyticsPropertiesWithStorage(DatasourceStorage datasourceStorage) { - Datasource datasource = new Datasource(datasourceStorage); Map analyticsProperties = new HashMap<>(); - - analyticsProperties.putAll(getAnalyticsProperties(datasource)); - + analyticsProperties.putAll(getAnalyticsProperties(datasourceStorage)); analyticsProperties.put("pluginName", datasourceStorage.getPluginName()); analyticsProperties.put("dsName", datasourceStorage.getName()); analyticsProperties.put("envId", datasourceStorage.getEnvironmentId()); diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/DatasourceServiceCE.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/DatasourceServiceCE.java index bebaf212a1..9d222c3837 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/DatasourceServiceCE.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/DatasourceServiceCE.java @@ -1,10 +1,12 @@ package com.appsmith.server.services.ce; import com.appsmith.external.models.Datasource; +import com.appsmith.external.models.DatasourceDTO; +import com.appsmith.external.models.DatasourceStorage; +import com.appsmith.external.models.DatasourceStorageDTO; import com.appsmith.external.models.DatasourceTestResult; import com.appsmith.external.models.MustacheBindingToken; import com.appsmith.server.acl.AclPermission; -import com.appsmith.external.models.DatasourceDTO; import org.springframework.util.MultiValueMap; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @@ -19,12 +21,12 @@ public interface DatasourceServiceCE { Mono validateDatasource(Datasource datasource); /** - * @param datasourceDTO - The datasource which is about to be tested - * @param environmentId - environmentName, name of the environment on which the datasource is getting tested, + * @param datasourceStorageDTO - The datasourceStorageDTO which is about to be tested + * @param activeEnvironmentId - environmentId, name of the environment on which the datasource is getting tested, * this variable is unused in the CE version of the code. * @return Mono - result whether the datasource secures a valid connection with the remote DB */ - Mono testDatasource(DatasourceDTO datasourceDTO, String environmentId); + Mono testDatasource(DatasourceStorageDTO datasourceStorageDTO, String activeEnvironmentId); Mono findByNameAndWorkspaceId(String name, String workspaceId, Optional permission); @@ -44,10 +46,11 @@ public interface DatasourceServiceCE { * Retrieves all datasources based on input params, currently only workspaceId. * The retrieved datasources will contain configuration from the default environment, * for compatibility. + * * @param params * @return A flux of DatsourceDTO, which will change after API contracts gets updated */ - Flux getAllWithStorages(MultiValueMap params); + Flux getAllWithStorages(MultiValueMap params); Flux getAllByWorkspaceIdWithoutStorages(String workspaceId, Optional permission); @@ -66,13 +69,9 @@ public interface DatasourceServiceCE { Mono createWithoutPermissions(Datasource datasource); - Mono create(DatasourceDTO resource, String environmentId); + Mono updateDatasourceStorage(DatasourceStorageDTO datasourceStorageDTO, String activeEnvironmentId, Boolean IsUserRefreshedUpdate); - Mono update(String id, DatasourceDTO datasourceDTO, String environmentId); - - Mono update(String id, DatasourceDTO datasourceDTO, String environmentId, Boolean isUserRefreshedUpdate); - - Mono updateByEnvironmentId(String id, Datasource datasource, String environmentId); + Mono updateDatasource(String id, Datasource datasource, String activeEnvironmentId, Boolean isUserRefreshedUpdate); Mono archiveById(String id); @@ -86,4 +85,6 @@ public interface DatasourceServiceCE { // TODO: Remove the following snippet after client side API changes Mono getTrueEnvironmentId(String workspaceId, String environmentId); + + Datasource createDatasourceFromDatasourceStorage(DatasourceStorage datasourceStorage); } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/DatasourceServiceCEImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/DatasourceServiceCEImpl.java index b2349ecbc7..2ae2d131f1 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/DatasourceServiceCEImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/DatasourceServiceCEImpl.java @@ -3,6 +3,7 @@ package com.appsmith.server.services.ce; import com.appsmith.external.constants.AnalyticsEvents; import com.appsmith.external.helpers.MustacheHelper; import com.appsmith.external.models.Datasource; +import com.appsmith.external.models.DatasourceConfiguration; import com.appsmith.external.models.DatasourceDTO; import com.appsmith.external.models.DatasourceStorage; import com.appsmith.external.models.DatasourceStorageDTO; @@ -45,6 +46,7 @@ import org.springframework.util.StringUtils; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import reactor.core.scheduler.Scheduler; +import reactor.core.scheduler.Schedulers; import reactor.util.function.Tuple2; import reactor.util.function.Tuples; @@ -59,8 +61,12 @@ import java.util.Optional; import java.util.Set; import static com.appsmith.external.helpers.AppsmithBeanUtils.copyNestedNonNullProperties; +import static com.appsmith.server.helpers.CollectionUtils.isNullOrEmpty; import static com.appsmith.server.helpers.DatasourceAnalyticsUtils.getAnalyticsPropertiesForTestEventStatus; import static com.appsmith.server.repositories.BaseAppsmithRepositoryImpl.fieldName; +import static java.lang.Boolean.FALSE; +import static java.lang.Boolean.TRUE; +import static org.springframework.util.StringUtils.hasText; @Slf4j public class DatasourceServiceCEImpl implements DatasourceServiceCE { @@ -113,14 +119,6 @@ public class DatasourceServiceCEImpl implements DatasourceServiceCE { this.repository = repository; } - // TODO: Remove the following snippet after client side API changes - @Override - public Mono create(DatasourceDTO datasourceDTO, String environmentId) { - return convertToDatasource(datasourceDTO, environmentId) - .flatMap(datasource -> this.create(datasource)) - .flatMap(this::convertToDatasourceDTO); - } - @Override public Mono create(Datasource datasource) { return createEx(datasource, Optional.of(workspacePermission.getDatasourceCreatePermission())); @@ -135,30 +133,30 @@ public class DatasourceServiceCEImpl implements DatasourceServiceCE { private Mono createEx(@NotNull Datasource datasource, Optional permission) { // Validate incoming request String workspaceId = datasource.getWorkspaceId(); - if (workspaceId == null) { + if (!hasText(workspaceId)) { return Mono.error(new AppsmithException(AppsmithError.INVALID_PARAMETER, FieldName.WORKSPACE_ID)); } - if (datasource.getId() != null) { - return Mono.error(new AppsmithException(AppsmithError.INVALID_PARAMETER, FieldName.ID)); - } - if (datasource.getPluginId() == null) { + if (!hasText(datasource.getPluginId())) { return Mono.error(new AppsmithException(AppsmithError.INVALID_PARAMETER, FieldName.PLUGIN_ID)); } - if (!StringUtils.hasLength(datasource.getGitSyncId())) { + if (!hasText(datasource.getGitSyncId())) { datasource.setGitSyncId(datasource.getWorkspaceId() + "_" + new ObjectId()); } - if (datasource.getDatasourceStorages() == null || datasource.getDatasourceStorages().isEmpty()) { + + if (isNullOrEmpty(datasource.getDatasourceStorages())) { return Mono.error(new AppsmithException(AppsmithError.INVALID_PARAMETER, FieldName.DATASOURCE)); } + // Set this to null, datasource configuration has moved to datasourceStorage, we will stop storing this. + datasource.nullifyStorageReplicaFields(); Mono datasourceMono = Mono.just(datasource); // First check if this is an existing datasource or whether we need to create one - if (datasource.getId() == null) { + if (!hasText(datasource.getId())) { // We need to create the datasource as well // Determine valid name for datasource - if (!StringUtils.hasLength(datasource.getName())) { + if (!hasText(datasource.getName())) { datasourceMono = sequenceService .getNextAsSuffix(Datasource.class, " for workspace with _id : " + workspaceId) .map(sequenceNumber -> { @@ -169,7 +167,6 @@ public class DatasourceServiceCEImpl implements DatasourceServiceCE { datasourceMono = datasourceMono .map(datasource1 -> { // Everything we create needs to use configs from storage - datasource1.setDatasourceConfiguration(null); datasource1.setHasDatasourceStorage(true); return datasource1; }) @@ -181,36 +178,60 @@ public class DatasourceServiceCEImpl implements DatasourceServiceCE { .flatMap(savedDatasource -> analyticsService.sendCreateEvent(savedDatasource, getAnalyticsProperties(savedDatasource)) ); + } else { + log.debug("datasource with name: {} already exists, Only configuration(s) will be created", datasource.getName()); + datasourceMono = datasourceMono + .flatMap(datasource1 -> findById(datasource1.getId(), datasourcePermission.getEditPermission()) + .map(datasource2 -> { + datasource2.setDatasourceStorages(datasource1.getDatasourceStorages()); + return datasource2; + }) + .switchIfEmpty(Mono.error(new AppsmithException(AppsmithError.NO_RESOURCE_FOUND, FieldName.DATASOURCE))) + ); } return datasourceMono - .flatMap(datasource1 -> { - // In case this was a newly created datasource, we need to update datasource reference first - return Flux.fromIterable(datasource.getDatasourceStorages().values()) - .flatMap(datasourceStorageDTO -> { - DatasourceStorage datasourceStorage = new DatasourceStorage(datasourceStorageDTO); - datasourceStorage.prepareTransientFields(datasource1); - return getTrueEnvironmentId(workspaceId, datasourceStorageDTO.getEnvironmentId()) - .map(trueEnvironmentId -> { - datasourceStorage.setEnvironmentId(trueEnvironmentId); - return datasourceStorage; - }); - }) - .flatMap(datasourceStorage -> { - // Make sure that we are creating entries only if the id is not already populated - if (datasourceStorage.getId() == null) { - return datasourceStorageService.create(datasourceStorage); - } - return Mono.just(datasourceStorage); - }) - .map(datasourceStorage -> new DatasourceStorageDTO(datasourceStorage)) - .collectMap(datasourceStorageDTO -> datasourceStorageDTO.getEnvironmentId(), - datasourceStorageDTO -> datasourceStorageDTO) - .map(storages -> { - datasource1.setDatasourceStorages(storages); - return datasource1; - }); - }); + .flatMap(savedDatasource -> this.organiseDatasourceStorages(savedDatasource) + .flatMap(datasourceStorage -> { + // Make sure that we are creating entries only if the id is not already populated + if (datasourceStorage.getId() == null) { + return datasourceStorageService.create(datasourceStorage); + } + return Mono.just(datasourceStorage); + }) + .map(DatasourceStorageDTO::new) + .collectMap(DatasourceStorageDTO::getEnvironmentId) + .map(savedStorages -> { + savedDatasource.setDatasourceStorages(savedStorages); + return savedDatasource; + }) + ); + } + + // this requires an EE override multiple environments + protected Flux organiseDatasourceStorages(@NotNull Datasource savedDatasource) { + Map storages = savedDatasource.getDatasourceStorages(); + int datasourceStorageDTOsAllowed = 1; + if (storages.size() > datasourceStorageDTOsAllowed) { + //ideally an error should be thrown; however, since datasource has already been created, it needs be returned. + log.debug("datasource has got {} configurations, which is more than: {} for datasourceId: {}", + storages.size(), datasourceStorageDTOsAllowed, savedDatasource.getId()); + } + + Map storagesToBeSaved = new HashMap<>(); + + return Flux.fromIterable(storages.values()) + .flatMap(datasourceStorageDTO -> + this.getTrueEnvironmentId(savedDatasource.getWorkspaceId(), datasourceStorageDTO.getEnvironmentId()) + .map(trueEnvironmentId -> { + datasourceStorageDTO.setEnvironmentId(trueEnvironmentId); + DatasourceStorage datasourceStorage = new DatasourceStorage(datasourceStorageDTO); + datasourceStorage.prepareTransientFields(savedDatasource); + storagesToBeSaved.put(trueEnvironmentId, datasourceStorage); + return datasourceStorage; + }) + ) + .thenMany(Flux.fromIterable(storagesToBeSaved.values())); } private Mono generateAndSetDatasourcePolicies(Mono userMono, Datasource datasource, Optional permission) { @@ -228,71 +249,21 @@ public class DatasourceServiceCEImpl implements DatasourceServiceCE { }); } - // TODO: Remove the following snippet after client side API changes @Override - public Mono update(String id, DatasourceDTO datasourceDTO, String environmentId) { - return this.update(id, datasourceDTO, environmentId, Boolean.FALSE); - } - - @Override - public Mono update(String id, - DatasourceDTO datasourceDTO, - String environmentId, - Boolean isUserRefreshedUpdate) { - datasourceDTO.setId(id); // This is for payload without datasourceId & workspaceId - return convertToDatasource(datasourceDTO, environmentId) - .flatMap(datasource -> updateByEnvironmentId(id, datasource, environmentId, isUserRefreshedUpdate)) - .flatMap(datasource -> convertToDatasourceDTO(datasource)); - } - - @Override - public Mono updateByEnvironmentId(String id, Datasource datasource, String environmentId) { - // since there was no datasource update differentiator between server invoked due to refresh token, - // and user invoked. Hence the update is overloaded to provide the boolean for key diff. - // adding a default false value here, the value is true only when - // the user calls the update event from datasource controller, else it's false. - return updateByEnvironmentId(id, datasource, environmentId, Boolean.FALSE); - } - - private Mono updateByEnvironmentId(String id, - Datasource datasource, - String environmentId, - Boolean isUserRefreshedUpdate) { - if (id == null) { + public Mono updateDatasource(String id, Datasource datasource, String activeEnvironmentId, Boolean isUserRefreshedUpdate) { + if (!hasText(id)) { return Mono.error(new AppsmithException(AppsmithError.INVALID_PARAMETER, FieldName.ID)); } // Since policies are a server only concept, first set the empty set (set by constructor) to null datasource.setPolicies(null); + // this is important to avoid polluting the collection with configuration when we are saving the datasource + // check method docstring for description + datasource.nullifyStorageReplicaFields(); Mono datasourceMono = repository.findById(id, datasourcePermission.getEditPermission()) .switchIfEmpty(Mono.error(new AppsmithException(AppsmithError.NO_RESOURCE_FOUND, FieldName.DATASOURCE, id))); - - // Check if this is an update for only datasource related properties - if (!datasource.getDatasourceStorages().isEmpty()) { - // This is meant to be an update for storage - return datasourceMono - .flatMap(dbDatasource -> getTrueEnvironmentId(dbDatasource.getWorkspaceId(), environmentId) - .flatMap(trueEnvironmentId -> datasourceStorageService - .updateByDatasourceAndEnvironmentId(datasource, trueEnvironmentId, isUserRefreshedUpdate) - .map(datasourceStorage -> { - datasource.getDatasourceStorages() - .put(trueEnvironmentId, new DatasourceStorageDTO(datasourceStorage)); - copyNestedNonNullProperties(datasource, dbDatasource); - return dbDatasource; - }))) - .flatMap(savedDatasource -> { - Map analyticsProperties = getAnalyticsProperties(savedDatasource); - if (isUserRefreshedUpdate.equals(Boolean.TRUE)) { - analyticsProperties.put(FieldName.IS_DATASOURCE_UPDATE_USER_INVOKED_KEY, Boolean.TRUE); - } else { - analyticsProperties.put(FieldName.IS_DATASOURCE_UPDATE_USER_INVOKED_KEY, Boolean.FALSE); - } - return analyticsService.sendUpdateEvent(savedDatasource, analyticsProperties); - }); - } - // This is meant to be an update for just the datasource - like a rename return datasourceMono .map(dbDatasource -> { @@ -300,16 +271,60 @@ public class DatasourceServiceCEImpl implements DatasourceServiceCE { return dbDatasource; }) .flatMap(this::validateAndSaveDatasourceToRepository) + .map(savedDatasource -> { + //not required by client side in order to avoid updating it to a null storage, + // one alternative is that we find and send datasourceStorages along, but that is an expensive call + savedDatasource.setDatasourceStorages(null); + return savedDatasource; + }) .flatMap(savedDatasource -> { Map analyticsProperties = getAnalyticsProperties(savedDatasource); - if (isUserRefreshedUpdate.equals(Boolean.TRUE)) { - analyticsProperties.put(FieldName.IS_DATASOURCE_UPDATE_USER_INVOKED_KEY, Boolean.TRUE); - } else { - analyticsProperties.put(FieldName.IS_DATASOURCE_UPDATE_USER_INVOKED_KEY, Boolean.FALSE); - } + Boolean userInvokedUpdate = TRUE.equals(isUserRefreshedUpdate) ? TRUE: FALSE; + analyticsProperties.put(FieldName.IS_DATASOURCE_UPDATE_USER_INVOKED_KEY, userInvokedUpdate); return analyticsService.sendUpdateEvent(savedDatasource, analyticsProperties); }); + } + @Override + public Mono updateDatasourceStorage(@NotNull DatasourceStorageDTO datasourceStorageDTO, String activeEnvironmentId, Boolean isUserRefreshedUpdate) { + + String datasourceId = datasourceStorageDTO.getDatasourceId(); + String environmentId = datasourceStorageDTO.getEnvironmentId(); + + if (!hasText(datasourceId)) { + return Mono.error(new AppsmithException(AppsmithError.INVALID_PARAMETER, FieldName.DATASOURCE_ID)); + } + + if (!hasText(environmentId)) { + //ideally the error would be thrown, but we would only throw error when complete client side changes + // have been done for multiple-environments. For now this call will go through + log.debug("environmentId not found while updating datasource storage with datasourceId : {}", datasourceId); + } + + // querying for each of the datasource + Mono datasourceMonoCached = findById(datasourceId, datasourcePermission.getEditPermission()) + .switchIfEmpty(Mono.error(new AppsmithException(AppsmithError.NO_RESOURCE_FOUND, FieldName.DATASOURCE, datasourceId))); + + Mono trueEnvironmentIdMono = datasourceMonoCached + .flatMap(datasource -> getTrueEnvironmentId(datasource.getWorkspaceId(), environmentId)); + + return datasourceMonoCached + .zipWith(trueEnvironmentIdMono) + .flatMap(tuple2 -> { + Datasource dbDatasource = tuple2.getT1(); + String trueEnvironmentId = tuple2.getT2(); + + datasourceStorageDTO.setEnvironmentId(trueEnvironmentId); + DatasourceStorage datasourceStorage = new DatasourceStorage(datasourceStorageDTO); + datasourceStorage.prepareTransientFields(dbDatasource); + + return datasourceStorageService.updateDatasourceStorage(datasourceStorage, activeEnvironmentId, Boolean.TRUE) + .map(DatasourceStorageDTO::new) + .map(datasourceStorageDTO1 -> { + dbDatasource.getDatasourceStorages().put(trueEnvironmentId, datasourceStorageDTO1); + return dbDatasource; + }); + }); } @Override @@ -384,43 +399,66 @@ public class DatasourceServiceCEImpl implements DatasourceServiceCE { * the password from the db if its a saved datasource before testing. */ @Override - public Mono testDatasource(DatasourceDTO datasourceDTO, String environmentId) { - // the datasource has been created with datasourceStorageKey as output of getTrueEnvironmentId - Mono datasourceStorageMono = this - .getTrueEnvironmentId(datasourceDTO.getWorkspaceId(), environmentId) - .zipWhen(trueEnvironmentId -> convertToDatasource(datasourceDTO, trueEnvironmentId)) - .flatMap(tuple2 -> { - String trueEnvironmentId = tuple2.getT1(); - Datasource datasource = tuple2.getT2(); + public Mono testDatasource(DatasourceStorageDTO datasourceStorageDTO, String activeEnvironmentId) { - DatasourceStorage datasourceStorage = datasourceStorageService - .getDatasourceStorageFromDatasource(datasource, trueEnvironmentId); + DatasourceStorage datasourceStorage = new DatasourceStorage(datasourceStorageDTO); + Mono datasourceStorageMono; - if (datasource.getId() == null) { - return Mono.just(datasourceStorage); - } - // Check if we have execute access on this datasource - return this.findById(datasource.getId(), datasourcePermission.getExecutePermission()) - .flatMap(dbDatasource -> { - // Fetch any fields that maybe encrypted from the db if the datasource being tested does not have those fields set. - // This scenario would happen whenever an existing datasource is being tested and no changes are present in the - // encrypted field (because encrypted fields are not sent over the network after encryption back to the client - if (datasourceStorage.getDatasourceId() != null - && datasourceStorage.getDatasourceConfiguration() != null - && datasourceStorage.getDatasourceConfiguration().getAuthentication() != null) { - return datasourceStorageService.findByDatasourceAndEnvironmentId(dbDatasource, trueEnvironmentId) - .map(datasourceStorage1 -> { - copyNestedNonNullProperties(datasourceStorage, datasourceStorage1); - return datasourceStorage1; - }); - } - return Mono.just(datasourceStorage); - }) - .switchIfEmpty(Mono.error(new AppsmithException(AppsmithError.UNAUTHORIZED_ACCESS))); - }) - .flatMap(datasourceStorageService::checkEnvironment); + // Ideally there should also be a check for missing environmentId, + // however since we are falling back to default this step is not required here. + + // Cases where the datasource hasn't been saved yet + if (!hasText(datasourceStorage.getDatasourceId())) { + + if (!hasText(datasourceStorage.getWorkspaceId())) { + return Mono.error(new AppsmithException(AppsmithError.INVALID_PARAMETER, FieldName.WORKSPACE_ID)); + } + + datasourceStorageMono = getTrueEnvironmentId(datasourceStorage.getWorkspaceId(), datasourceStorage.getEnvironmentId()) + .map(trueEnvironmentId -> { + datasourceStorage.setEnvironmentId(trueEnvironmentId); + return datasourceStorage; + }); + } else { + + datasourceStorageMono = findById(datasourceStorage.getDatasourceId(), datasourcePermission.getExecutePermission()) + .zipWhen(dbDatasource -> getTrueEnvironmentId(dbDatasource.getWorkspaceId(), datasourceStorage.getEnvironmentId())) + .map(tuple2 -> { + Datasource datasource = tuple2.getT1(); + String trueEnvironmentId = tuple2.getT2(); + + datasourceStorage.setEnvironmentId(trueEnvironmentId); + datasourceStorage.prepareTransientFields(datasource); + return datasourceStorage; + }) + .flatMap(datasourceStorage1 -> { + + DatasourceConfiguration datasourceConfiguration = datasourceStorage1.getDatasourceConfiguration(); + if (datasourceConfiguration == null || datasourceConfiguration.getAuthentication() == null) { + return Mono.just(datasourceStorage); + } + + String datasourceId = datasourceStorage1.getDatasourceId(); + String trueEnvironmentId = datasourceStorage1.getEnvironmentId(); + // Fetch any fields that maybe encrypted from the db if the datasource being tested does not have those fields set. + // This scenario would happen whenever an existing datasource is being tested and no changes are present in the + // encrypted field (because encrypted fields are not sent over the network after encryption back to the client + + if (!hasText(datasourceStorage.getId())) { + return Mono.just(datasourceStorage); + } + + return datasourceStorageService.findStrictlyByDatasourceIdAndEnvironmentId(datasourceId, trueEnvironmentId) + .map(dbDatasourceStorage -> { + copyNestedNonNullProperties(datasourceStorage, dbDatasourceStorage); + return dbDatasourceStorage; + }); + }) + .switchIfEmpty(Mono.error(new AppsmithException(AppsmithError.UNAUTHORIZED_ACCESS))); + } return datasourceStorageMono + .flatMap(datasourceStorageService::checkEnvironment) .flatMap(this::verifyDatasourceAndTest); } @@ -518,12 +556,10 @@ public class DatasourceServiceCEImpl implements DatasourceServiceCE { } @Override - public Flux getAllWithStorages(MultiValueMap params) { + public Flux getAllWithStorages(MultiValueMap params) { String workspaceId = params.getFirst(fieldName(QDatasource.datasource.workspaceId)); if (workspaceId != null) { - return this.getAllByWorkspaceIdWithStorages(workspaceId, Optional.of(datasourcePermission.getReadPermission())) - // TODO: Remove the following snippet after client side API changes - .flatMap(datasource -> convertToDatasourceDTO(datasource)); + return this.getAllByWorkspaceIdWithStorages(workspaceId, Optional.of(datasourcePermission.getReadPermission())); } return Flux.error(new AppsmithException(AppsmithError.INVALID_PARAMETER, FieldName.WORKSPACE_ID)); @@ -538,8 +574,10 @@ public class DatasourceServiceCEImpl implements DatasourceServiceCE { public Flux getAllByWorkspaceIdWithStorages(String workspaceId, Optional permission) { return repository.findAllByWorkspaceId(workspaceId, permission) + .publishOn(Schedulers.boundedElastic()) .flatMap(datasource -> datasourceStorageService .findByDatasource(datasource) + .publishOn(Schedulers.boundedElastic()) .flatMap(datasourceStorageService::populateHintMessages) .map(DatasourceStorageDTO::new) .collectMap(DatasourceStorageDTO::getEnvironmentId) @@ -579,6 +617,7 @@ public class DatasourceServiceCEImpl implements DatasourceServiceCE { }) .flatMap(toDelete -> { return datasourceStorageService.findStrictlyByDatasourceId(toDelete.getId()) + .publishOn(Schedulers.boundedElastic()) .map(datasourceStorage -> { datasourceStorage.prepareTransientFields(toDelete); return datasourceStorage; @@ -735,4 +774,26 @@ public class DatasourceServiceCEImpl implements DatasourceServiceCE { return Mono.just(FieldName.UNUSED_ENVIRONMENT_ID); } + @Override + public Datasource createDatasourceFromDatasourceStorage(DatasourceStorage datasourceStorage) { + Datasource datasource = new Datasource(); + datasource.setId(datasourceStorage.getDatasourceId()); + datasource.setName(datasourceStorage.getName()); + datasource.setPluginId(datasourceStorage.getPluginId()); + datasource.setPluginName(datasourceStorage.getPluginName()); + datasource.setWorkspaceId(datasourceStorage.getWorkspaceId()); + datasource.setTemplateName(datasourceStorage.getTemplateName()); + datasource.setIsAutoGenerated(datasourceStorage.getIsAutoGenerated()); + datasource.setIsRecentlyCreated(datasourceStorage.getIsRecentlyCreated()); + datasource.setIsTemplate(datasourceStorage.getIsTemplate()); + datasource.setIsMock(datasourceStorage.getIsMock()); + datasource.setGitSyncId(datasourceStorage.getGitSyncId()); + + if (hasText(datasourceStorage.getEnvironmentId())) { + datasource.getDatasourceStorages() + .put(datasourceStorage.getEnvironmentId(), new DatasourceStorageDTO(datasourceStorage)); + } + + return datasource; + } } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/DatasourceStorageServiceCE.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/DatasourceStorageServiceCE.java index 937b54a00a..dd07136de9 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/DatasourceStorageServiceCE.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/DatasourceStorageServiceCE.java @@ -28,7 +28,9 @@ public interface DatasourceStorageServiceCE { Mono findStrictlyByDatasourceIdAndEnvironmentId(String datasourceId, String environmentId); - Mono updateByDatasourceAndEnvironmentId(Datasource datasource, String environmentId, Boolean isUserRefreshedUpdate); + Mono updateDatasourceStorage(DatasourceStorage datasourceStorage, + String activeEnvironmentId, + Boolean IsUserRefreshedUpdate); Mono validateDatasourceStorage(DatasourceStorage datasourceStorage, Boolean onlyConfiguration); Mono validateDatasourceConfiguration(DatasourceStorage datasourceStorage); diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/DatasourceStorageServiceCEImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/DatasourceStorageServiceCEImpl.java index 7f7a913148..6d2543819e 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/DatasourceStorageServiceCEImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/DatasourceStorageServiceCEImpl.java @@ -32,7 +32,8 @@ import java.util.Map; import java.util.Set; import static com.appsmith.external.helpers.AppsmithBeanUtils.copyNestedNonNullProperties; - +import static java.lang.Boolean.FALSE; +import static java.lang.Boolean.TRUE; @Slf4j public class DatasourceStorageServiceCEImpl implements DatasourceStorageServiceCE { @@ -146,15 +147,17 @@ public class DatasourceStorageServiceCEImpl implements DatasourceStorageServiceC } @Override - public Mono updateByDatasourceAndEnvironmentId(Datasource datasource, - String environmentId, - Boolean isUserRefreshedUpdate) { - return this.findByDatasourceAndEnvironmentId(datasource, environmentId) + public Mono updateDatasourceStorage(DatasourceStorage datasourceStorage, + String activeEnvironmentId, + Boolean isUserRefreshedUpdate) { + String datasourceId = datasourceStorage.getDatasourceId(); + String environmentId = datasourceStorage.getEnvironmentId(); + + return this.findStrictlyByDatasourceIdAndEnvironmentId(datasourceId, environmentId) + .flatMap(this::checkEnvironment) .map(dbStorage -> { - DatasourceStorageDTO datasourceStorageDTO = getDatasourceStorageDTOFromDatasource(datasource, environmentId); - DatasourceStorage datasourceStorage = new DatasourceStorage(datasourceStorageDTO); - datasourceStorage.prepareTransientFields(datasource); copyNestedNonNullProperties(datasourceStorage, dbStorage); + if (datasourceStorage.getDatasourceConfiguration() != null && datasourceStorage.getDatasourceConfiguration().getAuthentication() == null) { if (dbStorage.getDatasourceConfiguration() != null) { @@ -164,14 +167,12 @@ public class DatasourceStorageServiceCEImpl implements DatasourceStorageServiceC return dbStorage; }) .flatMap(this::validateAndSaveDatasourceStorageToRepository) - .flatMap(datasourceStorage -> { - Map analyticsProperties = getAnalyticsProperties(datasourceStorage); - if (isUserRefreshedUpdate.equals(Boolean.TRUE)) { - analyticsProperties.put(FieldName.IS_DATASOURCE_UPDATE_USER_INVOKED_KEY, Boolean.TRUE); - } else { - analyticsProperties.put(FieldName.IS_DATASOURCE_UPDATE_USER_INVOKED_KEY, Boolean.FALSE); - } - return analyticsService.sendUpdateEvent(datasourceStorage, analyticsProperties); + .flatMap(savedDatasourceStorage -> { + Map analyticsProperties = getAnalyticsProperties(savedDatasourceStorage); + Boolean isUserInvokedUpdate = TRUE.equals(isUserRefreshedUpdate) ? TRUE : FALSE; + + analyticsProperties.put(FieldName.IS_DATASOURCE_UPDATE_USER_INVOKED_KEY, isUserInvokedUpdate); + return analyticsService.sendUpdateEvent(savedDatasourceStorage, analyticsProperties); }) .flatMap(this::populateHintMessages); } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/MockDataServiceCE.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/MockDataServiceCE.java index bcbb74b481..b185c43234 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/MockDataServiceCE.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/MockDataServiceCE.java @@ -1,6 +1,6 @@ package com.appsmith.server.services.ce; -import com.appsmith.external.models.DatasourceDTO; +import com.appsmith.external.models.Datasource; import com.appsmith.server.dtos.MockDataDTO; import com.appsmith.server.dtos.MockDataSource; import reactor.core.publisher.Mono; @@ -8,5 +8,5 @@ import reactor.core.publisher.Mono; public interface MockDataServiceCE { Mono getMockDataSet(); - Mono createMockDataSet(MockDataSource mockDataSource, String environmentId); + Mono createMockDataSet(MockDataSource mockDataSource, String environmentId); } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/MockDataServiceCEImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/MockDataServiceCEImpl.java index dd548f8f92..854f0523f5 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/MockDataServiceCEImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/MockDataServiceCEImpl.java @@ -5,8 +5,6 @@ import com.appsmith.external.models.Connection; import com.appsmith.external.models.DBAuth; import com.appsmith.external.models.Datasource; import com.appsmith.external.models.DatasourceConfiguration; -import com.appsmith.external.models.DatasourceDTO; -import com.appsmith.external.models.DatasourceStorage; import com.appsmith.external.models.DatasourceStorageDTO; import com.appsmith.external.models.Endpoint; import com.appsmith.external.models.Property; @@ -98,7 +96,7 @@ public class MockDataServiceCEImpl implements MockDataServiceCE { } @Override - public Mono createMockDataSet(MockDataSource mockDataSource, String environmentId) { + public Mono createMockDataSet(MockDataSource mockDataSource, String environmentId) { Mono mockDataSet; if (cacheExpiryTime == null || !Instant.now().isBefore(cacheExpiryTime)) { @@ -123,19 +121,17 @@ public class MockDataServiceCEImpl implements MockDataServiceCE { datasource.setWorkspaceId(mockDataSource.getWorkspaceId()); datasource.setPluginId(mockDataSource.getPluginId()); datasource.setName(mockDataSource.getName()); - datasource.setIsConfigured(true); - datasource.setDatasourceConfiguration(datasourceConfiguration); + HashMap storages = new HashMap<>(); return datasourceService.getTrueEnvironmentId(mockDataSource.getWorkspaceId(), environmentId) .flatMap(trueEnvironmentId -> { - DatasourceStorage datasourceStorage = new DatasourceStorage(datasource, trueEnvironmentId); - storages.put(trueEnvironmentId, new DatasourceStorageDTO(datasourceStorage)); + storages.put(trueEnvironmentId, + new DatasourceStorageDTO(null, trueEnvironmentId, datasourceConfiguration)); datasource.setDatasourceStorages(storages); return addAnalyticsForMockDataCreation(name, mockDataSource.getWorkspaceId()) - .then(createSuffixedDatasource(datasource, trueEnvironmentId)) - .flatMap(datasource1 -> datasourceService.convertToDatasourceDTO(datasource)); + .then(createSuffixedDatasource(datasource, trueEnvironmentId)); }); }); diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/NewActionServiceCEImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/NewActionServiceCEImpl.java index 9dde862002..ee1e834628 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/NewActionServiceCEImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/NewActionServiceCEImpl.java @@ -568,6 +568,13 @@ public class NewActionServiceCEImpl extends BaseService data = this.getAnalyticsProperties(newAction1, datasource); final Map eventData = Map.of( @@ -578,7 +585,7 @@ public class NewActionServiceCEImpl extends BaseService datasourceStorageService.findByDatasourceAndEnvironmentId(datasource1, environmentId)) .switchIfEmpty(Mono.error(new AppsmithException(AppsmithError.NO_RESOURCE_FOUND, FieldName.DATASOURCE, datasourceId))) .flatMap(this::validateRequiredFieldsForGenericOAuth2) - .flatMap((datasource -> { - OAuth2 oAuth2 = (OAuth2) datasource.getDatasourceConfiguration().getAuthentication(); + .flatMap((datasourceStorage -> { + OAuth2 oAuth2 = (OAuth2) datasourceStorage.getDatasourceConfiguration().getAuthentication(); final String redirectUri = redirectHelper.getRedirectDomain(httpRequest.getHeaders()); // Adding basic uri components UriComponentsBuilder uriComponentsBuilder = UriComponentsBuilder @@ -535,32 +534,21 @@ public class AuthenticationServiceCEImpl implements AuthenticationServiceCE { .getPluginExecutor(pluginService.findById(datasourceStorage.getPluginId())) .flatMap(pluginExecutor -> ((PluginExecutor) pluginExecutor) .getDatasourceMetadata(datasourceStorage.getDatasourceConfiguration())) - .then(Mono.zip(Mono.just(datasourceStorage), accessTokenMono, projectIdMono, datasourceMonoCache)); + .then(Mono.zip(Mono.just(datasourceStorage), accessTokenMono, projectIdMono)); }); }) .flatMap(tuple -> { DatasourceStorage datasourceStorage = tuple.getT1(); + datasourceStorage.setUserPermissions(null); + datasourceStorage.setPolicies(null); String accessToken = tuple.getT2(); String projectID = tuple.getT3(); - // This datasource is coming fresh from db which has all the metadata and permissions, - // which is required by client side in order to allow users to generate queries otherwise the - // buttons are disabled. we will be sending this datasource itself which has the permissions as a fix - // ref: https://github.com/appsmithorg/appsmith/issues/23840 - Datasource datasource = tuple.getT4(); OAuth2ResponseDTO response = new OAuth2ResponseDTO(); response.setToken(accessToken); response.setProjectID(projectID); - //since we are fetching our datasource fresh from db we are guaranteed that datasource won't have any storages - datasource.getDatasourceStorages() - .put(datasourceStorage.getEnvironmentId(), new DatasourceStorageDTO(datasourceStorage)); + response.setDatasource(datasourceStorage); + return datasourceStorageService.save(datasourceStorage).thenReturn(response); - return datasourceService - .convertToDatasourceDTO(datasource) - .flatMap(datasourceDTO -> { - response.setDatasource(datasourceDTO); - return datasourceStorageService.save(datasourceStorage) - .thenReturn(response); - }); }) .onErrorMap(ConnectException.class, error -> new AppsmithException( diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/solutions/ce/CreateDBTablePageSolutionCEImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/solutions/ce/CreateDBTablePageSolutionCEImpl.java index 8f0f53d3f9..45b5e13622 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/solutions/ce/CreateDBTablePageSolutionCEImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/solutions/ce/CreateDBTablePageSolutionCEImpl.java @@ -573,7 +573,7 @@ public class CreateDBTablePageSolutionCEImpl implements CreateDBTablePageSolutio ActionConfiguration templateActionConfiguration = templateAction.getUnpublishedAction().getActionConfiguration(); actionDTO.setPluginId(datasourceStorage.getPluginId()); actionDTO.setId(null); - actionDTO.setDatasource(new Datasource(datasourceStorage)); + actionDTO.setDatasource(datasourceService.createDatasourceFromDatasourceStorage(datasourceStorage)); actionDTO.setPageId(pageId); actionDTO.setName(templateAction.getUnpublishedAction().getName()); actionDTO.setDefaultResources(templateAction.getDefaultResources()); diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/solutions/ce/ForkExamplesWorkspaceServiceCEImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/solutions/ce/ForkExamplesWorkspaceServiceCEImpl.java index b2ac08a812..374be9e56a 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/solutions/ce/ForkExamplesWorkspaceServiceCEImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/solutions/ce/ForkExamplesWorkspaceServiceCEImpl.java @@ -150,10 +150,10 @@ public class ForkExamplesWorkspaceServiceCEImpl implements ForkExamplesWorkspace workspace.setSlug(null); return workspaceService.createDefault(workspace, user); }) - .zipWhen(newWorkspace -> workspaceService.getDefaultEnvironmentId(newWorkspace.getId())) + .zipWhen(newWorkspace -> workspaceService.getDefaultEnvironmentId(templateWorkspaceId)) .flatMap(tuple2 -> { Workspace newWorkspace = tuple2.getT1(); - String targetEnvironmentId = tuple2.getT2(); + String sourceEnvironmentId = tuple2.getT2(); User userUpdate = new User(); userUpdate.setExamplesWorkspaceId(newWorkspace.getId()); @@ -164,7 +164,7 @@ public class ForkExamplesWorkspaceServiceCEImpl implements ForkExamplesWorkspace return Mono .when( userService.update(user.getId(), userUpdate), - forkApplications(newWorkspace.getId(), applicationFlux, datasourceFlux, targetEnvironmentId) + forkApplications(newWorkspace.getId(), applicationFlux, datasourceFlux, sourceEnvironmentId) ) .thenReturn(newWorkspace); }) diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/solutions/ce/ImportExportApplicationServiceCE.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/solutions/ce/ImportExportApplicationServiceCE.java index 8489cc4ae3..c9508f6d18 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/solutions/ce/ImportExportApplicationServiceCE.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/solutions/ce/ImportExportApplicationServiceCE.java @@ -1,7 +1,6 @@ package com.appsmith.server.solutions.ce; import com.appsmith.external.models.Datasource; -import com.appsmith.external.models.DatasourceDTO; import com.appsmith.server.constants.SerialiseApplicationObjective; import com.appsmith.server.domains.Application; import com.appsmith.server.dtos.ApplicationImportDTO; @@ -101,8 +100,6 @@ public interface ImportExportApplicationServiceCE { */ Mono restoreSnapshot(String workspaceId, ApplicationJson importedDoc, String applicationId, String branchName); - // TODO: Remove this temporary call post client side changes - Mono> findDatasourceDTOByApplicationId(String applicationId, String workspaceId); Mono> findDatasourceByApplicationId(String applicationId, String orgId); Mono getApplicationImportDTO(String applicationId, String workspaceId, Application application); diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/solutions/ce/ImportExportApplicationServiceCEImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/solutions/ce/ImportExportApplicationServiceCEImpl.java index 4a8fc5d20d..607e404c2f 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/solutions/ce/ImportExportApplicationServiceCEImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/solutions/ce/ImportExportApplicationServiceCEImpl.java @@ -11,7 +11,6 @@ import com.appsmith.external.models.BearerTokenAuth; import com.appsmith.external.models.DBAuth; import com.appsmith.external.models.Datasource; import com.appsmith.external.models.DatasourceConfiguration; -import com.appsmith.external.models.DatasourceDTO; import com.appsmith.external.models.DatasourceStorage; import com.appsmith.external.models.DatasourceStorageDTO; import com.appsmith.external.models.DecryptedSensitiveFields; @@ -990,7 +989,7 @@ public class ImportExportApplicationServiceCEImpl implements ImportExportApplica datasourceStorage.setDatasourceConfiguration(null); datasourceStorage.setPluginId(null); datasourceStorage.setEnvironmentId(environmentId); - Datasource newDatasource = new Datasource(datasourceStorage); + Datasource newDatasource = datasourceService.createDatasourceFromDatasourceStorage(datasourceStorage); newDatasource.setPolicies(null); copyNestedNonNullProperties(newDatasource, existingDatasource); @@ -1705,9 +1704,9 @@ public class ImportExportApplicationServiceCEImpl implements ImportExportApplica ) .map(dsName -> { datasourceStorage.setName(datasourceStorage.getName() + dsName); - return new Datasource(datasourceStorage); + return datasourceService.createDatasourceFromDatasourceStorage(datasourceStorage); }) - .switchIfEmpty(Mono.just(new Datasource(datasourceStorage))) + .switchIfEmpty(Mono.just(datasourceService.createDatasourceFromDatasourceStorage(datasourceStorage))) .flatMap(datasourceService::createWithoutPermissions); })); } @@ -1798,14 +1797,6 @@ public class ImportExportApplicationServiceCEImpl implements ImportExportApplica return null; } - @Override - public Mono> findDatasourceDTOByApplicationId(String applicationId, String workspaceId) { - return findDatasourceByApplicationId(applicationId, workspaceId) - .flatMapMany(Flux::fromIterable) - .flatMap(datasourceService::convertToDatasourceDTO) - .collectList(); - } - public Mono> findDatasourceByApplicationId(String applicationId, String workspaceId) { // TODO: Investigate further why datasourcePermission.getReadPermission() is not being used. Mono> listMono = datasourceService diff --git a/app/server/appsmith-server/src/test/java/com/appsmith/server/domains/DatasourceContextIdentifierTest.java b/app/server/appsmith-server/src/test/java/com/appsmith/server/domains/DatasourceContextIdentifierTest.java new file mode 100644 index 0000000000..c50c8c056a --- /dev/null +++ b/app/server/appsmith-server/src/test/java/com/appsmith/server/domains/DatasourceContextIdentifierTest.java @@ -0,0 +1,59 @@ +package com.appsmith.server.domains; + +import org.bson.types.ObjectId; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; + +public class DatasourceContextIdentifierTest { + + @Test + public void verifyDsMapKeyEquality() { + String dsId = new ObjectId().toHexString(); + String defaultEnvironmentId = new ObjectId().toHexString(); + DatasourceContextIdentifier keyObj = new DatasourceContextIdentifier(dsId, defaultEnvironmentId ); + DatasourceContextIdentifier keyObj1 = new DatasourceContextIdentifier(dsId, defaultEnvironmentId); + assertEquals(keyObj, keyObj1); + } + + @Test + public void verifyDsMapKeyNotEqual() { + String dsId = new ObjectId().toHexString(); + String dsId1 = new ObjectId().toHexString(); + + // with different datasourceId and null environment id + DatasourceContextIdentifier keyObj = new DatasourceContextIdentifier(dsId, null); + DatasourceContextIdentifier keyObj1 = new DatasourceContextIdentifier(dsId1, null); + assertNotEquals(keyObj, keyObj1); + + // with same datasource but null environment id + DatasourceContextIdentifier keyObj2 = new DatasourceContextIdentifier(dsId, null); + assertNotEquals(keyObj, keyObj2); + + // with same datasource but different environment id + String differentEnvironmentId = new ObjectId().toHexString(); + DatasourceContextIdentifier keyObj3 = new DatasourceContextIdentifier(dsId, differentEnvironmentId); + assertNotEquals(keyObj, keyObj3); + } + + @Test + public void verifyDsMapKeyNotEqualWhenBothDatasourceIdNull() { + String defaultEnvironmentId = new ObjectId().toHexString(); + DatasourceContextIdentifier keyObj = new DatasourceContextIdentifier(null, defaultEnvironmentId); + DatasourceContextIdentifier keyObj1 = new DatasourceContextIdentifier(null, defaultEnvironmentId); + assertNotEquals(keyObj, keyObj1); + } + + @Test + public void verifyDatasourceContextIdentifierHashCode() { + String sampleDatasourceId = new ObjectId().toHexString(); + String defaultEnvironmentId = new ObjectId().toHexString(); + DatasourceContextIdentifier datasourceContextIdentifier = + new DatasourceContextIdentifier(sampleDatasourceId, defaultEnvironmentId); + + int hashCode = sampleDatasourceId.hashCode() * 31 + defaultEnvironmentId.hashCode(); + assertThat(datasourceContextIdentifier.hashCode()).isEqualTo(hashCode); + } +} diff --git a/app/server/appsmith-server/src/test/java/com/appsmith/server/services/DatasourceContextServiceTest.java b/app/server/appsmith-server/src/test/java/com/appsmith/server/services/DatasourceContextServiceTest.java index aaea3669b4..c36a32ee4c 100644 --- a/app/server/appsmith-server/src/test/java/com/appsmith/server/services/DatasourceContextServiceTest.java +++ b/app/server/appsmith-server/src/test/java/com/appsmith/server/services/DatasourceContextServiceTest.java @@ -10,7 +10,6 @@ import com.appsmith.external.models.DatasourceStorageDTO; import com.appsmith.external.models.UpdatableConnection; import com.appsmith.external.plugins.PluginExecutor; import com.appsmith.external.services.EncryptionService; -import com.appsmith.server.constants.FieldName; import com.appsmith.server.domains.DatasourceContext; import com.appsmith.server.domains.DatasourceContextIdentifier; import com.appsmith.server.domains.Plugin; @@ -24,6 +23,7 @@ import com.appsmith.server.repositories.WorkspaceRepository; import com.appsmith.server.solutions.DatasourcePermission; import lombok.extern.slf4j.Slf4j; import org.bson.types.ObjectId; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mockito; @@ -33,12 +33,14 @@ import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.boot.test.mock.mockito.SpyBean; import org.springframework.security.test.context.support.WithUserDetails; import org.springframework.test.context.junit.jupiter.SpringExtension; +import org.springframework.util.StringUtils; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import reactor.test.StepVerifier; import java.util.HashMap; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; @@ -89,6 +91,26 @@ public class DatasourceContextServiceTest { @SpyBean DatasourceContextServiceImpl datasourceContextService; + String defaultEnvironmentId; + + String workspaceId; + + + @BeforeEach + @WithUserDetails(value = "api_user") + public void setup() { + User apiUser = userService.findByEmail("api_user").block(); + Workspace toCreate = new Workspace(); + toCreate.setName("DatasourceServiceTest"); + + if (!StringUtils.hasLength(workspaceId)) { + Workspace workspace = workspaceService.create(toCreate, apiUser, Boolean.FALSE).block(); + workspaceId = workspace.getId(); + defaultEnvironmentId = workspaceService.getDefaultEnvironmentId(workspaceId).block(); + } + } + + @Test @WithUserDetails(value = "api_user") public void testDatasourceCache_afterDatasourceDeleted_doesNotReturnOldConnection() { @@ -102,10 +124,10 @@ public class DatasourceContextServiceTest { Mockito.when(pluginExecutorHelper.getPluginExecutor(any())).thenReturn(Mono.just(spyMockPluginExecutor)); DatasourceStorage datasourceStorage = new DatasourceStorage(); - datasourceStorage.setEnvironmentId(FieldName.UNUSED_ENVIRONMENT_ID); + datasourceStorage.setEnvironmentId(defaultEnvironmentId); datasourceStorage.setDatasourceId("id1"); datasourceStorage.setDatasourceConfiguration(new DatasourceConfiguration()); - datasourceStorage.setWorkspaceId("workspaceId"); + datasourceStorage.setWorkspaceId(workspaceId); DatasourceContextIdentifier datasourceContextIdentifier = new DatasourceContextIdentifier(datasourceStorage.getDatasourceId(), null); @@ -119,7 +141,7 @@ public class DatasourceContextServiceTest { datasource.setWorkspaceId("workspaceId1"); datasource.setPluginId("mockPluginId"); HashMap storages = new HashMap<>(); - storages.put(FieldName.UNUSED_ENVIRONMENT_ID, new DatasourceStorageDTO(datasourceStorage)); + storages.put(defaultEnvironmentId, new DatasourceStorageDTO(datasourceStorage)); datasource.setDatasourceStorages(storages); doReturn(Mono.just(datasource)).when(datasourceRepository).findById("id1", datasourcePermission.getDeletePermission()); @@ -172,11 +194,15 @@ public class DatasourceContextServiceTest { authenticationDTO.setUsername(username); authenticationDTO.setPassword(password); datasourceConfiguration.setAuthentication(authenticationDTO); - datasource.setDatasourceConfiguration(datasourceConfiguration); datasource.setWorkspaceId(workspaceId); - DatasourceStorage datasourceStorage = new DatasourceStorage(datasource, defaultEnvironmentId); + + DatasourceStorageDTO datasourceStorageDTO = new DatasourceStorageDTO(); + datasourceStorageDTO.setDatasourceConfiguration(datasourceConfiguration); + datasourceStorageDTO.setEnvironmentId(defaultEnvironmentId); + datasourceStorageDTO.setIsConfigured(Boolean.TRUE); + HashMap storages = new HashMap<>(); - storages.put(defaultEnvironmentId, new DatasourceStorageDTO(datasourceStorage)); + storages.put(defaultEnvironmentId, datasourceStorageDTO); datasource.setDatasourceStorages(storages); final Datasource createdDatasource = pluginMono @@ -198,9 +224,9 @@ public class DatasourceContextServiceTest { DBAuth authentication = (DBAuth) savedDatasource.getDatasourceConfiguration().getAuthentication(); assertEquals(password, authentication.getPassword()); - DatasourceStorageDTO datasourceStorageDTO = createdDatasource.getDatasourceStorages() + DatasourceStorageDTO savedDatasourceStorageDTO = createdDatasource.getDatasourceStorages() .get(defaultEnvironmentId); - DBAuth encryptedAuthentication = (DBAuth) datasourceStorageDTO.getDatasourceConfiguration().getAuthentication(); + DBAuth encryptedAuthentication = (DBAuth) savedDatasourceStorageDTO.getDatasourceConfiguration().getAuthentication(); assertEquals(password, encryptionService.decryptString(encryptedAuthentication.getPassword())); }) .verifyComplete(); @@ -276,11 +302,11 @@ public class DatasourceContextServiceTest { doReturn(Mono.just("connection_1")).doReturn(Mono.just("connection_2")).when(spyMockPluginExecutor).datasourceCreate(any()); DatasourceStorage datasourceStorage = new DatasourceStorage(); - datasourceStorage.setEnvironmentId(FieldName.UNUSED_ENVIRONMENT_ID); + datasourceStorage.setEnvironmentId(defaultEnvironmentId); datasourceStorage.setDatasourceId("id2"); datasourceStorage.setDatasourceConfiguration(new DatasourceConfiguration()); - DatasourceContextIdentifier datasourceContextIdentifier = new DatasourceContextIdentifier(datasourceStorage.getDatasourceId(), "envId"); + DatasourceContextIdentifier datasourceContextIdentifier = new DatasourceContextIdentifier(datasourceStorage.getDatasourceId(), defaultEnvironmentId); Object monitor = new Object(); DatasourceContext dsContext1 = (DatasourceContext) datasourceContextService @@ -398,11 +424,12 @@ public class DatasourceContextServiceTest { doReturn(Mono.error(new RuntimeException("error"))).when(spyMockPluginExecutor).datasourceCreate(any()); DatasourceStorage datasourceStorage = new DatasourceStorage(); - datasourceStorage.setEnvironmentId(FieldName.UNUSED_ENVIRONMENT_ID); + datasourceStorage.setEnvironmentId(defaultEnvironmentId); datasourceStorage.setDatasourceId("error_datasource_1"); datasourceStorage.setDatasourceConfiguration(new DatasourceConfiguration()); - DatasourceContextIdentifier datasourceContextIdentifier = new DatasourceContextIdentifier(datasourceStorage.getDatasourceId(), FieldName.UNUSED_ENVIRONMENT_ID); + DatasourceContextIdentifier datasourceContextIdentifier = + new DatasourceContextIdentifier(datasourceStorage.getDatasourceId(), defaultEnvironmentId); Object monitor = new Object(); @@ -436,11 +463,12 @@ public class DatasourceContextServiceTest { .when(spyMockPluginExecutor).datasourceCreate(any()); DatasourceStorage datasourceStorage = new DatasourceStorage(); - datasourceStorage.setEnvironmentId(FieldName.UNUSED_ENVIRONMENT_ID); + datasourceStorage.setEnvironmentId(defaultEnvironmentId); datasourceStorage.setDatasourceId("error_datasource_2"); datasourceStorage.setDatasourceConfiguration(new DatasourceConfiguration()); - DatasourceContextIdentifier datasourceContextIdentifier = new DatasourceContextIdentifier(datasourceStorage.getDatasourceId(), "envId"); + DatasourceContextIdentifier datasourceContextIdentifier = + new DatasourceContextIdentifier(datasourceStorage.getDatasourceId(), defaultEnvironmentId); Object monitor = new Object(); @@ -462,27 +490,18 @@ public class DatasourceContextServiceTest { @Test - public void verifyDsMapKeyEquality() { - String dsId = new ObjectId().toHexString(); - DatasourceContextIdentifier keyObj = new DatasourceContextIdentifier(dsId, null); - DatasourceContextIdentifier keyObj1 = new DatasourceContextIdentifier(dsId, null); - assertEquals(keyObj, keyObj1); + @WithUserDetails(value = "api_user") + public void verifyInitialiseDatasourceContextReturningRightIdentifier() { + String sampleDatasourceId = new ObjectId().toHexString(); + DatasourceStorage datasourceStorage = new DatasourceStorage(); + datasourceStorage.setDatasourceId(sampleDatasourceId); + datasourceStorage.setEnvironmentId(defaultEnvironmentId); + + DatasourceContextIdentifier datasourceContextIdentifier = + datasourceContextService.initializeDatasourceContextIdentifier(datasourceStorage); + + assertThat(datasourceContextIdentifier.getDatasourceId()).isEqualTo(sampleDatasourceId); + assertThat(datasourceContextIdentifier.getEnvironmentId()).isEqualTo(defaultEnvironmentId); } - @Test - public void verifyDsMapKeyNotEqual() { - String dsId = new ObjectId().toHexString(); - String dsId1 = new ObjectId().toHexString(); - DatasourceContextIdentifier keyObj = new DatasourceContextIdentifier(dsId, null); - DatasourceContextIdentifier keyObj1 = new DatasourceContextIdentifier(dsId1, null); - assertNotEquals(keyObj, keyObj1); - } - - @Test - public void verifyDsMapKeyNotEqualWhenBothDatasourceIdNull() { - String envId = new ObjectId().toHexString(); - DatasourceContextIdentifier keyObj = new DatasourceContextIdentifier(null, envId); - DatasourceContextIdentifier keyObj1 = new DatasourceContextIdentifier(null, envId); - assertNotEquals(keyObj, keyObj1); - } } diff --git a/app/server/appsmith-server/src/test/java/com/appsmith/server/services/DatasourceServiceTest.java b/app/server/appsmith-server/src/test/java/com/appsmith/server/services/DatasourceServiceTest.java index f1fa7872e0..cc897f13ed 100644 --- a/app/server/appsmith-server/src/test/java/com/appsmith/server/services/DatasourceServiceTest.java +++ b/app/server/appsmith-server/src/test/java/com/appsmith/server/services/DatasourceServiceTest.java @@ -6,7 +6,6 @@ import com.appsmith.external.models.Connection; import com.appsmith.external.models.DBAuth; import com.appsmith.external.models.Datasource; import com.appsmith.external.models.DatasourceConfiguration; -import com.appsmith.external.models.DatasourceDTO; import com.appsmith.external.models.DatasourceStorage; import com.appsmith.external.models.DatasourceStorageDTO; import com.appsmith.external.models.DatasourceTestResult; @@ -26,6 +25,7 @@ import com.appsmith.server.domains.User; import com.appsmith.server.domains.Workspace; import com.appsmith.server.dtos.PageDTO; import com.appsmith.server.exceptions.AppsmithError; +import com.appsmith.server.exceptions.AppsmithErrorCode; import com.appsmith.server.exceptions.AppsmithException; import com.appsmith.server.helpers.MockPluginExecutor; import com.appsmith.server.helpers.PluginExecutorHelper; @@ -40,6 +40,7 @@ import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.boot.test.mock.mockito.SpyBean; import org.springframework.http.HttpMethod; import org.springframework.security.test.context.support.WithUserDetails; import org.springframework.test.annotation.DirtiesContext; @@ -77,7 +78,7 @@ public class DatasourceServiceTest { @Autowired DatasourceService datasourceService; - @Autowired + @SpyBean DatasourceStorageService datasourceStorageService; @Autowired @@ -245,10 +246,16 @@ public class DatasourceServiceTest { storages.put(defaultEnvironmentId, new DatasourceStorageDTO(datasourceStorage)); datasource.setDatasourceStorages(storages); + Mono pluginMono = pluginService.findByPackageName("restapi-plugin"); + Mono datasourceMono = pluginMono.flatMap(plugin -> { + datasource.setPluginId(plugin.getId()); + return datasourceService.create(datasource); + }); + StepVerifier - .create(datasourceService.create(datasource)) + .create(datasourceMono) .expectErrorMatches(throwable -> throwable instanceof AppsmithException && - throwable.getMessage().equals(AppsmithError.INVALID_PARAMETER.getMessage(FieldName.ID))) + throwable.getMessage().equals(AppsmithError.NO_RESOURCE_FOUND.getMessage(FieldName.DATASOURCE))) .verify(); } @@ -413,13 +420,10 @@ public class DatasourceServiceTest { sslDetails.setCertificateFile(new UploadedFile("ssl_cert_file_id", "")); connection.setSsl(sslDetails); datasourceConfiguration.setConnection(connection); - datasource.setDatasourceConfiguration(datasourceConfiguration); - DatasourceStorage datasourceStorage = new DatasourceStorage(datasource, defaultEnvironmentId); - HashMap storages = new HashMap<>(); - storages.put(defaultEnvironmentId, new DatasourceStorageDTO(datasourceStorage)); - datasource.setDatasourceStorages(storages); - datasource.setWorkspaceId(workspaceId); + HashMap storages = new HashMap<>(); + storages.put(defaultEnvironmentId, new DatasourceStorageDTO(null, defaultEnvironmentId, datasourceConfiguration)); + datasource.setDatasourceStorages(storages); Mono pluginMono = pluginService.findByName("Installed Plugin Name"); @@ -436,10 +440,10 @@ public class DatasourceServiceTest { ssl.getKeyFile().setName("ssl_key_file_id2"); connection1.setSsl(ssl); datasourceConfiguration1.setConnection(connection1); - datasource1.getDatasourceStorages().get(defaultEnvironmentId) - .setDatasourceConfiguration(datasourceConfiguration1); + + DatasourceStorageDTO datasourceStorageDTO = new DatasourceStorageDTO(datasource1.getId(), defaultEnvironmentId, datasourceConfiguration1); return datasourceService - .updateByEnvironmentId(datasource1.getId(), datasource1, defaultEnvironmentId); + .updateDatasourceStorage(datasourceStorageDTO, defaultEnvironmentId, Boolean.FALSE); }); StepVerifier @@ -524,7 +528,10 @@ public class DatasourceServiceTest { datasourceConfiguration1.setConnection(connection1); datasource1.setDatasourceConfiguration(datasourceConfiguration1); - return datasourceService.updateByEnvironmentId(datasource1.getId(), datasource1, defaultEnvironmentId); + DatasourceStorageDTO datasourceStorageDTO = + new DatasourceStorageDTO(datasource1.getId(), defaultEnvironmentId, datasourceConfiguration1); + return datasourceService + .updateDatasourceStorage(datasourceStorageDTO, defaultEnvironmentId, Boolean.FALSE); }); StepVerifier @@ -533,8 +540,12 @@ public class DatasourceServiceTest { assertThat(createdDatasource.getId()).isNotEmpty(); assertThat(createdDatasource.getPluginId()).isEqualTo(datasource.getPluginId()); assertThat(createdDatasource.getName()).isEqualTo(datasource.getName()); - assertThat(createdDatasource.getDatasourceConfiguration().getConnection().getSsl().getKeyFile().getName()).isEqualTo("ssl_key_file_id2"); - assertThat(createdDatasource.getDatasourceConfiguration().getAuthentication() instanceof OAuth2).isTrue(); + assertThat(createdDatasource.getDatasourceStorages().get(defaultEnvironmentId)).isNotNull(); + DatasourceConfiguration datasourceConfiguration1 = + createdDatasource.getDatasourceStorages().get(defaultEnvironmentId).getDatasourceConfiguration(); + + assertThat(datasourceConfiguration1.getConnection().getSsl().getKeyFile().getName()).isEqualTo("ssl_key_file_id2"); + assertThat(datasourceConfiguration1.getAuthentication() instanceof OAuth2).isTrue(); }) .verifyComplete(); } @@ -615,31 +626,31 @@ public class DatasourceServiceTest { Mono pluginMono = pluginService.findByPackageName("restapi-plugin"); Datasource datasource = new Datasource(); datasource.setName("test datasource name for test"); - DatasourceConfiguration datasourceConfiguration = new DatasourceConfiguration(); - datasourceConfiguration.setUrl("http://test.com"); - datasource.setDatasourceConfiguration(datasourceConfiguration); datasource.setWorkspaceId(workspaceId); - DatasourceStorage datasourceStorage = new DatasourceStorage(datasource, defaultEnvironmentId); + DatasourceConfiguration datasourceConfiguration = new DatasourceConfiguration(); + datasourceConfiguration.setUrl("http://test.com"); + HashMap storages = new HashMap<>(); - storages.put(defaultEnvironmentId, new DatasourceStorageDTO(datasourceStorage)); + storages.put(defaultEnvironmentId, new DatasourceStorageDTO(null, defaultEnvironmentId, datasourceConfiguration)); datasource.setDatasourceStorages(storages); - Mono datasourceDTOMono = pluginMono + Mono datasourceMono = pluginMono .map(plugin -> { datasource.setPluginId(plugin.getId()); return datasource; }) - .flatMap(datasourceService::create) - .flatMap(datasource2 -> datasourceService.convertToDatasourceDTO(datasource2)); + .flatMap(datasourceService::create); Mockito.when(pluginExecutorHelper.getPluginExecutor(Mockito.any())).thenReturn(Mono.just(new MockPluginExecutor())); - Mono testResultMono = datasourceDTOMono - .flatMap(datasource1 -> datasourceService.testDatasource(datasource1, defaultEnvironmentId)); + Mono testResultMono = datasourceMono + .flatMap(datasource1 -> { + DatasourceStorageDTO datasourceStorageDTO = datasource1.getDatasourceStorages().get(defaultEnvironmentId); + return datasourceService.testDatasource(datasourceStorageDTO, defaultEnvironmentId); + }); - StepVerifier - .create(testResultMono) + StepVerifier.create(testResultMono) .assertNext(testResult -> { assertThat(testResult).isNotNull(); assertThat(testResult.getInvalids()).isEmpty(); @@ -666,6 +677,7 @@ public class DatasourceServiceTest { Datasource datasource = new Datasource(); datasource.setName("test db datasource empty"); datasource.setWorkspaceId(workspaceId); + DatasourceConfiguration datasourceConfiguration = new DatasourceConfiguration(); Connection connection = new Connection(); connection.setMode(Connection.Mode.READ_ONLY); @@ -680,28 +692,25 @@ public class DatasourceServiceTest { auth.setUsername("test"); auth.setPassword("test"); datasourceConfiguration.setAuthentication(auth); - datasource.setDatasourceConfiguration(datasourceConfiguration); - datasource.setWorkspaceId(workspaceId); - DatasourceStorage datasourceStorage = new DatasourceStorage(datasource, defaultEnvironmentId); HashMap storages = new HashMap<>(); - storages.put(defaultEnvironmentId, new DatasourceStorageDTO(datasourceStorage)); + storages.put(defaultEnvironmentId, new DatasourceStorageDTO(null, defaultEnvironmentId, datasourceConfiguration)); datasource.setDatasourceStorages(storages); - Mono datasourceDTOMono = pluginMono + Mono datasourceMono = pluginMono .map(plugin -> { datasource.setPluginId(plugin.getId()); return datasource; }) - .flatMap(datasourceService::create) - .flatMap(datasource2 -> datasourceService.convertToDatasourceDTO(datasource2)); + .flatMap(datasourceService::create); Mockito.when(pluginExecutorHelper.getPluginExecutor(Mockito.any())).thenReturn(Mono.just(new MockPluginExecutor())); - Mono testResultMono = datasourceDTOMono + Mono testResultMono = datasourceMono .flatMap(datasource1 -> { - ((DBAuth) datasource1.getDatasourceConfiguration().getAuthentication()).setPassword(null); - return datasourceService.testDatasource(datasource1, defaultEnvironmentId); + DatasourceStorageDTO datasourceStorageDTO = datasource1.getDatasourceStorages().get(defaultEnvironmentId); + ((DBAuth) datasourceStorageDTO.getDatasourceConfiguration().getAuthentication()).setPassword(null); + return datasourceService.testDatasource(datasourceStorageDTO, defaultEnvironmentId); }); StepVerifier @@ -1077,7 +1086,11 @@ public class DatasourceServiceTest { datasourceConfiguration.setAuthentication(partialAuthenticationDTO); original.getDatasourceStorages().get(defaultEnvironmentId) .setDatasourceConfiguration(datasourceConfiguration); - return datasourceService.updateByEnvironmentId(original.getId(), original, defaultEnvironmentId); + + DatasourceStorageDTO datasourceStorageDTO = + new DatasourceStorageDTO(original.getId(), defaultEnvironmentId, datasourceConfiguration); + return datasourceService + .updateDatasourceStorage(datasourceStorageDTO, defaultEnvironmentId, Boolean.FALSE); }); StepVerifier @@ -1299,33 +1312,33 @@ public class DatasourceServiceTest { Mono pluginMono = pluginService.findByPackageName("restapi-plugin"); Datasource datasource = new Datasource(); datasource.setName("testName 1"); + datasource.setWorkspaceId(workspaceId); + DatasourceConfiguration datasourceConfiguration = new DatasourceConfiguration(); Endpoint endpoint = new Endpoint("http://localhost", 0L); datasourceConfiguration.setEndpoints(new ArrayList<>()); datasourceConfiguration.getEndpoints().add(endpoint); - datasource.setDatasourceConfiguration(datasourceConfiguration); - datasource.setWorkspaceId(workspaceId); - DatasourceStorage datasourceStorage = new DatasourceStorage(datasource, defaultEnvironmentId); HashMap storages = new HashMap<>(); - storages.put(defaultEnvironmentId, new DatasourceStorageDTO(datasourceStorage)); + storages.put(defaultEnvironmentId, new DatasourceStorageDTO(null, defaultEnvironmentId, datasourceConfiguration)); datasource.setDatasourceStorages(storages); - Mono datasourceDTOMono = pluginMono + Mono datasourceMono = pluginMono .map(plugin -> { datasource.setPluginId(plugin.getId()); return datasource; }) - .flatMap(datasourceService::create) - .flatMap(datasource2 -> datasourceService.convertToDatasourceDTO(datasource2)); + .flatMap(datasourceService::create); Mockito.when(pluginExecutorHelper.getPluginExecutor(Mockito.any())).thenReturn(Mono.just(new MockPluginExecutor())); - Mono testResultMono = datasourceDTOMono - .flatMap(datasource1 -> datasourceService.testDatasource(datasource1, defaultEnvironmentId)); + Mono testResultMono = datasourceMono + .flatMap(datasource1 -> { + DatasourceStorageDTO datasourceStorageDTO = datasource1.getDatasourceStorages().get(defaultEnvironmentId); + return datasourceService.testDatasource(datasourceStorageDTO, defaultEnvironmentId); + }); - StepVerifier - .create(testResultMono) + StepVerifier.create(testResultMono) .assertNext(testResult -> { assertThat(testResult).isNotNull(); assertThat(testResult.getInvalids()).isEmpty(); @@ -1437,7 +1450,11 @@ public class DatasourceServiceTest { datasourceConfiguration1.setConnection(connection1); datasourceConfiguration1.setUrl("http://localhost"); datasource1.getDatasourceStorages().get(defaultEnvironmentId).setDatasourceConfiguration(datasourceConfiguration1); - return datasourceService.updateByEnvironmentId(datasource1.getId(), datasource1, defaultEnvironmentId); + + DatasourceStorageDTO datasourceStorageDTO = + new DatasourceStorageDTO(datasource1.getId(), defaultEnvironmentId, datasourceConfiguration1); + return datasourceService + .updateDatasourceStorage(datasourceStorageDTO, defaultEnvironmentId, Boolean.FALSE); }); StepVerifier @@ -1544,8 +1561,11 @@ public class DatasourceServiceTest { datasourceConfiguration1.getEndpoints().add(endpoint); datasource1.getDatasourceStorages().get(defaultEnvironmentId) .setDatasourceConfiguration(datasourceConfiguration1); + + DatasourceStorageDTO datasourceStorageDTO = + new DatasourceStorageDTO(datasource1.getId(), defaultEnvironmentId, datasourceConfiguration1); return datasourceService - .updateByEnvironmentId(datasource1.getId(), datasource1, defaultEnvironmentId); + .updateDatasourceStorage(datasourceStorageDTO, defaultEnvironmentId, Boolean.FALSE); }); StepVerifier @@ -1582,6 +1602,7 @@ public class DatasourceServiceTest { Datasource datasource = new Datasource(); datasource.setName("NPE check"); datasource.setWorkspaceId(workspaceId); + DatasourceConfiguration datasourceConfiguration = new DatasourceConfiguration(); datasourceConfiguration.setEndpoints(new ArrayList<>()); Endpoint nullEndpoint = null; @@ -1589,10 +1610,9 @@ public class DatasourceServiceTest { Endpoint nullHost = new Endpoint(null, 0L); datasourceConfiguration.getEndpoints().add(nullHost); - datasource.setDatasourceConfiguration(datasourceConfiguration); - DatasourceStorage datasourceStorage = new DatasourceStorage(datasource, defaultEnvironmentId); HashMap storages = new HashMap<>(); - storages.put(defaultEnvironmentId, new DatasourceStorageDTO(datasourceStorage)); + storages.put(defaultEnvironmentId, + new DatasourceStorageDTO(null, defaultEnvironmentId, datasourceConfiguration)); datasource.setDatasourceStorages(storages); Mono datasourceMono = pluginMono @@ -1602,10 +1622,11 @@ public class DatasourceServiceTest { }) .flatMap(datasourceService::create); - StepVerifier - .create(datasourceMono) + StepVerifier.create(datasourceMono) .assertNext(createdDatasource -> { - assertThat(createdDatasource.getMessages()).isEmpty(); + DatasourceStorageDTO datasourceStorageDTO = + createdDatasource.getDatasourceStorages().get(defaultEnvironmentId); + assertThat(datasourceStorageDTO.getMessages()).isEmpty(); }) .verifyComplete(); } @@ -1631,7 +1652,7 @@ public class DatasourceServiceTest { MultiValueMap params = new LinkedMultiValueMap<>(); params.add(fieldName(QDatasource.datasource.workspaceId), workspaceId); - Mono> listMono = datasourceService.getAllWithStorages(params).collectList(); + Mono> listMono = datasourceService.getAllWithStorages(params).collectList(); StepVerifier.create(listMono) .assertNext(datasources -> { @@ -1640,11 +1661,11 @@ public class DatasourceServiceTest { assertThat(datasources) .allMatch(datasourceDTO -> Set.of("A", "B", "C", "D").contains(datasourceDTO.getName())); - datasources.stream().forEach(datasourceDTO -> { - if (Set.of("A", "B", "C").contains(datasourceDTO.getName())) { - assertThat(datasourceDTO.getIsRecentlyCreated()).isTrue(); + datasources.stream().forEach(datasource -> { + if (Set.of("A", "B", "C").contains(datasource.getName())) { + assertThat(datasource.getIsRecentlyCreated()).isTrue(); } else { - assertThat(datasourceDTO.getIsRecentlyCreated()).isNull(); + assertThat(datasource.getIsRecentlyCreated()).isNull(); } }); }) @@ -1667,4 +1688,163 @@ public class DatasourceServiceTest { Datasource createdDatasource = datasourceService.create(datasource).block(); return createdDatasource; } + + private Datasource createDatasourceObject(String name, String workspaceId, String pluginName) { + Datasource datasource = new Datasource(); + datasource.setName(name); + datasource.setWorkspaceId(workspaceId); + + Plugin plugin = pluginService.findByPackageName(pluginName).block(); + datasource.setPluginName(pluginName); + datasource.setPluginId(plugin.getId()); + + return datasource; + } + + @Test + @WithUserDetails(value = "api_user") + public void getErrorOnCreatingEmptyDatasource() { + Datasource datasource = createDatasourceObject("testDs", workspaceId, "postgres-plugin"); + Mono datasourceMono = datasourceService.create(datasource); + + StepVerifier.create(datasourceMono) + .verifyErrorSatisfies(error -> { + assertThat(error).isInstanceOf(AppsmithException.class); + assertThat(((AppsmithException) error).getAppErrorCode()).isEqualTo(AppsmithErrorCode.INVALID_PARAMETER.getCode()); + }); + } + + public DatasourceStorageDTO generateSampleDatasourceStorageDTO() { + DatasourceConfiguration datasourceConfiguration = new DatasourceConfiguration(); + Endpoint endpoint = new Endpoint("https://sample.endpoint", 5432L); + DBAuth dbAuth = new DBAuth(); + dbAuth.setPassword("password"); + dbAuth.setUsername("username"); + dbAuth.setDatabaseName("databaseName"); + + datasourceConfiguration.setEndpoints(List.of(endpoint)); + datasourceConfiguration.setAuthentication(dbAuth); + return new DatasourceStorageDTO(null, defaultEnvironmentId, datasourceConfiguration); + } + + @Test + @WithUserDetails(value = "api_user") + public void verifyOnlyOneStorageIsSaved() { + Mockito.when(pluginExecutorHelper.getPluginExecutor(Mockito.any())) + .thenReturn(Mono.just(new MockPluginExecutor())); + + Datasource datasource = createDatasourceObject("testDs", workspaceId, "postgres-plugin"); + datasource.getDatasourceStorages().put("randomName", generateSampleDatasourceStorageDTO()); + datasource.getDatasourceStorages().put("randomName2", generateSampleDatasourceStorageDTO()); + + Mono datasourceMono = datasourceService.create(datasource); + + StepVerifier.create(datasourceMono) + .assertNext(dbDatasource -> { + assertThat(dbDatasource.getDatasourceStorages().size()).isEqualTo(1); + assertThat(dbDatasource.getDatasourceStorages().get(defaultEnvironmentId)).isNotNull(); + DatasourceStorageDTO datasourceStorageDTO = dbDatasource.getDatasourceStorages().get(defaultEnvironmentId); + assertThat(datasourceStorageDTO.getDatasourceId()).isEqualTo(dbDatasource.getId()); + assertThat(datasourceStorageDTO.getEnvironmentId()).isEqualTo(defaultEnvironmentId); + }) + .verifyComplete(); + } + + @Test + @WithUserDetails(value = "api_user") + public void verifyUpdateNameReturnsNullStorages() { + Mockito.when(pluginExecutorHelper.getPluginExecutor(Mockito.any())) + .thenReturn(Mono.just(new MockPluginExecutor())); + + Datasource datasource = createDatasourceObject("testDs", workspaceId, "postgres-plugin"); + datasource.getDatasourceStorages().put(defaultEnvironmentId, generateSampleDatasourceStorageDTO()); + Datasource createdDatasource = datasourceService.create(datasource).block(); + + createdDatasource.setName("renamedDs"); + Mono datasourceMono = datasourceService.updateDatasource(createdDatasource.getId(), createdDatasource, defaultEnvironmentId, Boolean.FALSE); + + StepVerifier.create(datasourceMono) + .assertNext(dbDatasource -> { + assertThat(dbDatasource.getDatasourceStorages()).isNull(); + assertThat(dbDatasource.getName()).isEqualTo("renamedDs"); + }) + .verifyComplete(); + } + + @Test + @WithUserDetails(value = "api_user") + public void verifyUpdateDatasourceStorageWithoutDatasourceId() { + Mockito.when(pluginExecutorHelper.getPluginExecutor(Mockito.any())) + .thenReturn(Mono.just(new MockPluginExecutor())); + + Datasource datasource = createDatasourceObject("testDs", workspaceId, "postgres-plugin"); + datasource.getDatasourceStorages().put(defaultEnvironmentId, generateSampleDatasourceStorageDTO()); + Datasource createdDatasource = datasourceService.create(datasource).block(); + + // this doesn't have the datasourceId yet + DatasourceStorageDTO sampleDatasourceStorageDTO = generateSampleDatasourceStorageDTO(); + sampleDatasourceStorageDTO.setWorkspaceId(createdDatasource.getWorkspaceId()); + sampleDatasourceStorageDTO.setPluginId(createdDatasource.getPluginId()); + + Mono datasourceMono = datasourceService.updateDatasourceStorage(sampleDatasourceStorageDTO, defaultEnvironmentId, Boolean.FALSE); + + StepVerifier.create(datasourceMono) + .verifyErrorSatisfies(error -> { + assertThat(error).isInstanceOf(AppsmithException.class); + assertThat(((AppsmithException) error).getAppErrorCode()).isEqualTo(AppsmithErrorCode.INVALID_PARAMETER.getCode()); + }); + } + + @Test + @WithUserDetails(value = "api_user") + public void verifyTestDatasourceWithSavedDatasourceButNoDatasourceStorageSucceeds() { + Datasource datasource = createDatasourceObject("sampleDatasource", workspaceId, "postgres-plugin"); + DatasourceStorageDTO datasourceStorageDTO = generateSampleDatasourceStorageDTO(); + datasource.getDatasourceStorages().put(defaultEnvironmentId, datasourceStorageDTO); + + Mockito.when(pluginExecutorHelper.getPluginExecutor(Mockito.any())) + .thenReturn(Mono.just(new MockPluginExecutor())).thenReturn(Mono.just(new MockPluginExecutor())); + DatasourceStorage datasourceStorage = new DatasourceStorage(datasourceStorageDTO); + Mockito.doReturn(Mono.just(datasourceStorage)).when(datasourceStorageService).create(Mockito.any()); + Datasource dbDatasource = datasourceService.create(datasource).block(); + + assertThat(dbDatasource.getId()).isNotNull(); + assertThat(dbDatasource.getDatasourceStorages()).isNotNull(); + assertThat(dbDatasource.getDatasourceStorages().get(defaultEnvironmentId)).isNotNull(); + assertThat(dbDatasource.getDatasourceStorages().get(defaultEnvironmentId).getId()).isNull(); + + datasourceStorageDTO.setDatasourceId(dbDatasource.getId()); + datasourceStorageDTO.setWorkspaceId(workspaceId); + datasourceStorageDTO.setPluginId(dbDatasource.getPluginId()); + + Mockito.when(pluginExecutorHelper.getPluginExecutor(Mockito.any())).thenReturn(Mono.just(new MockPluginExecutor())); + Mono testResultMono = datasourceService.testDatasource(datasourceStorageDTO, defaultEnvironmentId); + + StepVerifier.create(testResultMono) + .assertNext(testResult -> { + assertThat(testResult).isNotNull(); + assertThat(testResult.getInvalids()).isEmpty(); + }) + .verifyComplete(); + } + + @Test + @WithUserDetails(value = "api_user") + public void verifyTestDatasourceWithoutSavedDatasource() { + Datasource datasource = createDatasourceObject("sampleDatasource", workspaceId, "postgres-plugin"); + DatasourceStorageDTO datasourceStorageDTO = generateSampleDatasourceStorageDTO(); + + datasourceStorageDTO.setWorkspaceId(workspaceId); + datasourceStorageDTO.setPluginId(datasource.getPluginId()); + + Mockito.when(pluginExecutorHelper.getPluginExecutor(Mockito.any())).thenReturn(Mono.just(new MockPluginExecutor())); + Mono testResultMono = datasourceService.testDatasource(datasourceStorageDTO, defaultEnvironmentId); + + StepVerifier.create(testResultMono) + .assertNext(testResult -> { + assertThat(testResult).isNotNull(); + assertThat(testResult.getInvalids()).isEmpty(); + }) + .verifyComplete(); + } } diff --git a/app/server/appsmith-server/src/test/java/com/appsmith/server/services/MockDataServiceTest.java b/app/server/appsmith-server/src/test/java/com/appsmith/server/services/MockDataServiceTest.java index 9a2935f2da..7944cc2c0b 100644 --- a/app/server/appsmith-server/src/test/java/com/appsmith/server/services/MockDataServiceTest.java +++ b/app/server/appsmith-server/src/test/java/com/appsmith/server/services/MockDataServiceTest.java @@ -1,7 +1,9 @@ package com.appsmith.server.services; import com.appsmith.external.models.DBAuth; -import com.appsmith.external.models.DatasourceDTO; +import com.appsmith.external.models.Datasource; +import com.appsmith.external.models.DatasourceConfiguration; +import com.appsmith.external.models.DatasourceStorageDTO; import com.appsmith.external.models.Policy; import com.appsmith.server.domains.Application; import com.appsmith.server.domains.PermissionGroup; @@ -153,7 +155,7 @@ public class MockDataServiceTest { mockDataSource.setPluginId(pluginMono.getId()); Mono workspaceResponse = workspaceService.findById(workspaceId, READ_WORKSPACES); - String environmentId = workspaceService.getDefaultEnvironmentId(workspaceId).block(); + String defaultEnvironmentId = workspaceService.getDefaultEnvironmentId(workspaceId).block(); List permissionGroups = workspaceResponse .flatMapMany(savedWorkspace -> { @@ -176,7 +178,7 @@ public class MockDataServiceTest { .findFirst().get(); StepVerifier - .create(mockDataService.createMockDataSet(mockDataSource, environmentId)) + .create(mockDataService.createMockDataSet(mockDataSource, defaultEnvironmentId)) .assertNext(createdDatasource -> { assertThat(createdDatasource.getId()).isNotEmpty(); assertThat(createdDatasource.getPluginId()).isEqualTo(pluginMono.getId()); @@ -191,11 +193,13 @@ public class MockDataServiceTest { .permissionGroups(Set.of(adminPermissionGroup.getId(), developerPermissionGroup.getId(), viewerPermissionGroup.getId())) .build(); - DBAuth auth = (DBAuth) createdDatasource.getDatasourceConfiguration().getAuthentication(); + DatasourceStorageDTO createdDatasourceStorageDTO = createdDatasource.getDatasourceStorages().get(defaultEnvironmentId); + DatasourceConfiguration datasourceConfiguration = createdDatasourceStorageDTO.getDatasourceConfiguration(); + DBAuth auth = (DBAuth) datasourceConfiguration.getAuthentication(); assertThat(createdDatasource.getPolicies()).isNotEmpty(); assertThat(createdDatasource.getPolicies()).containsAll(Set.of(manageDatasourcePolicy, readDatasourcePolicy, executeDatasourcePolicy)); - assertThat(createdDatasource.getDatasourceConfiguration().getProperties().get(0).getValue()).isEqualTo("Yes"); - assertThat(createdDatasource.getDatasourceConfiguration().getProperties().get(0).getKey()).isEqualTo("Use mongo connection string URI"); + assertThat(datasourceConfiguration.getProperties().get(0).getValue()).isEqualTo("Yes"); + assertThat(datasourceConfiguration.getProperties().get(0).getKey()).isEqualTo("Use mongo connection string URI"); assertThat(auth.getDatabaseName()).isEqualTo("movies"); assertThat(auth.getUsername()).isEqualTo("mockdb-admin"); Assertions.assertTrue(createdDatasource.getIsMock()); @@ -218,7 +222,7 @@ public class MockDataServiceTest { mockDataSource.setPluginId(pluginMono.getId()); Mono workspaceResponse = workspaceService.findById(workspaceId, READ_WORKSPACES); - String environmentId = workspaceService.getDefaultEnvironmentId(workspaceId).block(); + String defaultEnvironmentId = workspaceService.getDefaultEnvironmentId(workspaceId).block(); List permissionGroups = workspaceResponse .flatMapMany(savedWorkspace -> { @@ -241,7 +245,7 @@ public class MockDataServiceTest { .findFirst().get(); StepVerifier - .create(mockDataService.createMockDataSet(mockDataSource, environmentId)) + .create(mockDataService.createMockDataSet(mockDataSource, defaultEnvironmentId)) .assertNext(createdDatasource -> { assertThat(createdDatasource.getId()).isNotEmpty(); assertThat(createdDatasource.getPluginId()).isEqualTo(pluginMono.getId()); @@ -256,7 +260,8 @@ public class MockDataServiceTest { .permissionGroups(Set.of(adminPermissionGroup.getId(), developerPermissionGroup.getId(), viewerPermissionGroup.getId())) .build(); - DBAuth auth = (DBAuth) createdDatasource.getDatasourceConfiguration().getAuthentication(); + DatasourceStorageDTO createdDatasourceStorageDTO = createdDatasource.getDatasourceStorages().get(defaultEnvironmentId); + DBAuth auth = (DBAuth) createdDatasourceStorageDTO.getDatasourceConfiguration().getAuthentication(); assertThat(createdDatasource.getPolicies()).isNotEmpty(); assertThat(createdDatasource.getPolicies()).containsAll(Set.of(manageDatasourcePolicy, readDatasourcePolicy, executeDatasourcePolicy)); assertThat(auth.getDatabaseName()).isEqualTo("users"); @@ -277,7 +282,7 @@ public class MockDataServiceTest { Workspace workspace = workspaceService.create(toCreate, apiUser, Boolean.FALSE).block(); String workspaceId = workspace.getId(); - String environmentId = workspaceService.getDefaultEnvironmentId(workspaceId).block(); + String defaultEnvironmentId = workspaceService.getDefaultEnvironmentId(workspaceId).block(); Plugin pluginMono = pluginService.findByName("Installed Plugin Name").block(); @@ -287,8 +292,8 @@ public class MockDataServiceTest { mockDataSource.setPackageName("mongo-plugin"); mockDataSource.setPluginId(pluginMono.getId()); - Mono datasourceMono = mockDataService.createMockDataSet(mockDataSource, environmentId ) - .flatMap(datasource -> mockDataService.createMockDataSet(mockDataSource, environmentId)); + Mono datasourceMono = mockDataService.createMockDataSet(mockDataSource, defaultEnvironmentId ) + .flatMap(datasource -> mockDataService.createMockDataSet(mockDataSource, defaultEnvironmentId)); List permissionGroups = Mono.just(workspace) .flatMapMany(savedWorkspace -> { @@ -326,11 +331,14 @@ public class MockDataServiceTest { .permissionGroups(Set.of(adminPermissionGroup.getId(), developerPermissionGroup.getId(), viewerPermissionGroup.getId())) .build(); - DBAuth auth = (DBAuth) createdDatasource.getDatasourceConfiguration().getAuthentication(); + DatasourceStorageDTO createdDatasourceStorageDTO = createdDatasource.getDatasourceStorages().get(defaultEnvironmentId); + DatasourceConfiguration datasourceConfiguration = createdDatasourceStorageDTO.getDatasourceConfiguration(); + DBAuth auth = (DBAuth) datasourceConfiguration.getAuthentication(); + assertThat(createdDatasource.getPolicies()).isNotEmpty(); assertThat(createdDatasource.getPolicies()).containsAll(Set.of(manageDatasourcePolicy, readDatasourcePolicy, executeDatasourcePolicy)); - assertThat(createdDatasource.getDatasourceConfiguration().getProperties().get(0).getValue()).isEqualTo("Yes"); - assertThat(createdDatasource.getDatasourceConfiguration().getProperties().get(0).getKey()).isEqualTo("Use mongo connection string URI"); + assertThat(datasourceConfiguration.getProperties().get(0).getValue()).isEqualTo("Yes"); + assertThat(datasourceConfiguration.getProperties().get(0).getKey()).isEqualTo("Use mongo connection string URI"); assertThat(auth.getDatabaseName()).isEqualTo("movies"); assertThat(auth.getUsername()).isEqualTo("mockdb-admin"); }) diff --git a/app/server/appsmith-server/src/test/java/com/appsmith/server/solutions/ce/ActionExecutionSolutionCETest.java b/app/server/appsmith-server/src/test/java/com/appsmith/server/solutions/ce/ActionExecutionSolutionCETest.java index 6077d4d622..2b51ed2fb3 100644 --- a/app/server/appsmith-server/src/test/java/com/appsmith/server/solutions/ce/ActionExecutionSolutionCETest.java +++ b/app/server/appsmith-server/src/test/java/com/appsmith/server/solutions/ce/ActionExecutionSolutionCETest.java @@ -10,7 +10,6 @@ import com.appsmith.external.models.ActionDTO; import com.appsmith.external.models.ActionExecutionResult; import com.appsmith.external.models.Datasource; import com.appsmith.external.models.DatasourceConfiguration; -import com.appsmith.external.models.DatasourceDTO; import com.appsmith.external.models.DatasourceStorage; import com.appsmith.external.models.DatasourceStorageDTO; import com.appsmith.external.models.PaginationField; @@ -1818,7 +1817,7 @@ public class ActionExecutionSolutionCETest { mockDataSource.setWorkspaceId(workspaceId); mockDataSource.setPackageName("postgres-plugin"); mockDataSource.setPluginId(installed_plugin.getId()); - DatasourceDTO mockDatasource = mockDataService.createMockDataSet(mockDataSource, defaultEnvironmentId).block(); + Datasource mockDatasource = mockDataService.createMockDataSet(mockDataSource, defaultEnvironmentId).block(); List widgetTypeList = new ArrayList<>(); widgetTypeList.add(WidgetSuggestionHelper.getWidget(WidgetType.TEXT_WIDGET)); @@ -1830,7 +1829,7 @@ public class ActionExecutionSolutionCETest { action.setActionConfiguration(actionConfiguration); action.setPageId(testPage.getId()); action.setName("testActionExecuteDbQuery"); - Datasource datasource1 = datasourceService.convertToDatasource(mockDatasource, defaultEnvironmentId).block(); + Datasource datasource1 = mockDatasource; action.setDatasource(datasource1); ActionDTO createdAction = layoutActionService.createSingleAction(action, Boolean.FALSE).block();