diff --git a/.github/workflows/github-release.yml b/.github/workflows/github-release.yml index c4dd4001be..cedac1f848 100644 --- a/.github/workflows/github-release.yml +++ b/.github/workflows/github-release.yml @@ -247,10 +247,14 @@ jobs: run: | scripts/generate_info_json.sh + # As pg docker image is continuously updated for each scheduled cron on release, we are using the nightly tag while building the latest tag - name: Place server artifacts-es run: | if [[ -f scripts/prepare_server_artifacts.sh ]]; then - scripts/prepare_server_artifacts.sh + PG_TAG=nightly scripts/prepare_server_artifacts.sh + else + echo "No script found to prepare server artifacts" + exit 1 fi - name: Login to DockerHub diff --git a/.github/workflows/test-build-docker-image.yml b/.github/workflows/test-build-docker-image.yml index f14c565e21..e3c21798e6 100644 --- a/.github/workflows/test-build-docker-image.yml +++ b/.github/workflows/test-build-docker-image.yml @@ -356,8 +356,12 @@ jobs: - name: Place server artifacts-es run: | + run: | if [[ -f scripts/prepare_server_artifacts.sh ]]; then scripts/prepare_server_artifacts.sh + else + echo "No script found to prepare server artifacts" + exit 1 fi - name: Set up Depot CLI @@ -439,6 +443,9 @@ jobs: run: | if [[ -f scripts/prepare_server_artifacts.sh ]]; then scripts/prepare_server_artifacts.sh + else + echo "No script found to prepare server artifacts" + exit 1 fi - name: Set up Depot CLI diff --git a/app/client/cypress/e2e/Regression/ClientSide/Git/GitAutocommit_spec.ts b/app/client/cypress/e2e/Regression/ClientSide/Git/GitAutocommit_spec.ts index 9d38aa357d..c55b51711c 100644 --- a/app/client/cypress/e2e/Regression/ClientSide/Git/GitAutocommit_spec.ts +++ b/app/client/cypress/e2e/Regression/ClientSide/Git/GitAutocommit_spec.ts @@ -1,5 +1,4 @@ import ReconnectLocators from "../../../../locators/ReconnectLocators"; -import { featureFlagIntercept } from "../../../../support/Objects/FeatureFlags"; import { agHelper, gitSync, @@ -7,7 +6,7 @@ import { } from "../../../../support/Objects/ObjectsCore"; let wsName: string; -let repoName: string = "TED-testrepo1"; +let repoName: string = "TED-autocommit-test-1"; describe( "Git Autocommit", @@ -15,8 +14,8 @@ describe( tags: [ "@tag.Git", "@tag.GitAutocommit", - "@tag.excludeForAirgap", "@tag.Sanity", + "@tag.TedMigration", ], }, function () { diff --git a/app/client/src/PluginActionEditor/components/PluginActionNameEditor.tsx b/app/client/src/PluginActionEditor/components/PluginActionNameEditor.tsx new file mode 100644 index 0000000000..a51d6006de --- /dev/null +++ b/app/client/src/PluginActionEditor/components/PluginActionNameEditor.tsx @@ -0,0 +1,81 @@ +import React from "react"; +import { useSelector } from "react-redux"; +import ActionNameEditor from "components/editorComponents/ActionNameEditor"; +import { usePluginActionContext } from "PluginActionEditor/PluginActionContext"; +import { useFeatureFlag } from "utils/hooks/useFeatureFlag"; +import { getHasManageActionPermission } from "ee/utils/BusinessFeatures/permissionPageHelpers"; +import { FEATURE_FLAG } from "ee/entities/FeatureFlag"; +import { PluginType } from "entities/Action"; +import type { ReduxAction } from "ee/constants/ReduxActionConstants"; +import styled from "styled-components"; +import { getSavingStatusForActionName } from "selectors/actionSelectors"; +import { getAssetUrl } from "ee/utils/airgapHelpers"; +import { ActionUrlIcon } from "pages/Editor/Explorer/ExplorerIcons"; + +export interface SaveActionNameParams { + id: string; + name: string; +} + +export interface PluginActionNameEditorProps { + saveActionName: ( + params: SaveActionNameParams, + ) => ReduxAction; +} + +const ActionNameEditorWrapper = styled.div` + & .ads-v2-box { + gap: var(--ads-v2-spaces-2); + } + + && .t--action-name-edit-field { + font-size: 12px; + + .bp3-editable-text-content { + height: unset !important; + line-height: unset !important; + } + } + + & .t--plugin-icon-box { + height: 12px; + width: 12px; + + img { + width: 12px; + height: auto; + } + } +`; + +const PluginActionNameEditor = (props: PluginActionNameEditorProps) => { + const { action, plugin } = usePluginActionContext(); + + const isFeatureEnabled = useFeatureFlag(FEATURE_FLAG.license_gac_enabled); + const isChangePermitted = getHasManageActionPermission( + isFeatureEnabled, + action?.userPermissions, + ); + + const saveStatus = useSelector((state) => + getSavingStatusForActionName(state, action?.id || ""), + ); + + const iconUrl = getAssetUrl(plugin?.iconLocation) || ""; + const icon = ActionUrlIcon(iconUrl); + + return ( + + + + ); +}; + +export default PluginActionNameEditor; diff --git a/app/client/src/PluginActionEditor/index.ts b/app/client/src/PluginActionEditor/index.ts index 20265c8bc5..e8083e1b0f 100644 --- a/app/client/src/PluginActionEditor/index.ts +++ b/app/client/src/PluginActionEditor/index.ts @@ -6,3 +6,8 @@ export { export { default as PluginActionToolbar } from "./components/PluginActionToolbar"; export { default as PluginActionForm } from "./components/PluginActionForm"; export { default as PluginActionResponse } from "./components/PluginActionResponse"; +export type { + SaveActionNameParams, + PluginActionNameEditorProps, +} from "./components/PluginActionNameEditor"; +export { default as PluginActionNameEditor } from "./components/PluginActionNameEditor"; diff --git a/app/client/src/components/editorComponents/ActionNameEditor.tsx b/app/client/src/components/editorComponents/ActionNameEditor.tsx index 8467855f47..1d9bf01aa7 100644 --- a/app/client/src/components/editorComponents/ActionNameEditor.tsx +++ b/app/client/src/components/editorComponents/ActionNameEditor.tsx @@ -1,19 +1,13 @@ import React, { memo } from "react"; -import { useSelector } from "react-redux"; -import { useParams } from "react-router-dom"; import EditableText, { EditInteractionKind, } from "components/editorComponents/EditableText"; import { removeSpecialChars } from "utils/helpers"; -import type { AppState } from "ee/reducers"; -import { saveActionName } from "actions/pluginActionActions"; import { Flex } from "@appsmith/ads"; -import { getActionByBaseId, getPlugin } from "ee/selectors/entitiesSelector"; import NameEditorComponent, { IconBox, - IconWrapper, NameWrapper, } from "components/utils/NameEditorComponent"; import { @@ -21,14 +15,13 @@ import { ACTION_NAME_PLACEHOLDER, createMessage, } from "ee/constants/messages"; -import { getAssetUrl } from "ee/utils/airgapHelpers"; -import { getSavingStatusForActionName } from "selectors/actionSelectors"; import type { ReduxAction } from "ee/constants/ReduxActionConstants"; +import type { SaveActionNameParams } from "PluginActionEditor"; +import { useFeatureFlag } from "utils/hooks/useFeatureFlag"; +import { FEATURE_FLAG } from "ee/entities/FeatureFlag"; +import type { Action } from "entities/Action"; +import type { ModuleInstance } from "ee/constants/ModuleInstanceConstants"; -interface SaveActionNameParams { - id: string; - name: string; -} interface ActionNameEditorProps { /* This prop checks if page is API Pane or Query Pane or Curl Pane @@ -38,38 +31,34 @@ interface ActionNameEditorProps { */ enableFontStyling?: boolean; disabled?: boolean; - saveActionName?: ( + saveActionName: ( params: SaveActionNameParams, ) => ReduxAction; + actionConfig?: Action | ModuleInstance; + icon?: JSX.Element; + saveStatus: { isSaving: boolean; error: boolean }; } function ActionNameEditor(props: ActionNameEditorProps) { - const params = useParams<{ baseApiId?: string; baseQueryId?: string }>(); + const { + actionConfig, + disabled = false, + enableFontStyling = false, + icon = "", + saveActionName, + saveStatus, + } = props; - const currentActionConfig = useSelector((state: AppState) => - getActionByBaseId(state, params.baseApiId || params.baseQueryId || ""), - ); - - const currentPlugin = useSelector((state: AppState) => - getPlugin(state, currentActionConfig?.pluginId || ""), - ); - - const saveStatus = useSelector((state) => - getSavingStatusForActionName(state, currentActionConfig?.id || ""), + const isActionRedesignEnabled = useFeatureFlag( + FEATURE_FLAG.release_actions_redesign_enabled, ); return ( {({ @@ -85,28 +74,22 @@ function ActionNameEditor(props: ActionNameEditorProps) { isNew: boolean; saveStatus: { isSaving: boolean; error: boolean }; }) => ( - + - {currentPlugin && ( - - - - )} + {icon && {icon}} ))} diff --git a/app/client/src/components/utils/NameEditorComponent.tsx b/app/client/src/components/utils/NameEditorComponent.tsx index b7aa2e03ce..876a9c9d1f 100644 --- a/app/client/src/components/utils/NameEditorComponent.tsx +++ b/app/client/src/components/utils/NameEditorComponent.tsx @@ -11,6 +11,8 @@ import { } from "ee/constants/messages"; import styled from "styled-components"; import { Classes } from "@blueprintjs/core"; +import type { SaveActionNameParams } from "PluginActionEditor"; +import type { ReduxAction } from "ee/constants/ReduxActionConstants"; export const NameWrapper = styled.div<{ enableFontStyling?: boolean }>` min-width: 50%; @@ -71,9 +73,9 @@ interface NameEditorProps { children: (params: any) => JSX.Element; id?: string; name?: string; - // TODO: Fix this the next time the file is edited - // eslint-disable-next-line @typescript-eslint/no-explicit-any - dispatchAction: (a: any) => any; + onSaveName: ( + params: SaveActionNameParams, + ) => ReduxAction; // TODO: Fix this the next time the file is edited // eslint-disable-next-line @typescript-eslint/no-explicit-any suffixErrorMessage?: (params?: any) => string; @@ -90,10 +92,10 @@ interface NameEditorProps { function NameEditor(props: NameEditorProps) { const { - dispatchAction, id: entityId, idUndefinedErrorMessage, name: entityName, + onSaveName, saveStatus, suffixErrorMessage = ACTION_NAME_CONFLICT_ERROR, } = props; @@ -131,8 +133,8 @@ function NameEditor(props: NameEditorProps) { const handleNameChange = useCallback( (name: string) => { - if (name !== entityName && !isInvalidNameForEntity(name)) { - dispatch(dispatchAction({ id: entityId, name })); + if (name !== entityName && !isInvalidNameForEntity(name) && entityId) { + dispatch(onSaveName({ id: entityId, name })); } }, [dispatch, isInvalidNameForEntity, entityId, entityName], diff --git a/app/client/src/pages/Editor/APIEditor/ApiEditorContext.tsx b/app/client/src/pages/Editor/APIEditor/ApiEditorContext.tsx index a5d3aa5e23..1a2d9d5178 100644 --- a/app/client/src/pages/Editor/APIEditor/ApiEditorContext.tsx +++ b/app/client/src/pages/Editor/APIEditor/ApiEditorContext.tsx @@ -1,11 +1,7 @@ import type { ReduxAction } from "ee/constants/ReduxActionConstants"; import type { PaginationField } from "api/ActionAPI"; import React, { createContext, useMemo } from "react"; - -interface SaveActionNameParams { - id: string; - name: string; -} +import type { SaveActionNameParams } from "PluginActionEditor"; interface ApiEditorContextContextProps { moreActionsMenu?: React.ReactNode; @@ -15,7 +11,7 @@ interface ApiEditorContextContextProps { // TODO: Fix this the next time the file is edited // eslint-disable-next-line @typescript-eslint/no-explicit-any settingsConfig: any; - saveActionName?: ( + saveActionName: ( params: SaveActionNameParams, ) => ReduxAction; closeEditorLink?: React.ReactNode; diff --git a/app/client/src/pages/Editor/APIEditor/CommonEditorForm.tsx b/app/client/src/pages/Editor/APIEditor/CommonEditorForm.tsx index b542cdfbd0..979c4edf1c 100644 --- a/app/client/src/pages/Editor/APIEditor/CommonEditorForm.tsx +++ b/app/client/src/pages/Editor/APIEditor/CommonEditorForm.tsx @@ -35,6 +35,9 @@ import { InfoFields, RequestTabs, } from "PluginActionEditor/components/PluginActionForm/components/CommonEditorForm"; +import { getSavingStatusForActionName } from "selectors/actionSelectors"; +import { getAssetUrl } from "ee/utils/airgapHelpers"; +import { ActionUrlIcon } from "../Explorer/ExplorerIcons"; const Form = styled.form` position: relative; @@ -245,6 +248,18 @@ function CommonEditorForm(props: CommonFormPropsWithExtraParams) { currentActionConfig?.userPermissions, ); + const currentPlugin = useSelector((state: AppState) => + getPlugin(state, currentActionConfig?.pluginId || ""), + ); + + const saveStatus = useSelector((state) => + getSavingStatusForActionName(state, currentActionConfig?.id || ""), + ); + + const iconUrl = getAssetUrl(currentPlugin?.iconLocation) || ""; + + const icon = ActionUrlIcon(iconUrl); + const plugin = useSelector((state: AppState) => getPlugin(state, pluginId ?? ""), ); @@ -281,9 +296,12 @@ function CommonEditorForm(props: CommonFormPropsWithExtraParams) { diff --git a/app/client/src/pages/Editor/APIEditor/index.tsx b/app/client/src/pages/Editor/APIEditor/index.tsx index 247c2cfd40..31328d2505 100644 --- a/app/client/src/pages/Editor/APIEditor/index.tsx +++ b/app/client/src/pages/Editor/APIEditor/index.tsx @@ -8,7 +8,11 @@ import { getPluginSettingConfigs, getPlugins, } from "ee/selectors/entitiesSelector"; -import { deleteAction, runAction } from "actions/pluginActionActions"; +import { + deleteAction, + runAction, + saveActionName, +} from "actions/pluginActionActions"; import AnalyticsUtil from "ee/utils/AnalyticsUtil"; import Editor from "./Editor"; import BackToCanvas from "components/common/BackToCanvas"; @@ -151,15 +155,7 @@ function ApiEditorWrapper(props: ApiEditorWrapperProps) { }); dispatch(runAction(action?.id ?? "", paginationField)); }, - [ - action?.id, - apiName, - pageName, - getPageName, - plugins, - pluginId, - datasourceId, - ], + [action?.id, apiName, pageName, plugins, pluginId, datasourceId, dispatch], ); const actionRightPaneBackLink = useMemo(() => { @@ -173,13 +169,13 @@ function ApiEditorWrapper(props: ApiEditorWrapperProps) { pageName, }); dispatch(deleteAction({ id: action?.id ?? "", name: apiName })); - }, [getPageName, pages, basePageId, apiName]); + }, [pages, basePageId, apiName, action?.id, dispatch, pageName]); const notification = useMemo(() => { if (!isConverting) return null; return ; - }, [action?.name, isConverting]); + }, [action?.name, isConverting, icon]); const isActionRedesignEnabled = useFeatureFlag( FEATURE_FLAG.release_actions_redesign_enabled, @@ -196,6 +192,7 @@ function ApiEditorWrapper(props: ApiEditorWrapperProps) { handleRunClick={handleRunClick} moreActionsMenu={moreActionsMenu} notification={notification} + saveActionName={saveActionName} settingsConfig={settingsConfig} > diff --git a/app/client/src/pages/Editor/Explorer/Entity/Name.tsx b/app/client/src/pages/Editor/Explorer/Entity/Name.tsx index de937dd86a..e0fb005bc0 100644 --- a/app/client/src/pages/Editor/Explorer/Entity/Name.tsx +++ b/app/client/src/pages/Editor/Explorer/Entity/Name.tsx @@ -16,6 +16,8 @@ import { import { Tooltip } from "@appsmith/ads"; import { useSelector } from "react-redux"; import { getSavingStatusForActionName } from "selectors/actionSelectors"; +import type { ReduxAction } from "ee/constants/ReduxActionConstants"; +import type { SaveActionNameParams } from "PluginActionEditor"; export const searchHighlightSpanClassName = "token"; export const searchTokenizationDelimiter = "!!"; @@ -84,7 +86,7 @@ export interface EntityNameProps { name: string; isEditing?: boolean; onChange?: (name: string) => void; - updateEntityName: (name: string) => void; + updateEntityName: (name: string) => ReduxAction; entityId: string; searchKeyword?: string; className?: string; @@ -164,10 +166,10 @@ export const EntityName = React.memo( return ( diff --git a/app/client/src/pages/Editor/JSEditor/JSObjectNameEditor.tsx b/app/client/src/pages/Editor/JSEditor/JSObjectNameEditor.tsx index f82860f134..ff1a598bc1 100644 --- a/app/client/src/pages/Editor/JSEditor/JSObjectNameEditor.tsx +++ b/app/client/src/pages/Editor/JSEditor/JSObjectNameEditor.tsx @@ -25,11 +25,8 @@ import NameEditorComponent, { } from "components/utils/NameEditorComponent"; import { getSavingStatusForJSObjectName } from "selectors/actionSelectors"; import type { ReduxAction } from "ee/constants/ReduxActionConstants"; +import type { SaveActionNameParams } from "PluginActionEditor"; -export interface SaveActionNameParams { - id: string; - name: string; -} export interface JSObjectNameEditorProps { /* This prop checks if page is API Pane or Query Pane or Curl Pane @@ -64,10 +61,10 @@ export function JSObjectNameEditor(props: JSObjectNameEditorProps) { return ( {({ diff --git a/app/client/src/pages/Editor/QueryEditor/QueryEditorContext.tsx b/app/client/src/pages/Editor/QueryEditor/QueryEditorContext.tsx index 03c46a99d0..c5089fc6bd 100644 --- a/app/client/src/pages/Editor/QueryEditor/QueryEditorContext.tsx +++ b/app/client/src/pages/Editor/QueryEditor/QueryEditorContext.tsx @@ -1,18 +1,14 @@ import type { ReduxAction } from "ee/constants/ReduxActionConstants"; +import type { SaveActionNameParams } from "PluginActionEditor"; import React, { createContext, useMemo } from "react"; -interface SaveActionNameParams { - id: string; - name: string; -} - interface QueryEditorContextContextProps { moreActionsMenu?: React.ReactNode; onCreateDatasourceClick?: () => void; onEntityNotFoundBackClick?: () => void; changeQueryPage?: (baseQueryId: string) => void; actionRightPaneBackLink?: React.ReactNode; - saveActionName?: ( + saveActionName: ( params: SaveActionNameParams, ) => ReduxAction; closeEditorLink?: React.ReactNode; diff --git a/app/client/src/pages/Editor/QueryEditor/QueryEditorHeader.tsx b/app/client/src/pages/Editor/QueryEditor/QueryEditorHeader.tsx index af1d28c83a..38f9a9b3d7 100644 --- a/app/client/src/pages/Editor/QueryEditor/QueryEditorHeader.tsx +++ b/app/client/src/pages/Editor/QueryEditor/QueryEditorHeader.tsx @@ -13,6 +13,7 @@ import { useActiveActionBaseId } from "ee/pages/Editor/Explorer/hooks"; import { useSelector } from "react-redux"; import { getActionByBaseId, + getPlugin, getPluginNameFromId, } from "ee/selectors/entitiesSelector"; import { QueryEditorContext } from "./QueryEditorContext"; @@ -21,6 +22,9 @@ import type { Datasource } from "entities/Datasource"; import type { AppState } from "ee/reducers"; import { SQL_DATASOURCES } from "constants/QueryEditorConstants"; import DatasourceSelector from "./DatasourceSelector"; +import { getSavingStatusForActionName } from "selectors/actionSelectors"; +import { getAssetUrl } from "ee/utils/airgapHelpers"; +import { ActionUrlIcon } from "../Explorer/ExplorerIcons"; const NameWrapper = styled.div` display: flex; @@ -79,6 +83,18 @@ const QueryEditorHeader = (props: Props) => { currentActionConfig?.userPermissions, ); + const currentPlugin = useSelector((state: AppState) => + getPlugin(state, currentActionConfig?.pluginId || ""), + ); + + const saveStatus = useSelector((state) => + getSavingStatusForActionName(state, currentActionConfig?.id || ""), + ); + + const iconUrl = getAssetUrl(currentPlugin?.iconLocation) || ""; + + const icon = ActionUrlIcon(iconUrl); + // get the current action's plugin name const currentActionPluginName = useSelector((state: AppState) => getPluginNameFromId(state, currentActionConfig?.pluginId || ""), @@ -106,8 +122,11 @@ const QueryEditorHeader = (props: Props) => { diff --git a/app/client/src/pages/Editor/QueryEditor/index.tsx b/app/client/src/pages/Editor/QueryEditor/index.tsx index f76cbffe1b..03457a6228 100644 --- a/app/client/src/pages/Editor/QueryEditor/index.tsx +++ b/app/client/src/pages/Editor/QueryEditor/index.tsx @@ -42,6 +42,7 @@ import { ENTITY_ICON_SIZE, EntityIcon } from "../Explorer/ExplorerIcons"; import { getIDEViewMode } from "selectors/ideSelectors"; import { EditorViewMode } from "ee/entities/IDE/constants"; import { AppPluginActionEditor } from "../AppPluginActionEditor"; +import { saveActionName } from "actions/pluginActionActions"; type QueryEditorProps = RouteComponentProps; @@ -126,6 +127,7 @@ function QueryEditor(props: QueryEditorProps) { }, [ action?.id, action?.name, + action?.pluginType, isChangePermitted, isDeletePermitted, basePageId, @@ -143,7 +145,7 @@ function QueryEditor(props: QueryEditorProps) { changeQuery({ baseQueryId: baseQueryId, basePageId, applicationId }), ); }, - [basePageId, applicationId], + [basePageId, applicationId, dispatch], ); const onCreateDatasourceClick = useCallback(() => { @@ -159,13 +161,7 @@ function QueryEditor(props: QueryEditorProps) { AnalyticsUtil.logEvent("NAVIGATE_TO_CREATE_NEW_DATASOURCE_PAGE", { entryPoint, }); - }, [ - basePageId, - history, - integrationEditorURL, - DatasourceCreateEntryPoints, - AnalyticsUtil, - ]); + }, [basePageId]); // custom function to return user to integrations page if action is not found const onEntityNotFoundBackClick = useCallback( @@ -176,7 +172,7 @@ function QueryEditor(props: QueryEditorProps) { selectedTab: INTEGRATION_TABS.ACTIVE, }), ), - [basePageId, history, integrationEditorURL], + [basePageId], ); const notification = useMemo(() => { @@ -189,7 +185,7 @@ function QueryEditor(props: QueryEditorProps) { withPadding /> ); - }, [action?.name, isConverting]); + }, [action?.name, isConverting, icon]); const isActionRedesignEnabled = useFeatureFlag( FEATURE_FLAG.release_actions_redesign_enabled, @@ -207,6 +203,7 @@ function QueryEditor(props: QueryEditorProps) { notification={notification} onCreateDatasourceClick={onCreateDatasourceClick} onEntityNotFoundBackClick={onEntityNotFoundBackClick} + saveActionName={saveActionName} > { + private T connection; + private String pluginId; + private Instant creationTime; + + public DatasourcePluginContext() { + creationTime = Instant.now(); + } +} diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/DatasourceContextServiceCEImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/DatasourceContextServiceCEImpl.java index 3693d1348c..adc1d2ea82 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/DatasourceContextServiceCEImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/DatasourceContextServiceCEImpl.java @@ -13,6 +13,7 @@ import com.appsmith.server.datasources.base.DatasourceService; import com.appsmith.server.datasourcestorages.base.DatasourceStorageService; import com.appsmith.server.domains.DatasourceContext; import com.appsmith.server.domains.DatasourceContextIdentifier; +import com.appsmith.server.domains.DatasourcePluginContext; import com.appsmith.server.domains.Plugin; import com.appsmith.server.exceptions.AppsmithError; import com.appsmith.server.exceptions.AppsmithException; @@ -20,6 +21,10 @@ import com.appsmith.server.helpers.PluginExecutorHelper; import com.appsmith.server.plugins.base.PluginService; import com.appsmith.server.services.ConfigService; import com.appsmith.server.solutions.DatasourcePermission; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.RemovalListener; +import com.google.common.cache.RemovalNotification; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; @@ -29,8 +34,12 @@ import reactor.core.scheduler.Schedulers; import java.time.Instant; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; import java.util.function.Function; +import static java.lang.Boolean.FALSE; +import static java.lang.Boolean.TRUE; + @Slf4j public class DatasourceContextServiceCEImpl implements DatasourceContextServiceCE { @@ -38,6 +47,21 @@ public class DatasourceContextServiceCEImpl implements DatasourceContextServiceC protected final Map>> datasourceContextMonoMap; protected final Map datasourceContextSynchronizationMonitorMap; protected final Map> datasourceContextMap; + + /** + * This cache is used to store the datasource context for a limited time and a limited max number of connections and + * then destroy the least recently used connection. The cleanup process is triggered when the cache is accessed and + * either the time limit or the max connections are reached. + * The purpose of this is to prevent the large number of open dangling connections to the movies mockDB. + * The removalListener method is called when the connection is removed from the cache. + */ + protected final Cache datasourcePluginContextMapLRUCache = + CacheBuilder.newBuilder() + .removalListener(createRemovalListener()) + .expireAfterAccess(2, TimeUnit.HOURS) + .maximumSize(300) // caches most recently used 300 mock connections per pod + .build(); + private final DatasourceService datasourceService; private final DatasourceStorageService datasourceStorageService; private final PluginService pluginService; @@ -67,6 +91,50 @@ public class DatasourceContextServiceCEImpl implements DatasourceContextServiceC this.datasourcePermission = datasourcePermission; } + private RemovalListener createRemovalListener() { + return (RemovalNotification removalNotification) -> { + handleRemoval(removalNotification); + }; + } + + private Object getConnectionFromDatasourceContextMap(DatasourceContextIdentifier datasourceContextIdentifier) { + return this.datasourceContextMap.containsKey(datasourceContextIdentifier) + && this.datasourceContextMap.get(datasourceContextIdentifier) != null + ? this.datasourceContextMap.get(datasourceContextIdentifier).getConnection() + : null; + } + + private void handleRemoval( + RemovalNotification removalNotification) { + final DatasourceContextIdentifier datasourceContextIdentifier = removalNotification.getKey(); + final DatasourcePluginContext datasourcePluginContext = removalNotification.getValue(); + + log.debug( + "Removing Datasource Context from cache and closing the open connection for DatasourceId: {} and environmentId: {}", + datasourceContextIdentifier.getDatasourceId(), + datasourceContextIdentifier.getEnvironmentId()); + log.info("LRU Cache Size after eviction: {}", datasourcePluginContextMapLRUCache.size()); + + // Close connection and remove entry from both cache maps + final Object connection = getConnectionFromDatasourceContextMap(datasourceContextIdentifier); + + Mono pluginMono = + pluginService.findById(datasourcePluginContext.getPluginId()).cache(); + if (connection != null) { + pluginExecutorHelper + .getPluginExecutor(pluginMono) + .flatMap(pluginExecutor -> Mono.fromRunnable(() -> pluginExecutor.datasourceDestroy(connection))) + .onErrorResume(e -> { + log.error("Error destroying stale datasource connection", e); + return Mono.empty(); + }) + .subscribe(); // Trigger the execution + } + // Remove the entries from both maps + datasourceContextMonoMap.remove(datasourceContextIdentifier); + datasourceContextMap.remove(datasourceContextIdentifier); + } + /** * This method defines a critical section that can be executed only by one thread at a time per datasource id - i * .e. if two threads want to create datasource for different datasource ids then they would not be synchronized. @@ -115,6 +183,11 @@ public class DatasourceContextServiceCEImpl implements DatasourceContextServiceC } datasourceContextMonoMap.remove(datasourceContextIdentifier); datasourceContextMap.remove(datasourceContextIdentifier); + log.info( + "Invalidating the LRU cache entry for datasource id {}, environment id {} as the connection is stale or in error state", + datasourceContextIdentifier.getDatasourceId(), + datasourceContextIdentifier.getEnvironmentId()); + datasourcePluginContextMapLRUCache.invalidate(datasourceContextIdentifier); } /* @@ -129,17 +202,13 @@ public class DatasourceContextServiceCEImpl implements DatasourceContextServiceC + ": Cached resource context mono exists for datasource id {}, environment id {}. Returning the same.", datasourceContextIdentifier.getDatasourceId(), datasourceContextIdentifier.getEnvironmentId()); + // Accessing the LRU cache to update the last accessed time + datasourcePluginContextMapLRUCache.getIfPresent(datasourceContextIdentifier); return datasourceContextMonoMap.get(datasourceContextIdentifier); } /* Create a fresh datasource context */ DatasourceContext datasourceContext = new DatasourceContext<>(); - if (datasourceContextIdentifier.isKeyValid() && shouldCacheContextForThisPlugin(plugin)) { - /* For this datasource, either the context doesn't exist, or the context is stale. Replace (or add) with - the new connection in the context map. */ - datasourceContextMap.put(datasourceContextIdentifier, datasourceContext); - } - Mono connectionMonoCache = pluginExecutor .datasourceCreate(datasourceStorage.getDatasourceConfiguration()) .cache(); @@ -159,15 +228,34 @@ public class DatasourceContextServiceCEImpl implements DatasourceContextServiceC datasourceContext) .cache(); /* Cache the value so that further evaluations don't result in new connections */ - if (datasourceContextIdentifier.isKeyValid() && shouldCacheContextForThisPlugin(plugin)) { - datasourceContextMonoMap.put(datasourceContextIdentifier, datasourceContextMonoCache); - } - log.debug( - Thread.currentThread().getName() - + ": Cached new datasource context for datasource id {}, environment id {}", - datasourceContextIdentifier.getDatasourceId(), - datasourceContextIdentifier.getEnvironmentId()); - return datasourceContextMonoCache; + return connectionMonoCache + .flatMap(connection -> { + datasourceContext.setConnection(connection); + if (datasourceContextIdentifier.isKeyValid() + && shouldCacheContextForThisPlugin(plugin)) { + datasourceContextMap.put(datasourceContextIdentifier, datasourceContext); + datasourceContextMonoMap.put( + datasourceContextIdentifier, datasourceContextMonoCache); + + if (TRUE.equals(datasourceStorage.getIsMock()) + && PluginConstants.PackageName.MONGO_PLUGIN.equals( + plugin.getPackageName())) { + log.info( + "Datasource is a mock mongo DB. Adding the connection to LRU cache!"); + DatasourcePluginContext datasourcePluginContext = + new DatasourcePluginContext<>(); + datasourcePluginContext.setConnection(datasourceContext.getConnection()); + datasourcePluginContext.setPluginId(plugin.getId()); + datasourcePluginContextMapLRUCache.put( + datasourceContextIdentifier, datasourcePluginContext); + log.info( + "LRU Cache Size after adding: {}", + datasourcePluginContextMapLRUCache.size()); + } + } + return datasourceContextMonoCache; + }) + .switchIfEmpty(datasourceContextMonoCache); } }) .flatMap(obj -> obj) @@ -195,7 +283,7 @@ public class DatasourceContextServiceCEImpl implements DatasourceContextServiceC .setAuthentication(updatableConnection.getAuthenticationDTO( datasourceStorage.getDatasourceConfiguration().getAuthentication())); datasourceStorageMono = datasourceStorageService.updateDatasourceStorage( - datasourceStorage, datasourceStorage.getEnvironmentId(), Boolean.FALSE, false); + datasourceStorage, datasourceStorage.getEnvironmentId(), FALSE, false); } return datasourceStorageMono.thenReturn(connection); } @@ -308,6 +396,8 @@ public class DatasourceContextServiceCEImpl implements DatasourceContextServiceC } else { if (isValidDatasourceContextAvailable(datasourceStorage, datasourceContextIdentifier)) { log.debug("Resource context exists. Returning the same."); + // Accessing the LRU cache to update the last accessed time + datasourcePluginContextMapLRUCache.getIfPresent(datasourceContextIdentifier); return Mono.just(datasourceContextMap.get(datasourceContextIdentifier)); } } @@ -399,7 +489,11 @@ public class DatasourceContextServiceCEImpl implements DatasourceContextServiceC log.info("Clearing datasource context for datasource storage ID {}.", datasourceStorage.getId()); pluginExecutor.datasourceDestroy(datasourceContext.getConnection()); datasourceContextMonoMap.remove(datasourceContextIdentifier); - + log.info( + "Invalidating the LRU cache entry for datasource id {}, environment id {} as delete datasource context is invoked", + datasourceContextIdentifier.getDatasourceId(), + datasourceContextIdentifier.getEnvironmentId()); + datasourcePluginContextMapLRUCache.invalidate(datasourceContextIdentifier); if (!datasourceContextMap.containsKey(datasourceContextIdentifier)) { log.info( "datasourceContextMap does not contain any entry for datasource storage with id: {} ", diff --git a/deploy/docker/fs/opt/appsmith/caddy-reconfigure.mjs b/deploy/docker/fs/opt/appsmith/caddy-reconfigure.mjs index f265b2da71..75d5e9296f 100644 --- a/deploy/docker/fs/opt/appsmith/caddy-reconfigure.mjs +++ b/deploy/docker/fs/opt/appsmith/caddy-reconfigure.mjs @@ -146,7 +146,7 @@ parts.push(` ${isRateLimitingEnabled ? `rate_limit { zone dynamic_zone { - key {http.request.remote_ip} + key {http.request.client_ip} events ${RATE_LIMIT} window 1s } diff --git a/deploy/docker/fs/opt/appsmith/pg-utils.sh b/deploy/docker/fs/opt/appsmith/pg-utils.sh new file mode 100755 index 0000000000..315446f552 --- /dev/null +++ b/deploy/docker/fs/opt/appsmith/pg-utils.sh @@ -0,0 +1,92 @@ +#!/bin/bash + +waitForPostgresAvailability() { + if [ -z "$PG_DB_HOST" ]; then + tlog "PostgreSQL host name is empty. Check env variables. Error. Exiting java setup" + exit 2 + else + + MAX_RETRIES=50 + RETRYSECONDS=10 + retry_count=0 + while true; do + su postgres -c "pg_isready -h '${PG_DB_HOST}' -p '${PG_DB_PORT}'" + status=$? + + case $status in + 0) + tlog "PostgreSQL host '$PG_DB_HOST' is ready." + break + ;; + 1) + tlog "PostgreSQL host '$PG_DB_HOST' is rejecting connections e.g. due to being in recovery mode or not accepting connections eg. connections maxed out." + ;; + 2) + tlog "PostgreSQL host '$PG_DB_HOST' is not responding or running." + ;; + 3) + tlog "The connection check failed e.g. due to network issues or incorrect parameters." + ;; + *) + tlog "pg_isready exited with unexpected status code: $status" + break + ;; + esac + + retry_count=$((retry_count + 1)) + if [ $retry_count -le $MAX_RETRIES ]; then + tlog "PostgreSQL connection failed. Retrying attempt $retry_count/$MAX_RETRIES in $RETRYSECONDS seconds..." + sleep $RETRYSECONDS + else + tlog "Exceeded maximum retry attempts ($MAX_RETRIES). Exiting." + # use exit code 2 to indicate that the script failed to connect to postgres and supervisor conf is set not to restart the program for 2. + exit 2 + fi + + done + fi +} + +# for PostgreSQL, we use APPSMITH_DB_URL=postgresql://username:password@postgresserver:5432/dbname +# Args: +# conn_string (string): PostgreSQL connection string +# Returns: +# None +# Example: +# postgres syntax +# "postgresql://user:password@localhost:5432/appsmith" +# "postgresql://user:password@localhost/appsmith" +# "postgresql://user@localhost:5432/appsmith" +# "postgresql://user@localhost/appsmith" +extract_postgres_db_params() { + local conn_string=$1 + + # Use node to parse the URI and extract components + IFS=' ' read -r USER PASSWORD HOST PORT DB <<<"$(node -e " + const connectionString = process.argv[1]; + const pgUri = connectionString.startsWith(\"postgresql://\") + ? connectionString + : 'http://' + connectionString; //Prepend a fake scheme for URL parsing + const url = require('url'); + const parsedUrl = new url.URL(pgUri); + + // Extract the pathname and remove the leading '/' + const db = parsedUrl.pathname.substring(1); + + // Default the port to 5432 if it's empty + const port = parsedUrl.port || '5432'; + + console.log(\`\${parsedUrl.username || '-'} \${parsedUrl.password || '-'} \${parsedUrl.hostname} \${port} \${db}\`); + " "$conn_string")" + + # Now, set the environment variables + export PG_DB_USER="$USER" + export PG_DB_PASSWORD="$PASSWORD" + export PG_DB_HOST="$HOST" + export PG_DB_PORT="$PORT" + export PG_DB_NAME="$DB" +} + +# Example usage of the functions +# waitForPostgresAvailability +# extract_postgres_db_params "postgresql://user:password@localhost:5432/dbname" \ No newline at end of file diff --git a/deploy/docker/fs/opt/appsmith/run-java.sh b/deploy/docker/fs/opt/appsmith/run-java.sh index 675c8e2651..ed88e26e11 100755 --- a/deploy/docker/fs/opt/appsmith/run-java.sh +++ b/deploy/docker/fs/opt/appsmith/run-java.sh @@ -1,5 +1,8 @@ #!/bin/bash +# Source the helper script +source pg-utils.sh + set -o errexit set -o pipefail set -o nounset @@ -29,6 +32,12 @@ match-proxy-url() { [[ -n $proxy_host ]] } +# Extract the database parameters from the APPSMITH_DB_URL and wait for the database to be available +if [[ "$mode" == "pg" ]]; then + extract_postgres_db_params "$APPSMITH_DB_URL" + waitForPostgresAvailability +fi + if match-proxy-url "${HTTP_PROXY-}"; then extra_args+=(-Dhttp.proxyHost="$proxy_host" -Dhttp.proxyPort="$proxy_port") if [[ -n $proxy_user ]]; then diff --git a/deploy/docker/tests/test-pg-utils.sh b/deploy/docker/tests/test-pg-utils.sh new file mode 100755 index 0000000000..56bec14456 --- /dev/null +++ b/deploy/docker/tests/test-pg-utils.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash + +set -e + +# Include the script to be tested +source /Users/appsmith/Work/appsmith-ce/deploy/docker/fs/opt/appsmith/pg-utils.sh + +assert_equals() { + if [ "$1" != "$2" ]; then + echo "Assertion failed: expected '$2', but got '$1'" + return 1 + fi +} + +# Test extract_postgres_db_params function +test_extract_postgres_db_params_valid_db_string() { + local conn_string="postgresql://user:password@localhost:5432/dbname" + extract_postgres_db_params "$conn_string" + + if [ "$PG_DB_USER" != "user" ] || [ "$PG_DB_PASSWORD" != "password" ] || [ "$PG_DB_HOST" != "localhost" ] || [ "$PG_DB_PORT" != "5432" ] || [ "$PG_DB_NAME" != "dbname" ]; then + echo "Test failed: test_extract_postgres_db_params_valid_db_string did not extract parameters correctly" + echo_params + exit 1 + fi + + echo "Test passed: test_extract_postgres_db_params_valid_db_string" +} + +test_extract_postgres_db_params_empty_dbname() { + local conn_string="postgresql://user:password@localhost:5432" + extract_postgres_db_params "$conn_string" + + if [ "$PG_DB_USER" != "user" ] || [ "$PG_DB_PASSWORD" != "password" ] || [ "$PG_DB_HOST" != "localhost" ] || [ "$PG_DB_PORT" != "5432" ] || [ "$PG_DB_NAME" != "" ]; then + echo "Test failed: test_extract_postgres_db_params_empty_dbname did not extract parameters correctly" + echo_params + exit 1 + fi + + echo "Test passed: test_extract_postgres_db_params_empty_dbname" +} + +test_extract_postgres_db_params_with_spaces() { + local conn_string="postgresql://user:p a s s w o r d@localhost:5432/db_name" + extract_postgres_db_params "$conn_string" + + if [ "$PG_DB_USER" != "user" ] || [ "$PG_DB_PASSWORD" != "p%20a%20s%20s%20w%20o%20r%20d" ] || [ "$PG_DB_HOST" != "localhost" ] || [ "$PG_DB_PORT" != "5432" ] || [ "$PG_DB_NAME" != "db_name" ]; then + echo "Test failed: test_extract_postgres_db_params_with_spaces did not extract parameters correctly" + echo_params + exit 1 + fi + + echo "Test passed: test_extract_postgres_db_params_with_spaces" +} + +echo_params() { + echo "PG_DB_USER: $PG_DB_USER" + echo "PG_DB_PASSWORD: $PG_DB_PASSWORD" + echo "PG_DB_HOST: $PG_DB_HOST" + echo "PG_DB_PORT: $PG_DB_PORT" + echo "PG_DB_NAME: $PG_DB_NAME" +} + +# Run tests +test_extract_postgres_db_params_valid_db_string +test_extract_postgres_db_params_empty_dbname +test_extract_postgres_db_params_with_spaces + +echo "All Tests Pass!" \ No newline at end of file