diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/ActionCollectionServiceCEImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/ActionCollectionServiceCEImpl.java index 5b6bb5bad4..1ec1089598 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/ActionCollectionServiceCEImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/ActionCollectionServiceCEImpl.java @@ -663,159 +663,177 @@ public class ActionCollectionServiceCEImpl extends BaseService { - ImportActionCollectionResultDTO resultDTO = new ImportActionCollectionResultDTO(); - final String workspaceId = importedApplication.getWorkspaceId(); + return Mono.just(application) + .flatMap(importedApplication -> { + ImportActionCollectionResultDTO resultDTO = new ImportActionCollectionResultDTO(); + final String workspaceId = importedApplication.getWorkspaceId(); - // Map of gitSyncId to actionCollection of the existing records in DB - Mono> actionCollectionsInCurrentAppMono = repository - .findByApplicationId(importedApplication.getId()) - .filter(collection -> collection.getGitSyncId() != null) - .collectMap(ActionCollection::getGitSyncId); + // Map of gitSyncId to actionCollection of the existing records in DB + Mono> actionCollectionsInCurrentAppMono = repository + .findByApplicationId(importedApplication.getId()) + .filter(collection -> collection.getGitSyncId() != null) + .collectMap(ActionCollection::getGitSyncId); - Mono> actionCollectionsInBranchesMono; - if (importedApplication.getGitApplicationMetadata() != null) { - final String defaultApplicationId = - importedApplication.getGitApplicationMetadata().getDefaultApplicationId(); - actionCollectionsInBranchesMono = repository - .findByDefaultApplicationId(defaultApplicationId, Optional.empty()) - .filter(actionCollection -> actionCollection.getGitSyncId() != null) - .collectMap(ActionCollection::getGitSyncId); - } else { - actionCollectionsInBranchesMono = Mono.just(Collections.emptyMap()); - } + Mono> actionCollectionsInBranchesMono; + if (importedApplication.getGitApplicationMetadata() != null) { + final String defaultApplicationId = + importedApplication.getGitApplicationMetadata().getDefaultApplicationId(); + actionCollectionsInBranchesMono = repository + .findByDefaultApplicationId(defaultApplicationId, Optional.empty()) + .filter(actionCollection -> actionCollection.getGitSyncId() != null) + .collectMap(ActionCollection::getGitSyncId); + } else { + actionCollectionsInBranchesMono = Mono.just(Collections.emptyMap()); + } - return Mono.zip(actionCollectionsInCurrentAppMono, actionCollectionsInBranchesMono) - .flatMap(objects -> { - Map actionsCollectionsInCurrentApp = objects.getT1(); - Map actionsCollectionsInBranches = objects.getT2(); + return Mono.zip(actionCollectionsInCurrentAppMono, actionCollectionsInBranchesMono) + .flatMap(objects -> { + Map actionsCollectionsInCurrentApp = objects.getT1(); + Map actionsCollectionsInBranches = objects.getT2(); - // set the existing action collections in the result DTO, this will be required in next phases - resultDTO.setExistingActionCollections(actionsCollectionsInCurrentApp.values()); + // set the existing action collections in the result DTO, this will be required in next + // phases + resultDTO.setExistingActionCollections(actionsCollectionsInCurrentApp.values()); - List newActionCollections = new ArrayList<>(); - List existingActionCollections = new ArrayList<>(); + List newActionCollections = new ArrayList<>(); + List existingActionCollections = new ArrayList<>(); - for (ActionCollection actionCollection : importedActionCollectionList) { - if (actionCollection.getUnpublishedCollection() == null - || StringUtils.isEmpty(actionCollection - .getUnpublishedCollection() - .getPageId())) { - continue; // invalid action collection, skip it - } - final String idFromJsonFile = actionCollection.getId(); - NewPage parentPage = new NewPage(); - final ActionCollectionDTO unpublishedCollection = - actionCollection.getUnpublishedCollection(); - final ActionCollectionDTO publishedCollection = actionCollection.getPublishedCollection(); + for (ActionCollection actionCollection : importedActionCollectionList) { + if (actionCollection.getUnpublishedCollection() == null + || StringUtils.isEmpty(actionCollection + .getUnpublishedCollection() + .getPageId())) { + continue; // invalid action collection, skip it + } + final String idFromJsonFile = actionCollection.getId(); + NewPage parentPage = new NewPage(); + final ActionCollectionDTO unpublishedCollection = + actionCollection.getUnpublishedCollection(); + final ActionCollectionDTO publishedCollection = + actionCollection.getPublishedCollection(); - // If pageId is missing in the actionCollectionDTO create a fallback pageId - final String fallbackParentPageId = unpublishedCollection.getPageId(); + // If pageId is missing in the actionCollectionDTO create a fallback pageId + final String fallbackParentPageId = unpublishedCollection.getPageId(); - if (unpublishedCollection.getName() != null) { - unpublishedCollection.setDefaultToBranchedActionIdsMap(importActionResultDTO - .getUnpublishedCollectionIdToActionIdsMap() - .get(idFromJsonFile)); - unpublishedCollection.setPluginId(pluginMap.get(unpublishedCollection.getPluginId())); - parentPage = updatePageInActionCollection(unpublishedCollection, pageNameMap); - } + if (unpublishedCollection.getName() != null) { + unpublishedCollection.setDefaultToBranchedActionIdsMap(importActionResultDTO + .getUnpublishedCollectionIdToActionIdsMap() + .get(idFromJsonFile)); + unpublishedCollection.setPluginId( + pluginMap.get(unpublishedCollection.getPluginId())); + parentPage = updatePageInActionCollection(unpublishedCollection, pageNameMap); + } - if (publishedCollection != null && publishedCollection.getName() != null) { - publishedCollection.setDefaultToBranchedActionIdsMap(importActionResultDTO - .getPublishedCollectionIdToActionIdsMap() - .get(idFromJsonFile)); - publishedCollection.setPluginId(pluginMap.get(publishedCollection.getPluginId())); - if (StringUtils.isEmpty(publishedCollection.getPageId())) { - publishedCollection.setPageId(fallbackParentPageId); - } - NewPage publishedCollectionPage = - updatePageInActionCollection(publishedCollection, pageNameMap); - parentPage = parentPage == null ? publishedCollectionPage : parentPage; - } + if (publishedCollection != null && publishedCollection.getName() != null) { + publishedCollection.setDefaultToBranchedActionIdsMap(importActionResultDTO + .getPublishedCollectionIdToActionIdsMap() + .get(idFromJsonFile)); + publishedCollection.setPluginId( + pluginMap.get(publishedCollection.getPluginId())); + if (StringUtils.isEmpty(publishedCollection.getPageId())) { + publishedCollection.setPageId(fallbackParentPageId); + } + NewPage publishedCollectionPage = + updatePageInActionCollection(publishedCollection, pageNameMap); + parentPage = parentPage == null ? publishedCollectionPage : parentPage; + } - actionCollection.makePristine(); - actionCollection.setWorkspaceId(workspaceId); - actionCollection.setApplicationId(importedApplication.getId()); + actionCollection.makePristine(); + actionCollection.setWorkspaceId(workspaceId); + actionCollection.setApplicationId(importedApplication.getId()); - // Check if the action has gitSyncId and if it's already in DB - if (actionCollection.getGitSyncId() != null - && actionsCollectionsInCurrentApp.containsKey(actionCollection.getGitSyncId())) { + // Check if the action has gitSyncId and if it's already in DB + if (actionCollection.getGitSyncId() != null + && actionsCollectionsInCurrentApp.containsKey( + actionCollection.getGitSyncId())) { - // Since the resource is already present in DB, just update resource - ActionCollection existingActionCollection = - actionsCollectionsInCurrentApp.get(actionCollection.getGitSyncId()); + // Since the resource is already present in DB, just update resource + ActionCollection existingActionCollection = + actionsCollectionsInCurrentApp.get(actionCollection.getGitSyncId()); - Set existingPolicy = existingActionCollection.getPolicies(); - copyNestedNonNullProperties(actionCollection, existingActionCollection); - // Update branchName - existingActionCollection.getDefaultResources().setBranchName(branchName); - // Recover the deleted state present in DB from imported actionCollection - existingActionCollection - .getUnpublishedCollection() - .setDeletedAt(actionCollection + Set existingPolicy = existingActionCollection.getPolicies(); + copyNestedNonNullProperties(actionCollection, existingActionCollection); + // Update branchName + existingActionCollection + .getDefaultResources() + .setBranchName(branchName); + // Recover the deleted state present in DB from imported actionCollection + existingActionCollection .getUnpublishedCollection() - .getDeletedAt()); - existingActionCollection.setDeletedAt(actionCollection.getDeletedAt()); - existingActionCollection.setDeleted(actionCollection.getDeleted()); - existingActionCollection.setPolicies(existingPolicy); + .setDeletedAt(actionCollection + .getUnpublishedCollection() + .getDeletedAt()); + existingActionCollection.setDeletedAt(actionCollection.getDeletedAt()); + existingActionCollection.setDeleted(actionCollection.getDeleted()); + existingActionCollection.setPolicies(existingPolicy); - existingActionCollection.updateForBulkWriteOperation(); - existingActionCollections.add(existingActionCollection); - resultDTO.getSavedActionCollectionIds().add(existingActionCollection.getId()); - resultDTO.getSavedActionCollectionMap().put(idFromJsonFile, existingActionCollection); - } else { - if (!permissionProvider.canCreateAction(parentPage)) { - throw new AppsmithException( - AppsmithError.ACL_NO_RESOURCE_FOUND, FieldName.PAGE, parentPage.getId()); - } - - if (importedApplication.getGitApplicationMetadata() != null) { - final String defaultApplicationId = importedApplication - .getGitApplicationMetadata() - .getDefaultApplicationId(); - if (actionsCollectionsInBranches.containsKey(actionCollection.getGitSyncId())) { - ActionCollection branchedActionCollection = - actionsCollectionsInBranches.get(actionCollection.getGitSyncId()); - populateDefaultResources( - actionCollection, branchedActionCollection, branchName); + existingActionCollection.updateForBulkWriteOperation(); + existingActionCollections.add(existingActionCollection); + resultDTO.getSavedActionCollectionIds().add(existingActionCollection.getId()); + resultDTO + .getSavedActionCollectionMap() + .put(idFromJsonFile, existingActionCollection); } else { - DefaultResources defaultResources = new DefaultResources(); - defaultResources.setApplicationId(defaultApplicationId); - defaultResources.setBranchName(branchName); - actionCollection.setDefaultResources(defaultResources); + if (!permissionProvider.canCreateAction(parentPage)) { + throw new AppsmithException( + AppsmithError.ACL_NO_RESOURCE_FOUND, + FieldName.PAGE, + parentPage.getId()); + } + + if (importedApplication.getGitApplicationMetadata() != null) { + final String defaultApplicationId = importedApplication + .getGitApplicationMetadata() + .getDefaultApplicationId(); + if (actionsCollectionsInBranches.containsKey( + actionCollection.getGitSyncId())) { + ActionCollection branchedActionCollection = + actionsCollectionsInBranches.get( + actionCollection.getGitSyncId()); + populateDefaultResources( + actionCollection, branchedActionCollection, branchName); + } else { + DefaultResources defaultResources = new DefaultResources(); + defaultResources.setApplicationId(defaultApplicationId); + defaultResources.setBranchName(branchName); + actionCollection.setDefaultResources(defaultResources); + } + } + + // this will generate the id and other auto generated fields e.g. createdAt + actionCollection.updateForBulkWriteOperation(); + generateAndSetPolicies(parentPage, actionCollection); + + // create or update default resources for the action + // values already set to defaultResources are kept unchanged + DefaultResourcesUtils.createDefaultIdsOrUpdateWithGivenResourceIds( + actionCollection, branchName); + + // generate gitSyncId if it's not present + if (actionCollection.getGitSyncId() == null) { + actionCollection.setGitSyncId( + actionCollection.getApplicationId() + "_" + new ObjectId()); + } + + // it's new actionCollection + newActionCollections.add(actionCollection); + resultDTO.getSavedActionCollectionIds().add(actionCollection.getId()); + resultDTO.getSavedActionCollectionMap().put(idFromJsonFile, actionCollection); } } - - // this will generate the id and other auto generated fields e.g. createdAt - actionCollection.updateForBulkWriteOperation(); - generateAndSetPolicies(parentPage, actionCollection); - - // create or update default resources for the action - // values already set to defaultResources are kept unchanged - DefaultResourcesUtils.createDefaultIdsOrUpdateWithGivenResourceIds( - actionCollection, branchName); - - // generate gitSyncId if it's not present - if (actionCollection.getGitSyncId() == null) { - actionCollection.setGitSyncId( - actionCollection.getApplicationId() + "_" + new ObjectId()); - } - - // it's new actionCollection - newActionCollections.add(actionCollection); - resultDTO.getSavedActionCollectionIds().add(actionCollection.getId()); - resultDTO.getSavedActionCollectionMap().put(idFromJsonFile, actionCollection); - } - } - log.info( - "Saving action collections in bulk. New: {}, Updated: {}", - newActionCollections.size(), - existingActionCollections.size()); - return repository - .bulkInsert(newActionCollections) - .then(repository.bulkUpdate(existingActionCollections)) - .thenReturn(resultDTO); - }); - }); + log.info( + "Saving action collections in bulk. New: {}, Updated: {}", + newActionCollections.size(), + existingActionCollections.size()); + return repository + .bulkInsert(newActionCollections) + .then(repository.bulkUpdate(existingActionCollections)) + .thenReturn(resultDTO); + }); + }) + .onErrorResume(e -> { + log.error("Error saving action collections", e); + return Mono.error(e); + }); } } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/ApplicationServiceCEImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/ApplicationServiceCEImpl.java index 79c732b1bd..b2cf5dcae2 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/ApplicationServiceCEImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/ApplicationServiceCEImpl.java @@ -289,6 +289,7 @@ public class ApplicationServiceCEImpl extends BaseService repository .updateById(appId, application, applicationPermission.getEditPermission()) .onErrorResume(error -> { + log.error("failed to update application {}", appId, error); if (error instanceof DuplicateKeyException) { // Error message : E11000 duplicate key error collection: appsmith.application index: // workspace_app_deleted_gitApplicationMetadata dup key: diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/NewActionServiceCEImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/NewActionServiceCEImpl.java index d670789574..8060c3dc09 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/NewActionServiceCEImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/NewActionServiceCEImpl.java @@ -1698,149 +1698,177 @@ public class NewActionServiceCEImpl extends BaseService { - Mono> actionsInCurrentAppMono = repository - .findByApplicationId(importedApplication.getId()) - .filter(newAction -> newAction.getGitSyncId() != null) - .collectMap(NewAction::getGitSyncId); + return Mono.just(application) + .flatMap(importedApplication -> { + Mono> actionsInCurrentAppMono = repository + .findByApplicationId(importedApplication.getId()) + .filter(newAction -> newAction.getGitSyncId() != null) + .collectMap(NewAction::getGitSyncId); - // find existing actions in all the branches of this application and put them in a map - Mono> actionsInOtherBranchesMono; - if (importedApplication.getGitApplicationMetadata() != null) { - final String defaultApplicationId = - importedApplication.getGitApplicationMetadata().getDefaultApplicationId(); - actionsInOtherBranchesMono = repository - .findByDefaultApplicationId(defaultApplicationId, Optional.empty()) - .filter(newAction -> newAction.getGitSyncId() != null) - .collectMap(NewAction::getGitSyncId); - } else { - actionsInOtherBranchesMono = Mono.just(Collections.emptyMap()); - } - - return Mono.zip(actionsInCurrentAppMono, actionsInOtherBranchesMono).flatMap(objects -> { - Map actionsInCurrentApp = objects.getT1(); - Map actionsInOtherBranches = objects.getT2(); - - List newNewActionList = new ArrayList<>(); - List existingNewActionList = new ArrayList<>(); - - final String workspaceId = importedApplication.getWorkspaceId(); - - ImportActionResultDTO importActionResultDTO = new ImportActionResultDTO(); - - // existing actions will be required when we'll delete the outdated actions later - importActionResultDTO.setExistingActions(actionsInCurrentApp.values()); - - for (NewAction newAction : importedNewActionList) { - if (newAction.getUnpublishedAction() == null - || !StringUtils.hasLength( - newAction.getUnpublishedAction().getPageId())) { - continue; - } - - NewPage parentPage = new NewPage(); - ActionDTO unpublishedAction = newAction.getUnpublishedAction(); - ActionDTO publishedAction = newAction.getPublishedAction(); - - // If pageId is missing in the actionDTO create a fallback pageId - final String fallbackParentPageId = unpublishedAction.getPageId(); - - if (unpublishedAction.getValidName() != null) { - unpublishedAction.setId(newAction.getId()); - parentPage = updatePageInAction( - unpublishedAction, pageNameMap, importActionResultDTO.getActionIdMap()); - sanitizeDatasourceInActionDTO(unpublishedAction, datasourceMap, pluginMap, workspaceId, false); - } - - if (publishedAction != null && publishedAction.getValidName() != null) { - publishedAction.setId(newAction.getId()); - if (!StringUtils.hasLength(publishedAction.getPageId())) { - publishedAction.setPageId(fallbackParentPageId); - } - NewPage publishedActionPage = updatePageInAction( - publishedAction, pageNameMap, importActionResultDTO.getActionIdMap()); - parentPage = parentPage == null ? publishedActionPage : parentPage; - sanitizeDatasourceInActionDTO(publishedAction, datasourceMap, pluginMap, workspaceId, false); - } - - newAction.makePristine(); - newAction.setWorkspaceId(workspaceId); - newAction.setApplicationId(importedApplication.getId()); - newAction.setPluginId(pluginMap.get(newAction.getPluginId())); - this.generateAndSetActionPolicies(parentPage, newAction); - - // Check if the action has gitSyncId and if it's already in DB - if (newAction.getGitSyncId() != null && actionsInCurrentApp.containsKey(newAction.getGitSyncId())) { - - // Since the resource is already present in DB, just update resource - NewAction existingAction = actionsInCurrentApp.get(newAction.getGitSyncId()); - updateExistingAction(existingAction, newAction, branchName, permissionProvider); - - // Add it to actions list that'll be updated in bulk - existingNewActionList.add(existingAction); - importActionResultDTO.getImportedActionIds().add(existingAction.getId()); - putActionIdInMap(existingAction, importActionResultDTO); + // find existing actions in all the branches of this application and put them in a map + Mono> actionsInOtherBranchesMono; + if (importedApplication.getGitApplicationMetadata() != null) { + final String defaultApplicationId = + importedApplication.getGitApplicationMetadata().getDefaultApplicationId(); + actionsInOtherBranchesMono = repository + .findByDefaultApplicationId(defaultApplicationId, Optional.empty()) + .filter(newAction -> newAction.getGitSyncId() != null) + .collectMap(NewAction::getGitSyncId); } else { - // check whether user has permission to add new action - if (!permissionProvider.canCreateAction(parentPage)) { - log.error( - "User does not have permission to create action in page with id: {}", - parentPage.getId()); - throw new AppsmithException( - AppsmithError.ACL_NO_RESOURCE_FOUND, FieldName.PAGE, parentPage.getId()); - } - - // this will generate the id and other auto generated fields e.g. createdAt - newAction.updateForBulkWriteOperation(); - - // set gitSyncId if doesn't exist - if (newAction.getGitSyncId() == null) { - newAction.setGitSyncId(newAction.getApplicationId() + "_" - + Instant.now().toString()); - } - - if (importedApplication.getGitApplicationMetadata() != null) { - // application is git connected, check if the action is already present in any other branch - if (actionsInOtherBranches.containsKey(newAction.getGitSyncId())) { - // action found in other branch, copy the default resources from that action - NewAction branchedAction = actionsInOtherBranches.get(newAction.getGitSyncId()); - populateDefaultResources(newAction, branchedAction, branchName); - } else { - // This is the first action we are saving with given gitSyncId in this instance - DefaultResources defaultResources = new DefaultResources(); - defaultResources.setApplicationId(importedApplication - .getGitApplicationMetadata() - .getDefaultApplicationId()); - defaultResources.setActionId(newAction.getId()); - defaultResources.setBranchName(branchName); - newAction.setDefaultResources(defaultResources); - } - } else { - DefaultResources defaultResources = new DefaultResources(); - defaultResources.setApplicationId(importedApplication.getId()); - defaultResources.setActionId(newAction.getId()); - newAction.setDefaultResources(defaultResources); - } - - // Add it to actions list that'll be inserted or updated in bulk - newNewActionList.add(newAction); - importActionResultDTO.getImportedActionIds().add(newAction.getId()); - putActionIdInMap(newAction, importActionResultDTO); + actionsInOtherBranchesMono = Mono.just(Collections.emptyMap()); } - } - log.info( - "Saving actions in bulk. New: {}, Updated: {}", - newNewActionList.size(), - existingNewActionList.size()); + return Mono.zip(actionsInCurrentAppMono, actionsInOtherBranchesMono) + .flatMap(objects -> { + Map actionsInCurrentApp = objects.getT1(); + Map actionsInOtherBranches = objects.getT2(); - // Save all the new actions in bulk - return repository - .bulkInsert(newNewActionList) - .then(repository.bulkUpdate(existingNewActionList)) - .thenReturn(importActionResultDTO); - }); - }); + List newNewActionList = new ArrayList<>(); + List existingNewActionList = new ArrayList<>(); + + final String workspaceId = importedApplication.getWorkspaceId(); + + ImportActionResultDTO importActionResultDTO = new ImportActionResultDTO(); + + // existing actions will be required when we'll delete the outdated actions later + importActionResultDTO.setExistingActions(actionsInCurrentApp.values()); + + for (NewAction newAction : importedNewActionList) { + if (newAction.getUnpublishedAction() == null + || !StringUtils.hasLength(newAction + .getUnpublishedAction() + .getPageId())) { + continue; + } + + NewPage parentPage = new NewPage(); + ActionDTO unpublishedAction = newAction.getUnpublishedAction(); + ActionDTO publishedAction = newAction.getPublishedAction(); + + // If pageId is missing in the actionDTO create a fallback pageId + final String fallbackParentPageId = unpublishedAction.getPageId(); + + if (unpublishedAction.getValidName() != null) { + unpublishedAction.setId(newAction.getId()); + parentPage = updatePageInAction( + unpublishedAction, pageNameMap, importActionResultDTO.getActionIdMap()); + sanitizeDatasourceInActionDTO( + unpublishedAction, datasourceMap, pluginMap, workspaceId, false); + } + + if (publishedAction != null && publishedAction.getValidName() != null) { + publishedAction.setId(newAction.getId()); + if (!StringUtils.hasLength(publishedAction.getPageId())) { + publishedAction.setPageId(fallbackParentPageId); + } + NewPage publishedActionPage = updatePageInAction( + publishedAction, pageNameMap, importActionResultDTO.getActionIdMap()); + parentPage = parentPage == null ? publishedActionPage : parentPage; + sanitizeDatasourceInActionDTO( + publishedAction, datasourceMap, pluginMap, workspaceId, false); + } + + newAction.makePristine(); + newAction.setWorkspaceId(workspaceId); + newAction.setApplicationId(importedApplication.getId()); + newAction.setPluginId(pluginMap.get(newAction.getPluginId())); + this.generateAndSetActionPolicies(parentPage, newAction); + + // Check if the action has gitSyncId and if it's already in DB + if (newAction.getGitSyncId() != null + && actionsInCurrentApp.containsKey(newAction.getGitSyncId())) { + + // Since the resource is already present in DB, just update resource + NewAction existingAction = actionsInCurrentApp.get(newAction.getGitSyncId()); + updateExistingAction(existingAction, newAction, branchName, permissionProvider); + + // Add it to actions list that'll be updated in bulk + existingNewActionList.add(existingAction); + importActionResultDTO + .getImportedActionIds() + .add(existingAction.getId()); + putActionIdInMap(existingAction, importActionResultDTO); + } else { + // check whether user has permission to add new action + if (!permissionProvider.canCreateAction(parentPage)) { + log.error( + "User does not have permission to create action in page with id: {}", + parentPage.getId()); + throw new AppsmithException( + AppsmithError.ACL_NO_RESOURCE_FOUND, + FieldName.PAGE, + parentPage.getId()); + } + + // this will generate the id and other auto generated fields e.g. createdAt + newAction.updateForBulkWriteOperation(); + + // set gitSyncId if doesn't exist + if (newAction.getGitSyncId() == null) { + newAction.setGitSyncId(newAction.getApplicationId() + "_" + + Instant.now().toString()); + } + + if (importedApplication.getGitApplicationMetadata() != null) { + // application is git connected, check if the action is already present in + // any other branch + if (actionsInOtherBranches.containsKey(newAction.getGitSyncId())) { + // action found in other branch, copy the default resources from that + // action + NewAction branchedAction = + actionsInOtherBranches.get(newAction.getGitSyncId()); + populateDefaultResources(newAction, branchedAction, branchName); + } else { + // This is the first action we are saving with given gitSyncId in this + // instance + DefaultResources defaultResources = new DefaultResources(); + defaultResources.setApplicationId(importedApplication + .getGitApplicationMetadata() + .getDefaultApplicationId()); + defaultResources.setActionId(newAction.getId()); + defaultResources.setBranchName(branchName); + newAction.setDefaultResources(defaultResources); + } + } else { + DefaultResources defaultResources = new DefaultResources(); + defaultResources.setApplicationId(importedApplication.getId()); + defaultResources.setActionId(newAction.getId()); + newAction.setDefaultResources(defaultResources); + } + + // Add it to actions list that'll be inserted or updated in bulk + newNewActionList.add(newAction); + importActionResultDTO + .getImportedActionIds() + .add(newAction.getId()); + putActionIdInMap(newAction, importActionResultDTO); + } + } + + log.info( + "Saving actions in bulk. New: {}, Updated: {}", + newNewActionList.size(), + existingNewActionList.size()); + + // Save all the new actions in bulk + return repository + .bulkInsert(newNewActionList) + .then(repository.bulkUpdate(existingNewActionList)) + .thenReturn(importActionResultDTO); + }); + }) + .onErrorResume(e -> { + log.error("Error importing actions", e); + return Mono.error(e); + }) + .elapsed() + .map(tuple -> { + log.debug( + "time to import {} actions: {} ms", + tuple.getT2().getImportedActionIds().size(), + tuple.getT1()); + return tuple.getT2(); + }); } @Override @@ -1892,7 +1920,7 @@ public class NewActionServiceCEImpl extends BaseService importApplicationInWorkspace( - String workspaceId, - ApplicationJson applicationJson, - String applicationId, - String branchName, - boolean appendToApp, + private Mono> importDatasources( + ApplicationJson importedDoc, + Mono> existingDatasourceMono, + Flux existingDatasourcesFlux, + Workspace workspace, + Mono> pluginMapMno, ImportApplicationPermissionProvider permissionProvider) { - /* - 1. Migrate resource to latest schema - 2. Fetch workspace by id - 3. Extract datasources and update plugin information - 4. Create new datasource if same datasource is not present - 5. Extract and save application - 6. Extract and save pages in the application - 7. Extract and save actions in the application - */ - ApplicationJson importedDoc = JsonSchemaMigration.migrateApplicationToLatestSchema(applicationJson); - - // check for validation error and raise exception if error found - String errorField = validateApplicationJson(importedDoc); - if (!errorField.isEmpty()) { - log.error("Error in importing application. Field {} is missing", errorField); - return Mono.error(new AppsmithException(AppsmithError.NO_RESOURCE_FOUND, errorField, INVALID_JSON_FILE)); - } - - Map pluginMap = new HashMap<>(); - Map datasourceMap = new HashMap<>(); - Map pageNameMap = new HashMap<>(); - - Application importedApplication = importedDoc.getExportedApplication(); - - List importedDatasourceList = importedDoc.getDatasourceList(); - List importedNewPageList = importedDoc.getPageList(); - List importedNewActionList = importedDoc.getActionList(); - List importedActionCollectionList = - CollectionUtils.isEmpty(importedDoc.getActionCollectionList()) - ? new ArrayList<>() - : importedDoc.getActionCollectionList(); - - Mono workspaceMono = workspaceService - .findById(workspaceId, permissionProvider.getRequiredPermissionOnTargetWorkspace()) - .switchIfEmpty(Mono.defer(() -> { - log.error( - "No workspace found with id: {} and permission: {}", - workspaceId, - permissionProvider.getRequiredPermissionOnTargetWorkspace()); - return Mono.error(new AppsmithException( - AppsmithError.ACL_NO_RESOURCE_FOUND, FieldName.WORKSPACE, workspaceId)); - })); - - /* We need to take care of the null case in case someone is trying to import an older app where JS libs did - not exist */ - List customJSLibs = - importedDoc.getCustomJSLibList() == null ? new ArrayList<>() : importedDoc.getCustomJSLibList(); - - Mono installedJSLibMono = Flux.fromIterable(customJSLibs) - .flatMap(customJSLib -> { - customJSLib.setId(null); - customJSLib.setCreatedAt(null); - customJSLib.setUpdatedAt(null); - return customJSLibService.persistCustomJSLibMetaDataIfDoesNotExistAndGetDTO(customJSLib, false); - }) - .collectList() - .map(jsLibDTOList -> { - importedApplication.setUnpublishedCustomJSLibs(new HashSet<>(jsLibDTOList)); - return importedApplication; - }); - - Mono currUserMono = sessionUserService.getCurrentUser().cache(); - final Flux existingDatasourceFlux = datasourceService - .getAllByWorkspaceIdWithStorages(workspaceId, Optional.empty()) - .cache(); - - assert importedApplication != null : "Received invalid application object!"; - if (importedApplication.getApplicationVersion() == null) { - importedApplication.setApplicationVersion(ApplicationVersion.EARLIEST_VERSION); - } - - final List publishedPages = importedApplication.getPublishedPages(); - importedApplication.setViewMode(false); - final List unpublishedPages = importedApplication.getPages(); - - importedApplication.setPages(null); - importedApplication.setPublishedPages(null); - // re-setting the properties - importedApplication.setForkWithConfiguration(null); - importedApplication.setExportWithConfiguration(null); - // Start the stopwatch to log the execution time - Stopwatch stopwatch = new Stopwatch(AnalyticsEvents.IMPORT.getEventName()); - - final Mono importedApplicationMono = pluginRepository - .findAll() - .map(plugin -> { - final String pluginReference = StringUtils.isEmpty(plugin.getPluginName()) - ? plugin.getPackageName() - : plugin.getPluginName(); - pluginMap.put(pluginReference, plugin.getId()); - return plugin; - }) - .then(installedJSLibMono) - .then(workspaceMono) - .flatMap(workspace -> { - Mono> existingDatasourceMono; - // Check if the request is to hydrate the application to DB for particular branch - // Application id will be present for GIT sync - if (!StringUtils.isEmpty(applicationId)) { - // No need to hydrate the datasource as we expect user will configure the datasource - existingDatasourceMono = existingDatasourceFlux.collectList(); - } else { - existingDatasourceMono = Mono.just(new ArrayList<>()); - } - return Mono.zip(existingDatasourceMono, Mono.just(workspace)); - }) - .zipWhen(objects -> workspaceService.getDefaultEnvironmentId(workspaceId, null)) + return Mono.zip( + existingDatasourceMono, + workspaceService.getDefaultEnvironmentId(workspace.getId(), null), + pluginMapMno) .flatMapMany(objects -> { - List existingDatasources = objects.getT1().getT1(); - Workspace workspace = objects.getT1().getT2(); + List existingDatasources = objects.getT1(); String environmentId = objects.getT2(); + Map pluginMap = objects.getT3(); + List importedDatasourceList = importedDoc.getDatasourceList(); if (CollectionUtils.isEmpty(importedDatasourceList)) { return Mono.empty(); } @@ -1100,6 +991,7 @@ public class ImportExportApplicationServiceCEImpl implements ImportExportApplica AppsmithError.UNKNOWN_PLUGIN_REFERENCE, datasource.getPluginId())); } } + return Flux.fromIterable(importedDatasourceList) // Check for duplicate datasources to avoid duplicates in target workspace .flatMap(datasourceStorage -> { @@ -1140,7 +1032,7 @@ public class ImportExportApplicationServiceCEImpl implements ImportExportApplica // This is explicitly copied over from the map we created before datasourceStorage.setPluginId(pluginMap.get(datasourceStorage.getPluginId())); - datasourceStorage.setWorkspaceId(workspaceId); + datasourceStorage.setWorkspaceId(workspace.getId()); datasourceStorage.setEnvironmentId(environmentId); // Check if any decrypted fields are present for datasource @@ -1154,7 +1046,7 @@ public class ImportExportApplicationServiceCEImpl implements ImportExportApplica } return createUniqueDatasourceIfNotPresent( - existingDatasourceFlux, + existingDatasourcesFlux, datasourceStorage, workspace, environmentId, @@ -1162,346 +1054,184 @@ public class ImportExportApplicationServiceCEImpl implements ImportExportApplica }); }) .collectMap(Datasource::getName, Datasource::getId) - .flatMap(map -> { - datasourceMap.putAll(map); - // 1. Assign the policies for the imported application - // 2. Check for possible duplicate names, - // 3. Save the updated application - - return Mono.just(importedApplication) - .zipWith(currUserMono) - .map(objects -> { - Application application = objects.getT1(); - application.setModifiedBy(objects.getT2().getUsername()); - return application; - }) - .flatMap(application -> { - importedApplication.setWorkspaceId(workspaceId); - // Application Id will be present for GIT sync - if (!StringUtils.isEmpty(applicationId)) { - return applicationService - .findById( - applicationId, - permissionProvider.getRequiredPermissionOnTargetApplication()) - .switchIfEmpty(Mono.defer(() -> { - log.error( - "No application found with id: {} and permission: {}", - applicationId, - permissionProvider.getRequiredPermissionOnTargetApplication()); - return Mono.error(new AppsmithException( - AppsmithError.ACL_NO_RESOURCE_FOUND, - FieldName.APPLICATION, - applicationId)); - })) - .flatMap(existingApplication -> { - if (appendToApp) { - // When we are appending the pages to the existing application - // e.g. import template we are only importing this in unpublished - // version. At the same time we want to keep the existing page ref - unpublishedPages.addAll(existingApplication.getPages()); - return Mono.just(existingApplication); - } - // This method sets the published mode properties in the imported - // application.When a user imports an application from the git repo, - // since the git only stores the unpublished version, the current - // deployed version in the newly imported app is not updated. - // This function sets the initial deployed version to the same as the - // edit mode one. - setPublishedApplicationProperties(importedApplication); - setPropertiesToExistingApplication( - importedApplication, existingApplication); - // We are expecting the changes present in DB are committed to git - // directory - // so that these won't be lost when we are pulling changes from remote - // and - // rehydrate the application. We are now rehydrating the application - // with/without - // the changes from remote - // We are using the save instead of update as we are using @Encrypted - // for GitAuth - Mono parentApplicationMono; - if (existingApplication.getGitApplicationMetadata() != null) { - parentApplicationMono = - applicationService.findById(existingApplication - .getGitApplicationMetadata() - .getDefaultApplicationId()); - } else { - parentApplicationMono = Mono.just(existingApplication); - } - - return parentApplicationMono.flatMap(application1 -> { - // Set the policies from the defaultApplication - existingApplication.setPolicies(application1.getPolicies()); - importedApplication.setPolicies(application1.getPolicies()); - return applicationService - .save(existingApplication) - .onErrorResume(DuplicateKeyException.class, error -> { - if (error.getMessage() != null) { - return applicationPageService - .createOrUpdateSuffixedApplication( - existingApplication, - existingApplication.getName(), - 0); - } - throw error; - }); - }); - }); - } - return applicationPageService.createOrUpdateSuffixedApplication( - application, application.getName(), 0); - }); + .onErrorResume(error -> { + log.error("Error importing datasources", error); + return Mono.error(error); }) - .flatMap(savedApp -> importThemes(savedApp, importedDoc, appendToApp)) - .flatMap(savedApp -> { - importedApplication.setId(savedApp.getId()); - if (savedApp.getGitApplicationMetadata() != null) { - importedApplication.setGitApplicationMetadata(savedApp.getGitApplicationMetadata()); + .elapsed() + .map(tuple -> { + log.debug("Time taken to import datasources: {} ms", tuple.getT1()); + return tuple.getT2(); + }); + } + + private Mono getImportApplicationMono( + Application importedApplication, + String applicationId, + boolean appendToApp, + String workspaceId, + ImportApplicationPermissionProvider permissionProvider, + Mono currUserMono, + Mono> installedJsLibsMono) { + Mono importApplicationMono = Mono.just(importedApplication) + .map(application -> { + if (application.getApplicationVersion() == null) { + application.setApplicationVersion(ApplicationVersion.EARLIEST_VERSION); } - - // Import and save pages, also update the pages related fields in saved application - assert importedNewPageList != null : "Unable to find pages in the imported application"; - - if (appendToApp) { - // add existing pages to importedApplication so that they are not lost - // when we update application from importedApplication - importedApplication.setPages(savedApp.getPages()); - } - - // For git-sync this will not be empty - Mono> existingPagesMono = newPageService - .findNewPagesByApplicationId(importedApplication.getId(), Optional.empty()) - .collectList() - .cache(); - - Flux importNewPageFlux = importAndSavePages( - importedNewPageList, savedApp, branchName, existingPagesMono, permissionProvider); - Flux importedNewPagesMono; - - if (appendToApp) { - // we need to rename page if there is a conflict - // also need to remap the renamed page - importedNewPagesMono = updateNewPagesBeforeMerge(existingPagesMono, importedNewPageList) - .flatMapMany(newToOldNameMap -> importNewPageFlux.map(newPage -> { - // we need to map the newly created page with old name - // because other related resources e.g. actions will refer the page with old name - String newPageName = - newPage.getUnpublishedPage().getName(); - String oldPageName = newToOldNameMap.get(newPageName); - if (!newPageName.equals(oldPageName)) { - renamePageInActions(importedNewActionList, oldPageName, newPageName); - renamePageInActionCollections( - importedActionCollectionList, oldPageName, newPageName); - unpublishedPages.stream() - .filter(applicationPage -> oldPageName.equals(applicationPage.getId())) - .findAny() - .ifPresent(applicationPage -> applicationPage.setId(newPageName)); - } - return newPage; - })); - } else { - importedNewPagesMono = importNewPageFlux; - } - importedNewPagesMono = importedNewPagesMono.map(newPage -> { - // Save the map of pageName and NewPage - if (newPage.getUnpublishedPage() != null - && newPage.getUnpublishedPage().getName() != null) { - pageNameMap.put(newPage.getUnpublishedPage().getName(), newPage); - } - if (newPage.getPublishedPage() != null - && newPage.getPublishedPage().getName() != null) { - pageNameMap.put(newPage.getPublishedPage().getName(), newPage); - } - return newPage; - }); - - return importedNewPagesMono - .collectList() - .map(newPageList -> { - log.info("New pages imported for application: {}", savedApp.getId()); - Map> applicationPages = new HashMap<>(); - applicationPages.put(EDIT, unpublishedPages); - applicationPages.put(VIEW, publishedPages); - - Iterator unpublishedPageItr = unpublishedPages.iterator(); - while (unpublishedPageItr.hasNext()) { - ApplicationPage applicationPage = unpublishedPageItr.next(); - NewPage newPage = pageNameMap.get(applicationPage.getId()); - if (newPage == null) { - if (appendToApp) { - // Don't remove the page reference if doing the partial import and appending - // to the existing application - continue; - } - log.debug( - "Unable to find the page during import for appId {}, with name {}", - applicationId, - applicationPage.getId()); - unpublishedPageItr.remove(); - } else { - applicationPage.setId(newPage.getId()); - applicationPage.setDefaultPageId( - newPage.getDefaultResources().getPageId()); - // Keep the existing page as the default one - if (appendToApp) { - applicationPage.setIsDefault(false); - } - } - } - - Iterator publishedPagesItr; - // Remove the newly added pages from merge app flow. Keep only the existing page from - // the old app - if (appendToApp) { - List existingPagesId = savedApp.getPublishedPages().stream() - .map(applicationPage -> applicationPage.getId()) - .collect(Collectors.toList()); - List publishedApplicationPages = publishedPages.stream() - .filter(applicationPage -> - existingPagesId.contains(applicationPage.getId())) - .collect(Collectors.toList()); - applicationPages.replace(VIEW, publishedApplicationPages); - publishedPagesItr = publishedApplicationPages.iterator(); - } else { - publishedPagesItr = publishedPages.iterator(); - } - while (publishedPagesItr.hasNext()) { - ApplicationPage applicationPage = publishedPagesItr.next(); - NewPage newPage = pageNameMap.get(applicationPage.getId()); - if (newPage == null) { - log.debug( - "Unable to find the page during import for appId {}, with name {}", - applicationId, - applicationPage.getId()); - if (!appendToApp) { - publishedPagesItr.remove(); - } - } else { - applicationPage.setId(newPage.getId()); - applicationPage.setDefaultPageId( - newPage.getDefaultResources().getPageId()); - if (appendToApp) { - applicationPage.setIsDefault(false); - } - } - } - - return applicationPages; - }) - .flatMap(applicationPages -> { - // During partial import/appending to the existing application keep the resources - // attached to the application: - // Delete the invalid resources (which are not the part of applicationJsonDTO) in - // the git flow only - if (!StringUtils.isEmpty(applicationId) && !appendToApp) { - Set validPageIds = applicationPages.get(EDIT).stream() - .map(ApplicationPage::getId) - .collect(Collectors.toSet()); - - validPageIds.addAll(applicationPages.get(VIEW).stream() - .map(ApplicationPage::getId) - .collect(Collectors.toSet())); - - return existingPagesMono.flatMap(existingPagesList -> { - Set invalidPageIds = new HashSet<>(); - for (NewPage newPage : existingPagesList) { - if (!validPageIds.contains(newPage.getId())) { - invalidPageIds.add(newPage.getId()); - } - } - - // Delete the pages which were removed during git merge operation - // This does not apply to the traditional import via file approach - return Flux.fromIterable(invalidPageIds) - .flatMap(applicationPageService::deleteWithoutPermissionUnpublishedPage) - .flatMap(page -> newPageService - .archiveWithoutPermissionById(page.getId()) - .onErrorResume(e -> { - log.debug( - "Unable to archive page {} with error {}", - page.getId(), - e.getMessage()); - return Mono.empty(); - })) - .then() - .thenReturn(applicationPages); - }); - } - return Mono.just(applicationPages); - }); + application.setViewMode(false); + application.setForkWithConfiguration(null); + application.setExportWithConfiguration(null); + application.setWorkspaceId(workspaceId); + application.setIsPublic(null); + application.setPolicies(null); + application.setPages(null); + application.setPublishedPages(null); + return application; }) - .map(applicationPageMap -> { - log.info( - "Imported pages. Edit: {}, View: {}", - applicationPageMap.get(EDIT).size(), - applicationPageMap.get(VIEW).size()); - log.info("Importing actions for applicationId: {}, ", importedApplication.getId()); - // Set page sequence based on the order for published and unpublished pages - importedApplication.setPages(applicationPageMap.get(EDIT)); - importedApplication.setPublishedPages(applicationPageMap.get(VIEW)); - return applicationPageMap; - }) - .flatMap(unused -> newActionService.importActions( - importedNewActionList, - importedApplication, - branchName, - pageNameMap, - pluginMap, - datasourceMap, - permissionProvider)) - .flatMap(importActionResultDTO -> { - log.info( - "Actions imported. applicationId {}, result: {}", - importedApplication.getId(), - importActionResultDTO.getGist()); - // Updating the existing application for git-sync - // During partial import/appending to the existing application keep the resources - // attached to the application: - // Delete the invalid resources (which are not the part of applicationJsonDTO) in - // the git flow only - if (!StringUtils.isEmpty(applicationId) && !appendToApp) { - // Remove unwanted actions - Set invalidActionIds = new HashSet<>(); - for (NewAction action : importActionResultDTO.getExistingActions()) { - if (!importActionResultDTO.getImportedActionIds().contains(action.getId())) { - invalidActionIds.add(action.getId()); + .zipWith(installedJsLibsMono) + .map(tuple -> { + Application application = tuple.getT1(); + List customJSLibApplicationDTOList = tuple.getT2(); + application.setUnpublishedCustomJSLibs(new HashSet<>(customJSLibApplicationDTOList)); + return application; + }); + + importApplicationMono = importApplicationMono.zipWith(currUserMono).map(objects -> { + Application application = objects.getT1(); + application.setModifiedBy(objects.getT2().getUsername()); + return application; + }); + + if (StringUtils.isEmpty(applicationId)) { + importApplicationMono = importApplicationMono.flatMap(application -> { + return applicationPageService.createOrUpdateSuffixedApplication(application, application.getName(), 0); + }); + } else { + Mono existingApplicationMono = applicationService + .findById(applicationId, permissionProvider.getRequiredPermissionOnTargetApplication()) + .switchIfEmpty(Mono.defer(() -> { + log.error( + "No application found with id: {} and permission: {}", + applicationId, + permissionProvider.getRequiredPermissionOnTargetApplication()); + return Mono.error(new AppsmithException( + AppsmithError.ACL_NO_RESOURCE_FOUND, FieldName.APPLICATION, applicationId)); + })) + .cache(); + + // this can be a git sync, import page from template, update app with json, restore snapshot + if (appendToApp) { // we don't need to do anything with the imported application + importApplicationMono = existingApplicationMono; + } else { + importApplicationMono = importApplicationMono + .zipWith(existingApplicationMono) + .map(objects -> { + Application newApplication = objects.getT1(); + Application existingApplication = objects.getT2(); + // This method sets the published mode properties in the imported + // application.When a user imports an application from the git repo, + // since the git only stores the unpublished version, the current + // deployed version in the newly imported app is not updated. + // This function sets the initial deployed version to the same as the + // edit mode one. + setPublishedApplicationProperties(newApplication); + setPropertiesToExistingApplication(newApplication, existingApplication); + return existingApplication; + }) + .flatMap(application -> { + Mono parentApplicationMono; + if (application.getGitApplicationMetadata() != null) { + parentApplicationMono = applicationService.findById( + application.getGitApplicationMetadata().getDefaultApplicationId()); + } else { + parentApplicationMono = Mono.just(application); } - } - log.info("Deleting {} actions which are no more used", invalidActionIds.size()); - return Flux.fromIterable(invalidActionIds) - .flatMap(actionId -> newActionService - .deleteUnpublishedAction(actionId) - // return an empty action so that the filter can remove it from the list - .onErrorResume(throwable -> { - log.debug("Failed to delete action with id {} during import", actionId); - log.error(throwable.getMessage()); - return Mono.empty(); - })) - .then() - .thenReturn(importActionResultDTO); - } - return Mono.just(importActionResultDTO); + return Mono.zip(Mono.just(application), parentApplicationMono); + }) + .flatMap(objects -> { + Application application = objects.getT1(); + Application parentApplication = objects.getT2(); + application.setPolicies(parentApplication.getPolicies()); + return applicationService + .save(application) + .onErrorResume(DuplicateKeyException.class, error -> { + if (error.getMessage() != null) { + return applicationPageService.createOrUpdateSuffixedApplication( + application, application.getName(), 0); + } + throw error; + }); + }); + } + } + return importApplicationMono + .elapsed() + .map(tuples -> { + log.debug("time to create or update application object: {}", tuples.getT1()); + return tuples.getT2(); }) - .flatMap(importActionResultDTO -> { - log.info("Importing action collections for applicationId {}", importedApplication.getId()); - return actionCollectionService - .importActionCollections( - importActionResultDTO, - importedApplication, - branchName, - importedActionCollectionList, - pluginMap, - pageNameMap, - permissionProvider) - .zipWith(Mono.just(importActionResultDTO)); + .onErrorResume(error -> { + log.error("Error while creating or updating application object", error); + return Mono.error(error); + }); + } + + private Mono> getPluginMapMono() { + return pluginRepository + .findAll() + .collectList() + .map(plugins -> { + Map pluginMap = new HashMap<>(); + plugins.forEach(plugin -> { + final String pluginReference = StringUtils.isEmpty(plugin.getPluginName()) + ? plugin.getPackageName() + : plugin.getPluginName(); + pluginMap.put(pluginReference, plugin.getId()); + }); + return pluginMap; }) - .flatMap(resultDtos -> { - ImportActionCollectionResultDTO importActionCollectionResultDTO = resultDtos.getT1(); - ImportActionResultDTO importActionResultDTO = resultDtos.getT2(); + .elapsed() + .map(tuples -> { + log.debug("time to get plugin map: {}", tuples.getT1()); + return tuples.getT2(); + }); + } + + private Mono> getPageNameMapMono( + Mono, Map>> importedNewPagesMono) { + return importedNewPagesMono.map(objects -> { + Map pageNameMap = new HashMap<>(); + objects.getT1().forEach(newPage -> { + // Save the map of pageName and NewPage + if (newPage.getUnpublishedPage() != null + && newPage.getUnpublishedPage().getName() != null) { + pageNameMap.put(newPage.getUnpublishedPage().getName(), newPage); + } + if (newPage.getPublishedPage() != null + && newPage.getPublishedPage().getName() != null) { + pageNameMap.put(newPage.getPublishedPage().getName(), newPage); + } + }); + return pageNameMap; + }); + } + + private Mono> setCollectionIdToActionsMono( + Mono importActionsMono, + Mono importActionCollectionMono, + Mono importApplicationMono, + String applicationId, + boolean appendToApp) { + return Mono.zip(importActionsMono, importActionCollectionMono, importApplicationMono) + .flatMap(objects -> { + ImportActionResultDTO importActionResultDTO = objects.getT1(); + ImportActionCollectionResultDTO importActionCollectionResultDTO = objects.getT2(); + Application importedApplication1 = objects.getT3(); + List savedCollectionIds = importActionCollectionResultDTO.getSavedActionCollectionIds(); log.info( "Action collections imported. applicationId {}, result: {}", - importedApplication.getId(), + importedApplication1.getId(), importActionCollectionResultDTO.getGist()); return newActionService .updateActionsWithImportedCollectionIds( @@ -1509,7 +1239,7 @@ public class ImportExportApplicationServiceCEImpl implements ImportExportApplica .flatMap(actionAndCollectionMapsDTO -> { log.info( "Updated actions with imported collection ids. applicationId {}", - importedApplication.getId()); + importedApplication1.getId()); // Updating the existing application for git-sync // During partial import/appending to the existing application keep the resources // attached to the application: @@ -1546,14 +1276,24 @@ public class ImportExportApplicationServiceCEImpl implements ImportExportApplica }) .zipWith(Mono.just(importActionResultDTO)); }) + .onErrorResume(error -> { + log.error("Error while updating collection id to actions", error); + return Mono.error(error); + }); + } + + private Mono> setActionIdInPages( + Mono> updateCollectionIdToActionsMono, + Mono, Map>> importedNewPagesMono, + String branchName) { + return Mono.zip(updateCollectionIdToActionsMono, importedNewPagesMono) .flatMap(objects -> { - ImportedActionAndCollectionMapsDTO actionAndCollectionMapsDTO = objects.getT1(); - ImportActionResultDTO importActionResultDTO = objects.getT2(); - // Don't update gitAuth as we are using @Encrypted for private key - importedApplication.setGitApplicationMetadata(null); - // Map layoutOnLoadActions ids with relevant actions - return newPageService - .findNewPagesByApplicationId(importedApplication.getId(), Optional.empty()) + ImportedActionAndCollectionMapsDTO actionAndCollectionMapsDTO = + objects.getT1().getT1(); + ImportActionResultDTO importActionResultDTO = + objects.getT1().getT2(); + List newPages = objects.getT2().getT1(); + return Flux.fromIterable(newPages) .flatMap(newPage -> { if (newPage.getDefaultResources() != null) { newPage.getDefaultResources().setBranchName(branchName); @@ -1566,47 +1306,634 @@ public class ImportExportApplicationServiceCEImpl implements ImportExportApplica }) .collectList() .flatMapMany(newPageService::saveAll) - .then(applicationService.update(importedApplication.getId(), importedApplication)) - .then(sendImportExportApplicationAnalyticsEvent( - importedApplication.getId(), AnalyticsEvents.IMPORT)) - .zipWith(currUserMono) - .flatMap(tuple -> { - Application application = tuple.getT1(); - stopwatch.stopTimer(); - stopwatch.stopAndLogTimeInMillis(); - final Map data = Map.of( - FieldName.APPLICATION_ID, - application.getId(), - FieldName.WORKSPACE_ID, - application.getWorkspaceId(), - "pageCount", - applicationJson.getPageList().size(), - "actionCount", - applicationJson.getActionList().size(), - "JSObjectCount", - applicationJson - .getActionCollectionList() - .size(), - FieldName.FLOW_NAME, - stopwatch.getFlow(), - "executionTime", - stopwatch.getExecutionTime()); - return analyticsService - .sendEvent( - AnalyticsEvents.UNIT_EXECUTION_TIME.getEventName(), - tuple.getT2().getUsername(), - data) - .thenReturn(application); - }); + .collectList(); }) .onErrorResume(throwable -> { - String errorMessage = ImportExportUtils.getErrorMessage(throwable); - log.error("Error importing application. Error: {}", errorMessage, throwable); - return Mono.error( - new AppsmithException(AppsmithError.GENERIC_JSON_IMPORT_ERROR, workspaceId, errorMessage)); + log.error("Failed to set action ids in pages", throwable); + return Mono.error(throwable); + }); + } + + private Mono createImportActionCollectionMono( + List importedActionCollectionList, + Mono importApplicationMono, + Mono, Map>> importedNewPagesMono, + Mono importActionsMono, + Mono> pluginMapMno, + Mono> pageNameMapMono, + ImportApplicationPermissionProvider permissionProvider, + boolean appendToApp, + String branchName) { + Mono> importedActionCollectionMono = Mono.just(importedActionCollectionList); + + if (appendToApp) { + importedActionCollectionMono = importedActionCollectionMono + .zipWith(importedNewPagesMono) + .map(objects -> { + List importedActionCollectionList1 = objects.getT1(); + List importedNewPages = objects.getT2().getT1(); + Map newToOldNameMap = objects.getT2().getT2(); + + for (NewPage newPage : importedNewPages) { + String newPageName = newPage.getUnpublishedPage().getName(); + String oldPageName = newToOldNameMap.get(newPageName); + + if (!newPageName.equals(oldPageName)) { + renamePageInActionCollections(importedActionCollectionList1, oldPageName, newPageName); + } + } + return importedActionCollectionList1; + }); + } + + return Mono.zip( + importActionsMono, + importApplicationMono, + importedActionCollectionMono, + pluginMapMno, + pageNameMapMono) + .flatMap(objects -> { + log.info("Importing action collections"); + return actionCollectionService.importActionCollections( + objects.getT1(), + objects.getT2(), + branchName, + objects.getT3(), + objects.getT4(), + objects.getT5(), + permissionProvider); + }) + .onErrorResume(throwable -> { + log.error("Error importing action collections", throwable); + return Mono.error(throwable); + }); + } + + Mono getImportActionsMono( + ApplicationJson importedDoc, + Mono, Map>> importedNewPagesMono, + Mono importApplicationMono, + Mono> pageNameMapMono, + Mono> pluginMapMno, + boolean appendToApp, + Workspace workspace, + String applicationId, + String branchName, + ImportApplicationPermissionProvider permissionProvider) { + List importedNewActionList = importedDoc.getActionList(); + + final Flux existingDatasourceFlux = datasourceService + .getAllByWorkspaceIdWithStorages(workspace.getId(), Optional.empty()) + .cache(); + + Mono> existingDatasourceMono = + getExistingDatasourceMono(applicationId, existingDatasourceFlux); + Mono> datasourceMapMono = importDatasources( + importedDoc, + existingDatasourceMono, + existingDatasourceFlux, + workspace, + pluginMapMno, + permissionProvider); + + Mono> importedNewActionMono = Mono.just(importedNewActionList); + if (appendToApp) { + importedNewActionMono = importedNewActionMono + .zipWith(importedNewPagesMono) + .map(objects -> { + List importedNewActionList1 = objects.getT1(); + List importedNewPages = objects.getT2().getT1(); + Map newToOldNameMap = objects.getT2().getT2(); + + for (NewPage newPage : importedNewPages) { + String newPageName = newPage.getUnpublishedPage().getName(); + String oldPageName = newToOldNameMap.get(newPageName); + + if (!newPageName.equals(oldPageName)) { + renamePageInActions(importedNewActionList1, oldPageName, newPageName); + } + } + return importedNewActionList1; + }); + } + + return Mono.zip(importedNewActionMono, importApplicationMono, pageNameMapMono, pluginMapMno, datasourceMapMono) + .flatMap(objects -> newActionService.importActions( + objects.getT1(), + objects.getT2(), + branchName, + objects.getT3(), + objects.getT4(), + objects.getT5(), + permissionProvider)) + .flatMap(importActionResultDTO -> { + log.info("Actions imported. result: {}", importActionResultDTO.getGist()); + // Updating the existing application for git-sync + // During partial import/appending to the existing application keep the resources + // attached to the application: + // Delete the invalid resources (which are not the part of applicationJsonDTO) in + // the git flow only + if (!StringUtils.isEmpty(applicationId) + && !appendToApp + && CollectionUtils.isNotEmpty(importActionResultDTO.getExistingActions())) { + // Remove unwanted actions + Set invalidActionIds = new HashSet<>(); + for (NewAction action : importActionResultDTO.getExistingActions()) { + if (!importActionResultDTO.getImportedActionIds().contains(action.getId())) { + invalidActionIds.add(action.getId()); + } + } + log.info("Deleting {} actions which are no more used", invalidActionIds.size()); + return Flux.fromIterable(invalidActionIds) + .flatMap(actionId -> newActionService + .deleteUnpublishedAction(actionId) + // return an empty action so that the filter can remove it from the list + .onErrorResume(throwable -> { + log.debug("Failed to delete action with id {} during import", actionId); + log.error(throwable.getMessage()); + return Mono.empty(); + })) + .then() + .thenReturn(importActionResultDTO); + } + return Mono.just(importActionResultDTO); + }) + .onErrorResume(throwable -> { + log.error("Error while importing actions and deleting unused ones", throwable); + return Mono.error(throwable); + }); + } + + private Mono> importUnpublishedPages( + List editModeApplicationPages, + boolean appendToApp, + Mono importApplicationMono, + Mono, Map>> importedNewPagesMono) { + Mono> unpublishedPagesMono = Mono.just(editModeApplicationPages); + if (appendToApp) { + unpublishedPagesMono = unpublishedPagesMono + .zipWith(importApplicationMono) + .map(objects -> { + Application application = objects.getT2(); + List applicationPages = objects.getT1(); + applicationPages.addAll(application.getPages()); + return applicationPages; + }) + .zipWith(importedNewPagesMono) + .map(objects -> { + List unpublishedPages = objects.getT1(); + Map newToOldNameMap = objects.getT2().getT2(); + List importedPages = objects.getT2().getT1(); + for (NewPage newPage : importedPages) { + // we need to map the newly created page with old name + // because other related resources e.g. actions will refer the page with old name + String newPageName = newPage.getUnpublishedPage().getName(); + if (newToOldNameMap.containsKey(newPageName)) { + String oldPageName = newToOldNameMap.get(newPageName); + unpublishedPages.stream() + .filter(applicationPage -> oldPageName.equals(applicationPage.getId())) + .findAny() + .ifPresent(applicationPage -> applicationPage.setId(newPageName)); + } + } + return unpublishedPages; + }); + } + return unpublishedPagesMono; + } + + private Mono, Map>> getImportNewPagesMono( + List importedNewPageList, + Mono> existingPagesMono, + Mono importApplicationMono, + boolean appendToApp, + String branchName, + ImportApplicationPermissionProvider permissionProvider) { + return Mono.just(importedNewPageList) + .zipWith(existingPagesMono) + .map(objects -> { + List importedNewPages = objects.getT1(); + List existingPages = objects.getT2(); + Map newToOldNameMap; + if (appendToApp) { + newToOldNameMap = updateNewPagesBeforeMerge(existingPages, importedNewPages); + } else { + newToOldNameMap = Map.of(); + } + return Tuples.of(importedNewPages, newToOldNameMap); + }) + .zipWith(importApplicationMono) + .flatMap(objects -> { + List importedNewPages = objects.getT1().getT1(); + Map newToOldNameMap = objects.getT1().getT2(); + Application application = objects.getT2(); + return importAndSavePages( + importedNewPages, application, branchName, existingPagesMono, permissionProvider) + .collectList() + .zipWith(Mono.just(newToOldNameMap)); + }) + .onErrorResume(throwable -> { + log.error("Error importing pages", throwable); + return Mono.error(throwable); + }) + .elapsed() + .map(objects -> { + log.debug("time to import {} pages: {}", objects.getT2().size(), objects.getT1()); + return objects.getT2(); + }); + } + + Mono savePagesToApplicationMono( + Application importedApplication, + Mono> pageNameMapMono, + Mono applicationMono, + boolean appendToApp, + String applicationId, + Mono> existingPagesMono, + Mono, Map>> importedNewPagesMono) { + + List editModeApplicationPages = importedApplication.getPages(); + List publishedModeApplicationPages = importedApplication.getPublishedPages(); + + Mono> unpublishedPagesMono = + importUnpublishedPages(editModeApplicationPages, appendToApp, applicationMono, importedNewPagesMono); + + Mono> publishedPagesMono = Mono.just(publishedModeApplicationPages); + + Mono>> applicationPagesMono = Mono.zip( + unpublishedPagesMono, publishedPagesMono, pageNameMapMono, applicationMono) + .map(objects -> { + List unpublishedPages = objects.getT1(); + List publishedPages = objects.getT2(); + Map pageNameMap = objects.getT3(); + Application savedApp = objects.getT4(); + + log.debug("New pages imported for application: {}", savedApp.getId()); + Map> applicationPages = new HashMap<>(); + applicationPages.put(EDIT, unpublishedPages); + applicationPages.put(VIEW, publishedPages); + + Iterator unpublishedPageItr = unpublishedPages.iterator(); + while (unpublishedPageItr.hasNext()) { + ApplicationPage applicationPage = unpublishedPageItr.next(); + NewPage newPage = pageNameMap.get(applicationPage.getId()); + if (newPage == null) { + if (appendToApp) { + // Don't remove the page reference if doing the partial import and appending + // to the existing application + continue; + } + log.debug( + "Unable to find the page during import for appId {}, with name {}", + applicationId, + applicationPage.getId()); + unpublishedPageItr.remove(); + } else { + applicationPage.setId(newPage.getId()); + applicationPage.setDefaultPageId( + newPage.getDefaultResources().getPageId()); + // Keep the existing page as the default one + if (appendToApp) { + applicationPage.setIsDefault(false); + } + } + } + + Iterator publishedPagesItr; + // Remove the newly added pages from merge app flow. Keep only the existing page from the old app + if (appendToApp) { + List existingPagesId = savedApp.getPublishedPages().stream() + .map(applicationPage -> applicationPage.getId()) + .collect(Collectors.toList()); + List publishedApplicationPages = publishedPages.stream() + .filter(applicationPage -> existingPagesId.contains(applicationPage.getId())) + .collect(Collectors.toList()); + applicationPages.replace(VIEW, publishedApplicationPages); + publishedPagesItr = publishedApplicationPages.iterator(); + } else { + publishedPagesItr = publishedPages.iterator(); + } + while (publishedPagesItr.hasNext()) { + ApplicationPage applicationPage = publishedPagesItr.next(); + NewPage newPage = pageNameMap.get(applicationPage.getId()); + if (newPage == null) { + log.debug( + "Unable to find the page during import for appId {}, with name {}", + applicationId, + applicationPage.getId()); + if (!appendToApp) { + publishedPagesItr.remove(); + } + } else { + applicationPage.setId(newPage.getId()); + applicationPage.setDefaultPageId( + newPage.getDefaultResources().getPageId()); + if (appendToApp) { + applicationPage.setIsDefault(false); + } + } + } + + return applicationPages; + }); + + if (!StringUtils.isEmpty(applicationId) && !appendToApp) { + applicationPagesMono = applicationPagesMono + .zipWith(existingPagesMono) + .flatMap(objects -> { + Map> applicationPages = objects.getT1(); + List existingPagesList = objects.getT2(); + Set validPageIds = applicationPages.get(EDIT).stream() + .map(ApplicationPage::getId) + .collect(Collectors.toSet()); + + validPageIds.addAll(applicationPages.get(VIEW).stream() + .map(ApplicationPage::getId) + .collect(Collectors.toSet())); + + Set invalidPageIds = new HashSet<>(); + for (NewPage newPage : existingPagesList) { + if (!validPageIds.contains(newPage.getId())) { + invalidPageIds.add(newPage.getId()); + } + } + + // Delete the pages which were removed during git merge operation + // This does not apply to the traditional import via file approach + return Flux.fromIterable(invalidPageIds) + .flatMap(applicationPageService::deleteWithoutPermissionUnpublishedPage) + .flatMap(page -> newPageService + .archiveWithoutPermissionById(page.getId()) + .onErrorResume(e -> { + log.debug( + "Unable to archive page {} with error {}", + page.getId(), + e.getMessage()); + return Mono.empty(); + })) + .then() + .thenReturn(applicationPages); + }); + } + return applicationMono.zipWith(applicationPagesMono).map(objects -> { + Application application = objects.getT1(); + Map> applicationPages = objects.getT2(); + application.setPages(applicationPages.get(EDIT)); + application.setPublishedPages(applicationPages.get(VIEW)); + return application; + }); + } + + private Mono> getCustomJslibImportMono(List customJSLibs) { + if (customJSLibs == null) { + customJSLibs = new ArrayList<>(); + } + + return Flux.fromIterable(customJSLibs) + .flatMap(customJSLib -> { + customJSLib.setId(null); + customJSLib.setCreatedAt(null); + customJSLib.setUpdatedAt(null); + return customJSLibService.persistCustomJSLibMetaDataIfDoesNotExistAndGetDTO(customJSLib, false); + }) + .collectList() + .elapsed() + .map(objects -> { + log.debug("time to import custom jslibs: {}", objects.getT1()); + return objects.getT2(); + }) + .onErrorResume(e -> { + log.error("Error importing custom jslibs", e); + return Mono.error(e); + }); + } + + private Mono> getExistingDatasourceMono(String applicationId, Flux datasourceFlux) { + Mono> existingDatasourceMono; + // Check if the request is to hydrate the application to DB for particular branch + // Application id will be present for GIT sync + if (!StringUtils.isEmpty(applicationId)) { + // No need to hydrate the datasource as we expect user will configure the datasource + existingDatasourceMono = datasourceFlux.collectList(); + } else { + existingDatasourceMono = Mono.just(new ArrayList<>()); + } + return existingDatasourceMono; + } + + private Mono> getImportActionsAndActionCollectionsMono( + ApplicationJson importedDoc, + Mono applicationMono, + Mono, Map>> importedNewPagesMono, + Mono> pluginMapMno, + Mono> pageNameMapMono, + boolean appendToApp, + String branchName, + String applicationId, + Workspace workspace, + ImportApplicationPermissionProvider permissionProvider) { + Mono importActionsMono = getImportActionsMono( + importedDoc, + importedNewPagesMono, + applicationMono, + pageNameMapMono, + pluginMapMno, + appendToApp, + workspace, + applicationId, + branchName, + permissionProvider) + .cache(); + + List importedActionCollectionList = + CollectionUtils.isEmpty(importedDoc.getActionCollectionList()) + ? new ArrayList<>() + : importedDoc.getActionCollectionList(); + + Mono importActionCollectionMono = createImportActionCollectionMono( + importedActionCollectionList, + applicationMono, + importedNewPagesMono, + importActionsMono, + pluginMapMno, + pageNameMapMono, + permissionProvider, + appendToApp, + branchName); + + Mono> updateCollectionIdToActionsMono = + setCollectionIdToActionsMono( + importActionsMono, importActionCollectionMono, applicationMono, applicationId, appendToApp); + + Mono> updateActionsInPages = + setActionIdInPages(updateCollectionIdToActionsMono, importedNewPagesMono, branchName); + return updateActionsInPages; + } + + /** + * This function will take the application reference object to hydrate the application in mongoDB + * + * @param workspaceId workspace to which application is going to be stored + * @param applicationJson application resource which contains necessary information to import the application + * @param applicationId application which needs to be saved with the updated resources + * @param branchName name of the branch of application with applicationId + * @param appendToApp whether applicationJson will be appended to the existing app or not + * @return Updated application + */ + private Mono importApplicationInWorkspace( + String workspaceId, + ApplicationJson applicationJson, + String applicationId, + String branchName, + boolean appendToApp, + ImportApplicationPermissionProvider permissionProvider) { + /* + 1. Migrate resource to latest schema + 2. Fetch workspace by id + 3. Extract datasources and update plugin information + 4. Create new datasource if same datasource is not present + 5. Extract and save application + 6. Extract and save pages in the application + 7. Extract and save actions in the application + */ + ApplicationJson importedDoc = JsonSchemaMigration.migrateApplicationToLatestSchema(applicationJson); + + // check for validation error and raise exception if error found + String errorField = validateApplicationJson(importedDoc); + if (!errorField.isEmpty()) { + log.error("Error in importing application. Field {} is missing", errorField); + return Mono.error(new AppsmithException(AppsmithError.NO_RESOURCE_FOUND, errorField, INVALID_JSON_FILE)); + } + + Application importedApplication = importedDoc.getExportedApplication(); + List importedNewPageList = importedDoc.getPageList(); + + Mono workspaceMono = workspaceService + .findById(workspaceId, permissionProvider.getRequiredPermissionOnTargetWorkspace()) + .switchIfEmpty(Mono.defer(() -> { + log.error( + "No workspace found with id: {} and permission: {}", + workspaceId, + permissionProvider.getRequiredPermissionOnTargetWorkspace()); + return Mono.error(new AppsmithException( + AppsmithError.ACL_NO_RESOURCE_FOUND, FieldName.WORKSPACE, workspaceId)); + })); + + Mono currUserMono = sessionUserService.getCurrentUser().cache(); + Mono> installedJsLibsMono = + getCustomJslibImportMono(importedDoc.getCustomJSLibList()); + Mono> pluginMapMno = getPluginMapMono().cache(); + + // Start the stopwatch to log the execution time + Stopwatch stopwatch = new Stopwatch(AnalyticsEvents.IMPORT.getEventName()); + Mono importedApplicationMono = workspaceMono + .flatMap(workspace -> { + Mono applicationMono = getImportApplicationMono( + importedApplication, + applicationId, + appendToApp, + workspaceId, + permissionProvider, + currUserMono, + installedJsLibsMono) + .flatMap(application -> importThemes(application, importedDoc, appendToApp)) + .cache(); + + // Import and save pages, also update the pages related fields in saved application + assert importedNewPageList != null : "Unable to find pages in the imported application"; + + // For git-sync this will not be empty + Mono> existingPagesMono = applicationMono + .flatMap(application -> newPageService + .findNewPagesByApplicationId(application.getId(), Optional.empty()) + .collectList()) + .cache(); + + Mono, Map>> importedNewPagesMono = getImportNewPagesMono( + importedNewPageList, + existingPagesMono, + applicationMono, + appendToApp, + branchName, + permissionProvider) + .cache(); + + Mono> pageNameMapMono = + getPageNameMapMono(importedNewPagesMono).cache(); + + applicationMono = savePagesToApplicationMono( + importedApplication, + pageNameMapMono, + applicationMono, + appendToApp, + applicationId, + existingPagesMono, + importedNewPagesMono) + .cache(); + + // this will trigger all the dependent monos + Mono> importActionAndActionCollections = getImportActionsAndActionCollectionsMono( + importedDoc, + applicationMono, + importedNewPagesMono, + pluginMapMno, + pageNameMapMono, + appendToApp, + branchName, + applicationId, + workspace, + permissionProvider); + + return importActionAndActionCollections + .then(applicationMono) + .flatMap(application -> { + log.info("Imported application with id {}", application.getId()); + // Need to update the application object with updated pages and publishedPages + Application updateApplication = new Application(); + updateApplication.setPages(application.getPages()); + updateApplication.setPublishedPages(application.getPublishedPages()); + return applicationService.update(application.getId(), updateApplication); + }) + .onErrorResume(throwable -> { + String errorMessage = ImportExportUtils.getErrorMessage(throwable); + log.error("Error importing application. Error: {}", errorMessage, throwable); + return Mono.error(new AppsmithException( + AppsmithError.GENERIC_JSON_IMPORT_ERROR, workspaceId, errorMessage)); + }); }) .as(transactionalOperator::transactional); + final Mono resultMono = importedApplicationMono + .flatMap(application -> + sendImportExportApplicationAnalyticsEvent(application.getId(), AnalyticsEvents.IMPORT)) + .zipWith(currUserMono) + .flatMap(tuple -> { + Application application = tuple.getT1(); + stopwatch.stopTimer(); + stopwatch.stopAndLogTimeInMillis(); + final Map data = Map.of( + FieldName.APPLICATION_ID, + application.getId(), + FieldName.WORKSPACE_ID, + application.getWorkspaceId(), + "pageCount", + applicationJson.getPageList().size(), + "actionCount", + applicationJson.getActionList().size(), + "JSObjectCount", + applicationJson.getActionCollectionList().size(), + FieldName.FLOW_NAME, + stopwatch.getFlow(), + "executionTime", + stopwatch.getExecutionTime()); + return analyticsService + .sendEvent( + AnalyticsEvents.UNIT_EXECUTION_TIME.getEventName(), + tuple.getT2().getUsername(), + data) + .thenReturn(application); + }); + // Import Application is currently a slow API because it needs to import and create application, pages, actions // and action collection. This process may take time and the client may cancel the request. This leads to the // flow @@ -1618,8 +1945,7 @@ public class ImportExportApplicationServiceCEImpl implements ImportExportApplica // To achieve this, we use a synchronous sink which does not take subscription cancellations into account. This // means that even if the subscriber has cancelled its subscription, the create method still generates its // event. - return Mono.create( - sink -> importedApplicationMono.subscribe(sink::success, sink::error, null, sink.currentContext())); + return Mono.create(sink -> resultMono.subscribe(sink::success, sink::error, null, sink.currentContext())); } private void renamePageInActions(List newActionList, String oldPageName, String newPageName) { @@ -1702,80 +2028,93 @@ public class ImportExportApplicationServiceCEImpl implements ImportExportApplica } }); - return existingPages.flatMapMany(existingSavedPages -> { - Map savedPagesGitIdToPageMap = new HashMap<>(); + return existingPages + .flatMapMany(existingSavedPages -> { + Map savedPagesGitIdToPageMap = new HashMap<>(); - existingSavedPages.stream() - .filter(newPage -> !StringUtils.isEmpty(newPage.getGitSyncId())) - .forEach(newPage -> savedPagesGitIdToPageMap.put(newPage.getGitSyncId(), newPage)); + existingSavedPages.stream() + .filter(newPage -> !StringUtils.isEmpty(newPage.getGitSyncId())) + .forEach(newPage -> savedPagesGitIdToPageMap.put(newPage.getGitSyncId(), newPage)); - return Flux.fromIterable(pages).flatMap(newPage -> { - - // Check if the page has gitSyncId and if it's already in DB - if (newPage.getGitSyncId() != null && savedPagesGitIdToPageMap.containsKey(newPage.getGitSyncId())) { - // Since the resource is already present in DB, just update resource - NewPage existingPage = savedPagesGitIdToPageMap.get(newPage.getGitSyncId()); - if (!permissionProvider.hasEditPermission(existingPage)) { - log.error("User does not have permission to edit page with id: {}", existingPage.getId()); - return Mono.error(new AppsmithException( - AppsmithError.ACL_NO_RESOURCE_FOUND, FieldName.PAGE, existingPage.getId())); - } - Set existingPagePolicy = existingPage.getPolicies(); - copyNestedNonNullProperties(newPage, existingPage); - // Update branchName - existingPage.getDefaultResources().setBranchName(branchName); - // Recover the deleted state present in DB from imported page - existingPage - .getUnpublishedPage() - .setDeletedAt(newPage.getUnpublishedPage().getDeletedAt()); - existingPage.setDeletedAt(newPage.getDeletedAt()); - existingPage.setDeleted(newPage.getDeleted()); - existingPage.setPolicies(existingPagePolicy); - return newPageService.save(existingPage); - } else { - // check if user has permission to add new page to the application - if (!permissionProvider.canCreatePage(application)) { - log.error( - "User does not have permission to create page in application with id: {}", - application.getId()); - return Mono.error(new AppsmithException( - AppsmithError.ACL_NO_RESOURCE_FOUND, FieldName.APPLICATION, application.getId())); - } - if (application.getGitApplicationMetadata() != null) { - final String defaultApplicationId = - application.getGitApplicationMetadata().getDefaultApplicationId(); - return newPageService - .findByGitSyncIdAndDefaultApplicationId( - defaultApplicationId, newPage.getGitSyncId(), Optional.empty()) - .switchIfEmpty(Mono.defer(() -> { - // This is the first page we are saving with given gitSyncId in this instance - DefaultResources defaultResources = new DefaultResources(); - defaultResources.setApplicationId(defaultApplicationId); - defaultResources.setBranchName(branchName); - newPage.setDefaultResources(defaultResources); - return saveNewPageAndUpdateDefaultResources(newPage, branchName); - })) - .flatMap(branchedPage -> { - DefaultResources defaultResources = branchedPage.getDefaultResources(); - // Create new page but keep defaultApplicationId and defaultPageId same for both the - // pages - defaultResources.setBranchName(branchName); - newPage.setDefaultResources(defaultResources); - newPage.getUnpublishedPage() - .setDeletedAt(branchedPage - .getUnpublishedPage() - .getDeletedAt()); - newPage.setDeletedAt(branchedPage.getDeletedAt()); - newPage.setDeleted(branchedPage.getDeleted()); - // Set policies from existing branch object - newPage.setPolicies(branchedPage.getPolicies()); - return newPageService.save(newPage); - }); - } - return saveNewPageAndUpdateDefaultResources(newPage, branchName); - } - }); - }); + return Flux.fromIterable(pages).flatMap(newPage -> { + log.debug( + "Importing page: {}", + newPage.getUnpublishedPage().getName()); + // Check if the page has gitSyncId and if it's already in DB + if (newPage.getGitSyncId() != null + && savedPagesGitIdToPageMap.containsKey(newPage.getGitSyncId())) { + // Since the resource is already present in DB, just update resource + NewPage existingPage = savedPagesGitIdToPageMap.get(newPage.getGitSyncId()); + if (!permissionProvider.hasEditPermission(existingPage)) { + log.error( + "User does not have permission to edit page with id: {}", existingPage.getId()); + return Mono.error(new AppsmithException( + AppsmithError.ACL_NO_RESOURCE_FOUND, FieldName.PAGE, existingPage.getId())); + } + Set existingPagePolicy = existingPage.getPolicies(); + copyNestedNonNullProperties(newPage, existingPage); + // Update branchName + existingPage.getDefaultResources().setBranchName(branchName); + // Recover the deleted state present in DB from imported page + existingPage + .getUnpublishedPage() + .setDeletedAt(newPage.getUnpublishedPage().getDeletedAt()); + existingPage.setDeletedAt(newPage.getDeletedAt()); + existingPage.setDeleted(newPage.getDeleted()); + existingPage.setPolicies(existingPagePolicy); + return newPageService.save(existingPage); + } else { + // check if user has permission to add new page to the application + if (!permissionProvider.canCreatePage(application)) { + log.error( + "User does not have permission to create page in application with id: {}", + application.getId()); + return Mono.error(new AppsmithException( + AppsmithError.ACL_NO_RESOURCE_FOUND, + FieldName.APPLICATION, + application.getId())); + } + if (application.getGitApplicationMetadata() != null) { + final String defaultApplicationId = + application.getGitApplicationMetadata().getDefaultApplicationId(); + return newPageService + .findByGitSyncIdAndDefaultApplicationId( + defaultApplicationId, newPage.getGitSyncId(), Optional.empty()) + .switchIfEmpty(Mono.defer(() -> { + // This is the first page we are saving with given gitSyncId in this + // instance + DefaultResources defaultResources = new DefaultResources(); + defaultResources.setApplicationId(defaultApplicationId); + defaultResources.setBranchName(branchName); + newPage.setDefaultResources(defaultResources); + return saveNewPageAndUpdateDefaultResources(newPage, branchName); + })) + .flatMap(branchedPage -> { + DefaultResources defaultResources = branchedPage.getDefaultResources(); + // Create new page but keep defaultApplicationId and defaultPageId same for + // both the + // pages + defaultResources.setBranchName(branchName); + newPage.setDefaultResources(defaultResources); + newPage.getUnpublishedPage() + .setDeletedAt(branchedPage + .getUnpublishedPage() + .getDeletedAt()); + newPage.setDeletedAt(branchedPage.getDeletedAt()); + newPage.setDeleted(branchedPage.getDeleted()); + // Set policies from existing branch object + newPage.setPolicies(branchedPage.getPolicies()); + return newPageService.save(newPage); + }); + } + return saveNewPageAndUpdateDefaultResources(newPage, branchName); + } + }); + }) + .onErrorResume(error -> { + log.error("Error importing page", error); + return Mono.error(error); + }); } private Mono saveNewPageAndUpdateDefaultResources(NewPage newPage, String branchName) { @@ -1845,74 +2184,81 @@ public class ImportExportApplicationServiceCEImpl implements ImportExportApplica Map> unpublishedActionIdToCollectionIdsMap, Map> publishedActionIdToCollectionIdsMap) { - return Mono.just(newPage).flatMap(page -> { - return newActionService - .findAllById(getLayoutOnLoadActionsForPage( - page, - actionIdMap, - unpublishedActionIdToCollectionIdsMap, - publishedActionIdToCollectionIdsMap)) - .map(newAction -> { - final String defaultActionId = - newAction.getDefaultResources().getActionId(); - if (page.getUnpublishedPage().getLayouts() != null) { - final String defaultCollectionId = newAction - .getUnpublishedAction() - .getDefaultResources() - .getCollectionId(); - page.getUnpublishedPage().getLayouts().forEach(layout -> { - if (layout.getLayoutOnLoadActions() != null) { - layout.getLayoutOnLoadActions().forEach(onLoadAction -> onLoadAction.stream() - .filter(actionDTO -> - StringUtils.equals(actionDTO.getId(), newAction.getId())) - .forEach(actionDTO -> { - actionDTO.setDefaultActionId(defaultActionId); - actionDTO.setDefaultCollectionId(defaultCollectionId); - })); + return Mono.just(newPage) + .flatMap(page -> { + return newActionService + .findAllById(getLayoutOnLoadActionsForPage( + page, + actionIdMap, + unpublishedActionIdToCollectionIdsMap, + publishedActionIdToCollectionIdsMap)) + .map(newAction -> { + final String defaultActionId = + newAction.getDefaultResources().getActionId(); + if (page.getUnpublishedPage().getLayouts() != null) { + final String defaultCollectionId = newAction + .getUnpublishedAction() + .getDefaultResources() + .getCollectionId(); + page.getUnpublishedPage().getLayouts().forEach(layout -> { + if (layout.getLayoutOnLoadActions() != null) { + layout.getLayoutOnLoadActions() + .forEach(onLoadAction -> onLoadAction.stream() + .filter(actionDTO -> StringUtils.equals( + actionDTO.getId(), newAction.getId())) + .forEach(actionDTO -> { + actionDTO.setDefaultActionId(defaultActionId); + actionDTO.setDefaultCollectionId(defaultCollectionId); + })); + } + }); } - }); - } - if (page.getPublishedPage() != null - && page.getPublishedPage().getLayouts() != null) { - page.getPublishedPage().getLayouts().forEach(layout -> { - if (layout.getLayoutOnLoadActions() != null) { - layout.getLayoutOnLoadActions().forEach(onLoadAction -> onLoadAction.stream() - .filter(actionDTO -> - StringUtils.equals(actionDTO.getId(), newAction.getId())) - .forEach(actionDTO -> { - actionDTO.setDefaultActionId(defaultActionId); - if (newAction.getPublishedAction() != null - && newAction - .getPublishedAction() - .getDefaultResources() - != null) { - actionDTO.setDefaultCollectionId(newAction - .getPublishedAction() - .getDefaultResources() - .getCollectionId()); - } - })); + if (page.getPublishedPage() != null + && page.getPublishedPage().getLayouts() != null) { + page.getPublishedPage().getLayouts().forEach(layout -> { + if (layout.getLayoutOnLoadActions() != null) { + layout.getLayoutOnLoadActions() + .forEach(onLoadAction -> onLoadAction.stream() + .filter(actionDTO -> StringUtils.equals( + actionDTO.getId(), newAction.getId())) + .forEach(actionDTO -> { + actionDTO.setDefaultActionId(defaultActionId); + if (newAction.getPublishedAction() != null + && newAction + .getPublishedAction() + .getDefaultResources() + != null) { + actionDTO.setDefaultCollectionId(newAction + .getPublishedAction() + .getDefaultResources() + .getCollectionId()); + } + })); + } + }); } - }); - } - return newAction; - }) - .collectList() - .thenReturn(page); - }); + return newAction; + }) + .collectList() + .thenReturn(page); + }) + .onErrorResume(error -> { + log.error("Error while updating action collection id in page layout", error); + return Mono.error(error); + }); } /** * This will check if the datasource is already present in the workspace and create a new one if unable to find one * - * @param existingDatasourceFlux already present datasource in the workspace + * @param existingDatasources already present datasource in the workspace * @param datasourceStorage which will be checked against existing datasources * @param workspace workspace where duplicate datasource should be checked * @return already present or brand new datasource depending upon the equality check */ private Mono createUniqueDatasourceIfNotPresent( - Flux existingDatasourceFlux, + Flux existingDatasources, DatasourceStorage datasourceStorage, Workspace workspace, String environmentId, @@ -1929,7 +2275,7 @@ public class ImportExportApplicationServiceCEImpl implements ImportExportApplica datasourceConfig.getAuthentication().setAuthenticationType(null); } - return existingDatasourceFlux + return existingDatasources // For git import exclude datasource configuration .filter(ds -> ds.getName().equals(datasourceStorage.getName()) && datasourceStorage.getPluginId().equals(ds.getPluginId())) @@ -1966,7 +2312,11 @@ public class ImportExportApplicationServiceCEImpl implements ImportExportApplica .switchIfEmpty(Mono.just( datasourceService.createDatasourceFromDatasourceStorage(datasourceStorage))) .flatMap(datasourceService::createWithoutPermissions); - })); + })) + .onErrorResume(throwable -> { + log.error("failed to import datasource", throwable); + return Mono.error(throwable); + }); } /** @@ -2202,36 +2552,33 @@ public class ImportExportApplicationServiceCEImpl implements ImportExportApplica }); } - private Mono> updateNewPagesBeforeMerge( - Mono> existingPagesMono, List newPagesList) { - return existingPagesMono.map(newPages -> { - Map newToOldToPageNameMap = new HashMap<>(); // maps new names with old names + private Map updateNewPagesBeforeMerge(List existingPages, List newPagesList) { + Map newToOldToPageNameMap = new HashMap<>(); // maps new names with old names - // get a list of unpublished page names that already exists - List unpublishedPageNames = newPages.stream() - .filter(newPage -> newPage.getUnpublishedPage() != null) - .map(newPage -> newPage.getUnpublishedPage().getName()) - .collect(Collectors.toList()); + // get a list of unpublished page names that already exists + List unpublishedPageNames = existingPages.stream() + .filter(newPage -> newPage.getUnpublishedPage() != null) + .map(newPage -> newPage.getUnpublishedPage().getName()) + .collect(Collectors.toList()); - // modify each new page - for (NewPage newPage : newPagesList) { - newPage.setPublishedPage(null); // we'll not merge published pages so removing this + // modify each new page + for (NewPage newPage : newPagesList) { + newPage.setPublishedPage(null); // we'll not merge published pages so removing this - // let's check if page name conflicts, rename in that case - String oldPageName = newPage.getUnpublishedPage().getName(), - newPageName = newPage.getUnpublishedPage().getName(); + // let's check if page name conflicts, rename in that case + String oldPageName = newPage.getUnpublishedPage().getName(), + newPageName = newPage.getUnpublishedPage().getName(); - int i = 1; - while (unpublishedPageNames.contains(newPageName)) { - i++; - newPageName = oldPageName + i; - } - newPage.getUnpublishedPage().setName(newPageName); // set new name. may be same as before or not - newPage.getUnpublishedPage().setSlug(TextUtils.makeSlug(newPageName)); // set the slug also - newToOldToPageNameMap.put(newPageName, oldPageName); // map: new name -> old name + int i = 1; + while (unpublishedPageNames.contains(newPageName)) { + i++; + newPageName = oldPageName + i; } - return newToOldToPageNameMap; - }); + newPage.getUnpublishedPage().setName(newPageName); // set new name. may be same as before or not + newPage.getUnpublishedPage().setSlug(TextUtils.makeSlug(newPageName)); // set the slug also + newToOldToPageNameMap.put(newPageName, oldPageName); // map: new name -> old name + } + return newToOldToPageNameMap; } /** diff --git a/app/server/appsmith-server/src/test/java/com/appsmith/server/services/ce/ApplicationSnapshotServiceUnitTest.java b/app/server/appsmith-server/src/test/java/com/appsmith/server/services/ce/ApplicationSnapshotServiceUnitTest.java index 95d759c166..c4024e6cf0 100644 --- a/app/server/appsmith-server/src/test/java/com/appsmith/server/services/ce/ApplicationSnapshotServiceUnitTest.java +++ b/app/server/appsmith-server/src/test/java/com/appsmith/server/services/ce/ApplicationSnapshotServiceUnitTest.java @@ -289,4 +289,33 @@ public class ApplicationSnapshotServiceUnitTest { }) .verifyComplete(); } + + @Test + public void test() { + Mono mono = Mono.just(1) + .map(s -> { + System.out.println("s at line 296: " + s); + if (s == 1) { + throw new RuntimeException("equal to 1"); + } + return s; + }) + .onErrorResume(e -> { + System.out.println("error at integer mono on Error resume"); + return Mono.error(e); + }); + + Mono mono1 = Mono.just("1"); + + Mono finalMono = mono.then(mono1) + .map(s -> { + System.out.println("s at line 300: " + s); + return s; + }) + .onErrorResume(e -> { + System.out.println("error at final mono on error resume "); + return Mono.error(e); + }); + finalMono.subscribe(); + } }