From 38e3d45fda1e09bcf815a3e2d54788a22c687b6a Mon Sep 17 00:00:00 2001 From: Nidhi Date: Thu, 28 Nov 2024 14:20:50 +0530 Subject: [PATCH] chore: json to map conversion with test template (#37788) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Description > [!TIP] > _Add a TL;DR when the description is longer than 500 words or extremely technical (helps the content, marketing, and DevRel team)._ > > _Please also include relevant motivation and context. List any dependencies that are required for this change. Add links to Notion, Figma or any other documents that might be relevant to the PR._ Fixes #`Issue Number` _or_ Fixes `Issue URL` > [!WARNING] > _If no issue exists, please create an issue first, and check with the maintainers if the issue is valid._ ## Automation /ok-to-test tags="@tag.Git" ### :mag: Cypress test results > [!TIP] > 🟢 🟢 🟢 All cypress tests have passed! 🎉 🎉 🎉 > Workflow run: > Commit: bf8b3bf7c9048b489246989285e0e4ede3386ca7 > Cypress dashboard. > Tags: `@tag.Git` > Spec: >
Thu, 28 Nov 2024 08:38:34 UTC ## Communication Should the DevRel and Marketing teams inform users about this change? - [ ] Yes - [ ] No ## Summary by CodeRabbit ## Release Notes - **New Features** - Updated Git resource type to include `CONTEXT_CONFIG`. - Introduced methods to manage artifact-dependent resources and context lists. - Enhanced Git resource management with new actions and collections. - Added new properties and components in the exported application structure, including new data sources, pages, and actions. - **Bug Fixes** - Improved handling of metadata fields in application exports. - **Documentation** - Added unit tests for artifact JSON to Git resource map conversion. - **Chores** - Updated JSON structure for exported applications with new components and properties. --- .../external/git/models/GitResourceType.java | 2 +- .../git/ApplicationGitFileUtilsCEImpl.java | 82 +++++++- .../server/dtos/ce/ApplicationJsonCE.java | 5 + .../dtos/ce/ArtifactExchangeJsonCE.java | 6 + .../server/helpers/CommonGitFileUtils.java | 8 +- .../helpers/ce/ArtifactGitFileUtilsCE.java | 3 + .../helpers/ce/CommonGitFileUtilsCE.java | 189 ++++++++++++++++++ .../ExchangeJsonConversionTests.java | 94 +++++++++ .../contexts/ExchangeJsonContext.java | 61 ++++++ .../ExchangeJsonTestTemplateProvider.java | 7 + .../ExchangeJsonTestTemplateProviderCE.java | 114 +++++++++++ .../valid-application.json | 6 +- 12 files changed, 563 insertions(+), 14 deletions(-) create mode 100644 app/server/appsmith-server/src/test/java/com/appsmith/server/git/resourcemap/ExchangeJsonConversionTests.java create mode 100644 app/server/appsmith-server/src/test/java/com/appsmith/server/git/resourcemap/templates/contexts/ExchangeJsonContext.java create mode 100644 app/server/appsmith-server/src/test/java/com/appsmith/server/git/resourcemap/templates/providers/ExchangeJsonTestTemplateProvider.java create mode 100644 app/server/appsmith-server/src/test/java/com/appsmith/server/git/resourcemap/templates/providers/ce/ExchangeJsonTestTemplateProviderCE.java diff --git a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/git/models/GitResourceType.java b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/git/models/GitResourceType.java index 623a52abf1..60fe3a3c4e 100644 --- a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/git/models/GitResourceType.java +++ b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/git/models/GitResourceType.java @@ -6,7 +6,7 @@ public enum GitResourceType { ROOT_CONFIG, DATASOURCE_CONFIG, JSLIB_CONFIG, - PAGE_CONFIG, + CONTEXT_CONFIG, JSOBJECT_CONFIG, JSOBJECT_DATA, QUERY_CONFIG, diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/applications/git/ApplicationGitFileUtilsCEImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/applications/git/ApplicationGitFileUtilsCEImpl.java index 99333df7b9..dbe2f5dc2a 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/applications/git/ApplicationGitFileUtilsCEImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/applications/git/ApplicationGitFileUtilsCEImpl.java @@ -1,13 +1,18 @@ package com.appsmith.server.applications.git; import com.appsmith.external.git.FileInterface; +import com.appsmith.external.git.models.GitResourceIdentity; +import com.appsmith.external.git.models.GitResourceMap; +import com.appsmith.external.git.models.GitResourceType; import com.appsmith.external.helpers.AppsmithBeanUtils; import com.appsmith.external.models.ActionDTO; import com.appsmith.external.models.ApplicationGitReference; import com.appsmith.external.models.ArtifactGitReference; import com.appsmith.external.models.DatasourceStorage; import com.appsmith.external.models.PluginType; +import com.appsmith.git.constants.CommonConstants; import com.appsmith.git.files.FileUtilsImpl; +import com.appsmith.git.helpers.DSLTransformerHelper; import com.appsmith.server.actioncollections.base.ActionCollectionService; import com.appsmith.server.constants.FieldName; import com.appsmith.server.domains.ActionCollection; @@ -55,14 +60,16 @@ import java.util.stream.Collectors; import static com.appsmith.external.git.constants.GitConstants.NAME_SEPARATOR; import static com.appsmith.external.helpers.AppsmithBeanUtils.copyNestedNonNullProperties; import static com.appsmith.external.helpers.AppsmithBeanUtils.copyProperties; -import static com.appsmith.server.constants.ce.FieldNameCE.ACTION_COLLECTION_LIST; -import static com.appsmith.server.constants.ce.FieldNameCE.ACTION_LIST; -import static com.appsmith.server.constants.ce.FieldNameCE.CUSTOM_JS_LIB_LIST; -import static com.appsmith.server.constants.ce.FieldNameCE.DATASOURCE_LIST; -import static com.appsmith.server.constants.ce.FieldNameCE.DECRYPTED_FIELDS; -import static com.appsmith.server.constants.ce.FieldNameCE.EDIT_MODE_THEME; -import static com.appsmith.server.constants.ce.FieldNameCE.EXPORTED_APPLICATION; -import static com.appsmith.server.constants.ce.FieldNameCE.PAGE_LIST; +import static com.appsmith.server.constants.FieldName.ACTION_COLLECTION_LIST; +import static com.appsmith.server.constants.FieldName.ACTION_LIST; +import static com.appsmith.server.constants.FieldName.CHILDREN; +import static com.appsmith.server.constants.FieldName.CUSTOM_JS_LIB_LIST; +import static com.appsmith.server.constants.FieldName.DATASOURCE_LIST; +import static com.appsmith.server.constants.FieldName.DECRYPTED_FIELDS; +import static com.appsmith.server.constants.FieldName.EDIT_MODE_THEME; +import static com.appsmith.server.constants.FieldName.EXPORTED_APPLICATION; +import static com.appsmith.server.constants.FieldName.PAGE_LIST; +import static com.appsmith.server.constants.FieldName.WIDGET_ID; import static com.appsmith.server.helpers.ce.CommonGitFileUtilsCE.removeUnwantedFieldsFromBaseDomain; @Slf4j @@ -124,6 +131,63 @@ public class ApplicationGitFileUtilsCEImpl implements ArtifactGitFileUtilsCE resourceMap = gitResourceMap.getGitResourceMap(); + + // application + Application application = applicationJson.getExportedApplication(); + removeUnwantedFieldsFromApplication(application); + GitResourceIdentity applicationIdentity = new GitResourceIdentity( + GitResourceType.ROOT_CONFIG, CommonConstants.APPLICATION + CommonConstants.JSON_EXTENSION); + resourceMap.put(applicationIdentity, application); + + // metadata + Iterable keys = AppsmithBeanUtils.getAllFields(applicationJson.getClass()) + .map(Field::getName) + .filter(name -> !getBlockedMetadataFields().contains(name)) + .collect(Collectors.toList()); + + ApplicationJson applicationMetadata = new ApplicationJson(); + applicationJson.setModifiedResources(null); + copyProperties(applicationJson, applicationMetadata, keys); + GitResourceIdentity metadataIdentity = new GitResourceIdentity( + GitResourceType.ROOT_CONFIG, CommonConstants.METADATA + CommonConstants.JSON_EXTENSION); + resourceMap.put(metadataIdentity, applicationMetadata); + + // pages and widgets + applicationJson.getPageList().stream() + // As we are expecting the commit will happen only after the application is published, so we can safely + // assume if the unpublished version is deleted entity should not be committed to git + .filter(newPage -> newPage.getUnpublishedPage() != null + && newPage.getUnpublishedPage().getDeletedAt() == null) + .forEach(newPage -> { + removeUnwantedFieldsFromPage(newPage); + JSONObject dsl = + newPage.getUnpublishedPage().getLayouts().get(0).getDsl(); + // Get MainContainer widget data, remove the children and club with Canvas.json file + JSONObject mainContainer = new JSONObject(dsl); + mainContainer.remove(CHILDREN); + newPage.getUnpublishedPage().getLayouts().get(0).setDsl(mainContainer); + // pageName will be used for naming the json file + GitResourceIdentity pageIdentity = + new GitResourceIdentity(GitResourceType.CONTEXT_CONFIG, newPage.getGitSyncId()); + resourceMap.put(pageIdentity, newPage); + + Map result = + DSLTransformerHelper.flatten(new org.json.JSONObject(dsl.toString())); + result.forEach((key, jsonObject) -> { + String widgetId = newPage.getGitSyncId() + "-" + jsonObject.getString(WIDGET_ID); + GitResourceIdentity widgetIdentity = + new GitResourceIdentity(GitResourceType.WIDGET_CONFIG, widgetId); + resourceMap.put(widgetIdentity, jsonObject); + }); + }); + } + private void setApplicationInApplicationReference( ApplicationJson applicationJson, ApplicationGitReference applicationReference) { Application application = applicationJson.getExportedApplication(); @@ -492,7 +556,7 @@ public class ApplicationGitFileUtilsCEImpl implements ArtifactGitFileUtilsCE formData = gson.fromJson(actionBody.get(keyName), Map.class); newAction .getUnpublishedAction() diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/dtos/ce/ApplicationJsonCE.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/dtos/ce/ApplicationJsonCE.java index 0df6bfe73b..7f4ec76ee8 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/dtos/ce/ApplicationJsonCE.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/dtos/ce/ApplicationJsonCE.java @@ -133,4 +133,9 @@ public class ApplicationJsonCE implements ArtifactExchangeJsonCE { public Theme getUnpublishedTheme() { return this.getEditModeTheme(); } + + @Override + public List getContextList() { + return this.pageList; + } } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/dtos/ce/ArtifactExchangeJsonCE.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/dtos/ce/ArtifactExchangeJsonCE.java index 0dffcb7edb..47bf907e3d 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/dtos/ce/ArtifactExchangeJsonCE.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/dtos/ce/ArtifactExchangeJsonCE.java @@ -3,12 +3,15 @@ package com.appsmith.server.dtos.ce; import com.appsmith.external.dtos.ModifiedResources; import com.appsmith.external.models.DatasourceStorage; import com.appsmith.external.models.DecryptedSensitiveFields; +import com.appsmith.external.views.Views; import com.appsmith.server.constants.ArtifactType; import com.appsmith.server.domains.ActionCollection; import com.appsmith.server.domains.Artifact; +import com.appsmith.server.domains.Context; import com.appsmith.server.domains.CustomJSLib; import com.appsmith.server.domains.NewAction; import com.appsmith.server.domains.Theme; +import com.fasterxml.jackson.annotation.JsonView; import java.util.List; import java.util.Map; @@ -62,4 +65,7 @@ public interface ArtifactExchangeJsonCE { default Theme getPublishedTheme() { return null; } + + @JsonView(Views.Internal.class) + List getContextList(); } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/CommonGitFileUtils.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/CommonGitFileUtils.java index e8324f564b..ecb77dd871 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/CommonGitFileUtils.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/CommonGitFileUtils.java @@ -4,11 +4,12 @@ import com.appsmith.external.git.FileInterface; import com.appsmith.external.git.operations.FileOperations; import com.appsmith.external.models.ApplicationGitReference; import com.appsmith.git.files.FileUtilsImpl; +import com.appsmith.server.actioncollections.base.ActionCollectionService; import com.appsmith.server.helpers.ce.CommonGitFileUtilsCE; import com.appsmith.server.migrations.JsonSchemaVersions; +import com.appsmith.server.newactions.base.NewActionService; import com.appsmith.server.services.AnalyticsService; import com.appsmith.server.services.SessionUserService; -import com.google.gson.Gson; import lombok.extern.slf4j.Slf4j; import org.springframework.context.annotation.Import; import org.springframework.stereotype.Component; @@ -24,7 +25,8 @@ public class CommonGitFileUtils extends CommonGitFileUtilsCE { FileOperations fileOperations, AnalyticsService analyticsService, SessionUserService sessionUserService, - Gson gson, + NewActionService newActionService, + ActionCollectionService actionCollectionService, JsonSchemaVersions jsonSchemaVersions) { super( applicationGitFileUtils, @@ -32,6 +34,8 @@ public class CommonGitFileUtils extends CommonGitFileUtilsCE { fileOperations, analyticsService, sessionUserService, + newActionService, + actionCollectionService, jsonSchemaVersions); } } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/ce/ArtifactGitFileUtilsCE.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/ce/ArtifactGitFileUtilsCE.java index 1844397507..0af8ce9323 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/ce/ArtifactGitFileUtilsCE.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/ce/ArtifactGitFileUtilsCE.java @@ -1,5 +1,6 @@ package com.appsmith.server.helpers.ce; +import com.appsmith.external.git.models.GitResourceMap; import com.appsmith.external.models.ArtifactGitReference; import com.appsmith.server.dtos.ArtifactExchangeJson; import lombok.NonNull; @@ -12,6 +13,8 @@ public interface ArtifactGitFileUtilsCE { T createArtifactReferenceObject(); + void setArtifactDependentResources(ArtifactExchangeJson artifactExchangeJson, GitResourceMap gitResourceMap); + Mono reconstructArtifactExchangeJsonFromFilesInRepository( String workspaceId, String baseArtifactId, String repoName, String branchName); diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/ce/CommonGitFileUtilsCE.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/ce/CommonGitFileUtilsCE.java index 5cfd3c2a2b..e0955455d6 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/ce/CommonGitFileUtilsCE.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/ce/CommonGitFileUtilsCE.java @@ -2,17 +2,26 @@ package com.appsmith.server.helpers.ce; import com.appsmith.external.constants.AnalyticsEvents; import com.appsmith.external.git.FileInterface; +import com.appsmith.external.git.models.GitResourceIdentity; +import com.appsmith.external.git.models.GitResourceMap; +import com.appsmith.external.git.models.GitResourceType; import com.appsmith.external.git.operations.FileOperations; import com.appsmith.external.helpers.Stopwatch; import com.appsmith.external.models.ApplicationGitReference; import com.appsmith.external.models.ArtifactGitReference; import com.appsmith.external.models.BaseDomain; import com.appsmith.external.models.DatasourceStorage; +import com.appsmith.external.models.PluginType; import com.appsmith.git.constants.CommonConstants; import com.appsmith.git.files.FileUtilsImpl; +import com.appsmith.server.actioncollections.base.ActionCollectionService; import com.appsmith.server.constants.ArtifactType; import com.appsmith.server.constants.FieldName; +import com.appsmith.server.domains.ActionCollection; +import com.appsmith.server.domains.CustomJSLib; import com.appsmith.server.domains.GitArtifactMetadata; +import com.appsmith.server.domains.NewAction; +import com.appsmith.server.domains.Theme; import com.appsmith.server.dtos.ApplicationJson; import com.appsmith.server.dtos.ArtifactExchangeJson; import com.appsmith.server.dtos.PageDTO; @@ -20,6 +29,7 @@ import com.appsmith.server.exceptions.AppsmithError; import com.appsmith.server.exceptions.AppsmithException; import com.appsmith.server.helpers.ArtifactGitFileUtils; import com.appsmith.server.migrations.JsonSchemaVersions; +import com.appsmith.server.newactions.base.NewActionService; import com.appsmith.server.services.AnalyticsService; import com.appsmith.server.services.SessionUserService; import com.google.gson.Gson; @@ -40,13 +50,17 @@ import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; import java.util.HashMap; +import java.util.List; import java.util.Map; import static com.appsmith.external.git.constants.ce.GitConstantsCE.GitCommandConstantsCE.CHECKOUT_BRANCH; import static com.appsmith.external.git.constants.ce.GitConstantsCE.RECONSTRUCT_PAGE; import static com.appsmith.git.constants.CommonConstants.CLIENT_SCHEMA_VERSION; import static com.appsmith.git.constants.CommonConstants.FILE_FORMAT_VERSION; +import static com.appsmith.git.constants.CommonConstants.JSON_EXTENSION; import static com.appsmith.git.constants.CommonConstants.SERVER_SCHEMA_VERSION; +import static com.appsmith.git.constants.CommonConstants.THEME; +import static com.appsmith.git.files.FileUtilsCEImpl.getJsLibFileName; import static org.springframework.util.StringUtils.hasText; @Slf4j @@ -61,6 +75,9 @@ public class CommonGitFileUtilsCE { private final AnalyticsService analyticsService; private final SessionUserService sessionUserService; + private final NewActionService newActionService; + private final ActionCollectionService actionCollectionService; + // Number of seconds after lock file is stale @Value("${appsmith.index.lock.file.time}") public final int INDEX_LOCK_FILE_STALE_TIME = 300; @@ -177,6 +194,178 @@ public class CommonGitFileUtilsCE { return artifactGitReference; } + public GitResourceMap createGitResourceMap(ArtifactExchangeJson artifactExchangeJson) { + ArtifactGitFileUtils artifactGitFileUtils = + getArtifactBasedFileHelper(artifactExchangeJson.getArtifactJsonType()); + GitResourceMap gitResourceMap = new GitResourceMap(); + gitResourceMap.setModifiedResources(artifactExchangeJson.getModifiedResources()); + + setArtifactIndependentResources(artifactExchangeJson, gitResourceMap); + + artifactGitFileUtils.setArtifactDependentResources(artifactExchangeJson, gitResourceMap); + + return gitResourceMap; + } + + protected void setArtifactIndependentResources( + ArtifactExchangeJson artifactExchangeJson, GitResourceMap gitResourceMap) { + Map resourceMap = gitResourceMap.getGitResourceMap(); + + // datasources + List datasourceList = artifactExchangeJson.getDatasourceList(); + if (datasourceList != null) { + datasourceList.forEach(datasource -> { + removeUnwantedFieldsFromDatasource(datasource); + GitResourceIdentity identity = + new GitResourceIdentity(GitResourceType.DATASOURCE_CONFIG, datasource.getGitSyncId()); + resourceMap.put(identity, datasource); + }); + } + + // themes + Theme theme = artifactExchangeJson.getUnpublishedTheme(); + // Only proceed if the current artifact supports themes + if (theme != null) { + // Reset published mode theme since it is not required + artifactExchangeJson.setThemes(theme, null); + // Remove internal fields from the themes + removeUnwantedFieldsFromBaseDomain(theme); + GitResourceIdentity identity = new GitResourceIdentity(GitResourceType.ROOT_CONFIG, THEME + JSON_EXTENSION); + resourceMap.put(identity, theme); + } + + // custom js libs + List customJSLibList = artifactExchangeJson.getCustomJSLibList(); + if (customJSLibList != null) { + customJSLibList.forEach(jsLib -> { + removeUnwantedFieldsFromBaseDomain(jsLib); + String jsLibFileName = getJsLibFileName(jsLib.getUidString()); + GitResourceIdentity identity = new GitResourceIdentity(GitResourceType.JSLIB_CONFIG, jsLibFileName); + resourceMap.put(identity, jsLib); + }); + } + + // actions + setNewActionsInResourceMap(artifactExchangeJson, resourceMap); + + // action collections + setActionCollectionsInResourceMap(artifactExchangeJson, resourceMap); + } + + protected void setNewActionsInResourceMap( + ArtifactExchangeJson artifactExchangeJson, Map resourceMap) { + if (artifactExchangeJson.getActionList() == null) { + return; + } + artifactExchangeJson.getActionList().stream() + // As we are expecting the commit will happen only after the application is published, so we can safely + // assume if the unpublished version is deleted entity should not be committed to git + .filter(newAction -> newAction.getUnpublishedAction() != null + && newAction.getUnpublishedAction().getDeletedAt() == null) + .peek(newAction -> newActionService.generateActionByViewMode(newAction, false)) + .forEach(newAction -> { + removeUnwantedFieldFromAction(newAction); + String body = newAction.getUnpublishedAction().getActionConfiguration() != null + && newAction + .getUnpublishedAction() + .getActionConfiguration() + .getBody() + != null + ? newAction + .getUnpublishedAction() + .getActionConfiguration() + .getBody() + : ""; + + // This is a special case where we are handling REMOTE type plugins based actions such as Twilio + // The user configured values are stored in an attribute called formData which is a map unlike the + // body + if (PluginType.REMOTE.equals(newAction.getPluginType()) + && newAction.getUnpublishedAction().getActionConfiguration() != null + && newAction + .getUnpublishedAction() + .getActionConfiguration() + .getFormData() + != null) { + body = new Gson() + .toJson( + newAction + .getUnpublishedAction() + .getActionConfiguration() + .getFormData(), + Map.class); + newAction + .getUnpublishedAction() + .getActionConfiguration() + .setFormData(null); + } + // This is a special case where we are handling JS actions as we don't want to commit the body of JS + // actions + if (PluginType.JS.equals(newAction.getPluginType())) { + if (newAction.getUnpublishedAction().getActionConfiguration() != null) { + newAction + .getUnpublishedAction() + .getActionConfiguration() + .setBody(null); + newAction.getUnpublishedAction().setJsonPathKeys(null); + } + } else { + // For the regular actions we save the body field to git repo + GitResourceIdentity actionDataIdentity = + new GitResourceIdentity(GitResourceType.QUERY_DATA, newAction.getGitSyncId()); + resourceMap.put(actionDataIdentity, body); + } + GitResourceIdentity actionConfigIdentity = + new GitResourceIdentity(GitResourceType.QUERY_CONFIG, newAction.getGitSyncId()); + resourceMap.put(actionConfigIdentity, newAction); + }); + } + + protected void setActionCollectionsInResourceMap( + ArtifactExchangeJson artifactExchangeJson, Map resourceMap) { + if (artifactExchangeJson.getActionCollectionList() == null) { + return; + } + artifactExchangeJson.getActionCollectionList().stream() + // As we are expecting the commit will happen only after the application is published, so we can safely + // assume if the unpublished version is deleted entity should not be committed to git + .filter(collection -> collection.getUnpublishedCollection() != null + && collection.getUnpublishedCollection().getDeletedAt() == null) + .peek(actionCollection -> + actionCollectionService.generateActionCollectionByViewMode(actionCollection, false)) + .forEach(actionCollection -> { + removeUnwantedFieldFromActionCollection(actionCollection); + String body = actionCollection.getUnpublishedCollection().getBody() != null + ? actionCollection.getUnpublishedCollection().getBody() + : ""; + actionCollection.getUnpublishedCollection().setBody(null); + + GitResourceIdentity collectionConfigIdentity = + new GitResourceIdentity(GitResourceType.JSOBJECT_CONFIG, actionCollection.getGitSyncId()); + resourceMap.put(collectionConfigIdentity, actionCollection); + + GitResourceIdentity collectionDataIdentity = + new GitResourceIdentity(GitResourceType.JSOBJECT_DATA, actionCollection.getGitSyncId()); + resourceMap.put(collectionDataIdentity, body); + }); + } + + private void removeUnwantedFieldFromAction(NewAction action) { + // As we are publishing the app and then committing to git we expect the published and unpublished ActionDTO + // will be same, so we only commit unpublished ActionDTO. + action.setPublishedAction(null); + action.getUnpublishedAction().sanitiseToExportDBObject(); + removeUnwantedFieldsFromBaseDomain(action); + } + + private void removeUnwantedFieldFromActionCollection(ActionCollection actionCollection) { + // As we are publishing the app and then committing to git we expect the published and unpublished + // ActionCollectionDTO will be same, so we only commit unpublished ActionCollectionDTO. + actionCollection.setPublishedCollection(null); + actionCollection.getUnpublishedCollection().sanitiseForExport(); + removeUnwantedFieldsFromBaseDomain(actionCollection); + } + private void setDatasourcesInArtifactReference( ArtifactExchangeJson artifactExchangeJson, ArtifactGitReference artifactGitReference) { Map resourceMap = new HashMap<>(); diff --git a/app/server/appsmith-server/src/test/java/com/appsmith/server/git/resourcemap/ExchangeJsonConversionTests.java b/app/server/appsmith-server/src/test/java/com/appsmith/server/git/resourcemap/ExchangeJsonConversionTests.java new file mode 100644 index 0000000000..c1df11a4be --- /dev/null +++ b/app/server/appsmith-server/src/test/java/com/appsmith/server/git/resourcemap/ExchangeJsonConversionTests.java @@ -0,0 +1,94 @@ +package com.appsmith.server.git.resourcemap; + +import com.appsmith.external.git.models.GitResourceIdentity; +import com.appsmith.external.git.models.GitResourceMap; +import com.appsmith.server.dtos.ArtifactExchangeJson; +import com.appsmith.server.git.resourcemap.templates.contexts.ExchangeJsonContext; +import com.appsmith.server.git.resourcemap.templates.providers.ExchangeJsonTestTemplateProvider; +import com.appsmith.server.helpers.CommonGitFileUtils; +import com.appsmith.server.migrations.JsonSchemaMigration; +import com.google.gson.Gson; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.TestTemplate; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.core.io.ClassPathResource; +import reactor.core.publisher.Mono; +import reactor.test.StepVerifier; +import reactor.util.function.Tuple2; + +import java.io.IOException; +import java.nio.charset.Charset; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +@SpringBootTest +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public class ExchangeJsonConversionTests { + + @Autowired + @RegisterExtension + public ExchangeJsonTestTemplateProvider templateProvider; + + @Autowired + Gson gson; + + @Autowired + JsonSchemaMigration jsonSchemaMigration; + + @Autowired + CommonGitFileUtils commonGitFileUtils; + + @TestTemplate + public void testConvertArtifactJsonToGitResourceMap_whenArtifactIsFullyPopulated_returnsCorrespondingResourceMap( + ExchangeJsonContext context) throws IOException { + Mono artifactJsonMono = + createArtifactJson(context).cache(); + + Mono> gitResourceMapAndArtifactJsonMono = + artifactJsonMono + .map(artifactJson -> commonGitFileUtils.createGitResourceMap(artifactJson)) + .zipWith(artifactJsonMono); + + StepVerifier.create(gitResourceMapAndArtifactJsonMono) + .assertNext(tuple2 -> { + GitResourceMap gitResourceMap = tuple2.getT1(); + ArtifactExchangeJson exchangeJson = tuple2.getT2(); + + assertThat(gitResourceMap).isNotNull(); + + if (exchangeJson.getModifiedResources() == null) { + assertThat(gitResourceMap.getModifiedResources()).isNull(); + } else { + assertThat(exchangeJson.getModifiedResources()) + .isEqualTo(gitResourceMap.getModifiedResources()); + } + Map resourceMap = gitResourceMap.getGitResourceMap(); + assertThat(resourceMap).isNotNull(); + + assertThat(resourceMap).hasSize(context.resourceMapKeyCount()); + + long count = templateProvider.assertResourceComparisons(exchangeJson, resourceMap); + + assertThat(count).isEqualTo(context.resourceMapKeyCount()); + }) + .verifyComplete(); + } + + private Mono createArtifactJson(ExchangeJsonContext context) throws IOException { + + String filePath = "test_assets/ImportExportServiceTest/" + context.getFileName(); + + ClassPathResource classPathResource = new ClassPathResource(filePath); + + String artifactJson = classPathResource.getContentAsString(Charset.defaultCharset()); + + Class exchangeJsonType = context.getArtifactExchangeJsonType(); + + ArtifactExchangeJson artifactExchangeJson = gson.fromJson(artifactJson, exchangeJsonType); + + return jsonSchemaMigration.migrateArtifactExchangeJsonToLatestSchema(artifactExchangeJson, null, null); + } +} diff --git a/app/server/appsmith-server/src/test/java/com/appsmith/server/git/resourcemap/templates/contexts/ExchangeJsonContext.java b/app/server/appsmith-server/src/test/java/com/appsmith/server/git/resourcemap/templates/contexts/ExchangeJsonContext.java new file mode 100644 index 0000000000..19b7fbe029 --- /dev/null +++ b/app/server/appsmith-server/src/test/java/com/appsmith/server/git/resourcemap/templates/contexts/ExchangeJsonContext.java @@ -0,0 +1,61 @@ +package com.appsmith.server.git.resourcemap.templates.contexts; + +import com.appsmith.server.dtos.ArtifactExchangeJson; +import org.junit.jupiter.api.extension.Extension; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.api.extension.ParameterContext; +import org.junit.jupiter.api.extension.ParameterResolutionException; +import org.junit.jupiter.api.extension.ParameterResolver; +import org.junit.jupiter.api.extension.TestTemplateInvocationContext; + +import java.util.List; + +public class ExchangeJsonContext implements TestTemplateInvocationContext, ParameterResolver { + + private final String fileName; + + private final Class artifactExchangeJsonType; + + private final int resourceMapKeyCount; + + public ExchangeJsonContext( + String fileName, Class artifactExchangeJsonType, int resourceMapKeyCount) { + this.fileName = fileName; + this.artifactExchangeJsonType = artifactExchangeJsonType; + this.resourceMapKeyCount = resourceMapKeyCount; + } + + @Override + public String getDisplayName(int invocationIndex) { + return fileName; + } + + @Override + public List getAdditionalExtensions() { + return List.of(this); + } + + public String getFileName() { + return fileName; + } + + public Class getArtifactExchangeJsonType() { + return artifactExchangeJsonType; + } + + @Override + public boolean supportsParameter(ParameterContext parameterContext, ExtensionContext extensionContext) + throws ParameterResolutionException { + return true; + } + + @Override + public Object resolveParameter(ParameterContext parameterContext, ExtensionContext extensionContext) + throws ParameterResolutionException { + return this; + } + + public int resourceMapKeyCount() { + return this.resourceMapKeyCount; + } +} diff --git a/app/server/appsmith-server/src/test/java/com/appsmith/server/git/resourcemap/templates/providers/ExchangeJsonTestTemplateProvider.java b/app/server/appsmith-server/src/test/java/com/appsmith/server/git/resourcemap/templates/providers/ExchangeJsonTestTemplateProvider.java new file mode 100644 index 0000000000..9543bfdacf --- /dev/null +++ b/app/server/appsmith-server/src/test/java/com/appsmith/server/git/resourcemap/templates/providers/ExchangeJsonTestTemplateProvider.java @@ -0,0 +1,7 @@ +package com.appsmith.server.git.resourcemap.templates.providers; + +import com.appsmith.server.git.resourcemap.templates.providers.ce.ExchangeJsonTestTemplateProviderCE; +import org.springframework.stereotype.Component; + +@Component +public class ExchangeJsonTestTemplateProvider extends ExchangeJsonTestTemplateProviderCE {} diff --git a/app/server/appsmith-server/src/test/java/com/appsmith/server/git/resourcemap/templates/providers/ce/ExchangeJsonTestTemplateProviderCE.java b/app/server/appsmith-server/src/test/java/com/appsmith/server/git/resourcemap/templates/providers/ce/ExchangeJsonTestTemplateProviderCE.java new file mode 100644 index 0000000000..0724d60170 --- /dev/null +++ b/app/server/appsmith-server/src/test/java/com/appsmith/server/git/resourcemap/templates/providers/ce/ExchangeJsonTestTemplateProviderCE.java @@ -0,0 +1,114 @@ +package com.appsmith.server.git.resourcemap.templates.providers.ce; + +import com.appsmith.external.git.models.GitResourceIdentity; +import com.appsmith.external.git.models.GitResourceType; +import com.appsmith.external.models.PluginType; +import com.appsmith.server.dtos.ApplicationJson; +import com.appsmith.server.dtos.ArtifactExchangeJson; +import com.appsmith.server.git.resourcemap.templates.contexts.ExchangeJsonContext; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.api.extension.TestTemplateInvocationContext; +import org.junit.jupiter.api.extension.TestTemplateInvocationContextProvider; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; + +public class ExchangeJsonTestTemplateProviderCE implements TestTemplateInvocationContextProvider { + + @Override + public boolean supportsTestTemplate(ExtensionContext extensionContext) { + return true; + } + + @Override + public Stream provideTestTemplateInvocationContexts( + ExtensionContext extensionContext) { + ExchangeJsonContext context = new ExchangeJsonContext("valid-application.json", ApplicationJson.class, 23); + return Stream.of(context); + } + + public long assertResourceComparisons( + ArtifactExchangeJson exchangeJson, Map resourceMap) { + List datasourceResources = getResourceListByType(resourceMap, GitResourceType.DATASOURCE_CONFIG); + long resourceMapDatasourceCount = datasourceResources.size(); + int jsonDatasourceCount = exchangeJson.getDatasourceList() != null + ? exchangeJson.getDatasourceList().size() + : 0; + assertThat(resourceMapDatasourceCount).isEqualTo(jsonDatasourceCount); + + List rootResources = getResourceListByType(resourceMap, GitResourceType.ROOT_CONFIG); + long resourceMapRootCount = rootResources.size(); + // artifact json, metadata and theme + assertThat(resourceMapRootCount).isEqualTo(3); + + List jsLibResources = getResourceListByType(resourceMap, GitResourceType.JSLIB_CONFIG); + long resourceMapJsLibCount = jsLibResources.size(); + int jsonJsLibCount = exchangeJson.getCustomJSLibList() != null + ? exchangeJson.getCustomJSLibList().size() + : 0; + assertThat(resourceMapJsLibCount).isEqualTo(jsonJsLibCount); + + List contextResources = getResourceListByType(resourceMap, GitResourceType.CONTEXT_CONFIG); + long resourceMapContextCount = contextResources.size(); + int jsonContextCount = exchangeJson.getContextList() != null + ? exchangeJson.getContextList().size() + : 0; + assertThat(resourceMapContextCount).isEqualTo(jsonContextCount); + + List jsObjectConfigResources = getResourceListByType(resourceMap, GitResourceType.JSOBJECT_CONFIG); + long resourceMapJsObjectConfigCount = jsObjectConfigResources.size(); + int jsonJsObjectCount = exchangeJson.getActionCollectionList() != null + ? exchangeJson.getActionCollectionList().size() + : 0; + assertThat(resourceMapJsObjectConfigCount).isEqualTo(jsonJsObjectCount); + + List jsObjectDataResources = getResourceListByType(resourceMap, GitResourceType.JSOBJECT_DATA); + long resourceMapJsObjectDataCount = jsObjectDataResources.size(); + assertThat(resourceMapJsObjectDataCount).isEqualTo(jsonJsObjectCount); + + List actionConfigResources = getResourceListByType(resourceMap, GitResourceType.QUERY_CONFIG); + long resourceMapActionConfigCount = actionConfigResources.size(); + int jsonActionCount = exchangeJson.getActionList() != null + ? exchangeJson.getActionList().size() + : 0; + assertThat(resourceMapActionConfigCount).isEqualTo(jsonActionCount); + + List actionDataResources = getResourceListByType(resourceMap, GitResourceType.QUERY_DATA); + long resourceMapActionDataCount = actionDataResources.size(); + long jsonActionDataCount = 0; + if (exchangeJson.getActionList() != null) { + jsonActionDataCount = exchangeJson.getActionList().stream() + .filter(action -> !PluginType.JS.equals(action.getPluginType())) + .count(); + } + assertThat(resourceMapActionDataCount).isEqualTo(jsonActionDataCount); + + List widgetResources = getResourceListByType(resourceMap, GitResourceType.WIDGET_CONFIG); + + return resourceMapDatasourceCount + + resourceMapRootCount + + resourceMapJsLibCount + + resourceMapContextCount + + resourceMapJsObjectConfigCount + + resourceMapJsObjectDataCount + + resourceMapActionConfigCount + + resourceMapActionDataCount + + widgetResources.size(); + } + + protected List getResourceListByType( + Map resourceMap, GitResourceType resourceType) { + return resourceMap.entrySet().stream() + .filter(entry -> { + GitResourceIdentity key = entry.getKey(); + + return resourceType.equals(key.getResourceType()); + }) + .map(Map.Entry::getValue) + .collect(Collectors.toList()); + } +} diff --git a/app/server/appsmith-server/src/test/resources/test_assets/ImportExportServiceTest/valid-application.json b/app/server/appsmith-server/src/test/resources/test_assets/ImportExportServiceTest/valid-application.json index 1e886c9940..1676cfa0bd 100644 --- a/app/server/appsmith-server/src/test/resources/test_assets/ImportExportServiceTest/valid-application.json +++ b/app/server/appsmith-server/src/test/resources/test_assets/ImportExportServiceTest/valid-application.json @@ -630,7 +630,8 @@ } ] }, - "new": false + "new": false, + "gitSyncId": "jso1" }, { "id": "Page1_JSObject2", @@ -650,7 +651,8 @@ } ] }, - "new": false + "new": false, + "gitSyncId": "jso2" } ], "decryptedFields": {