From a2c5caa81960b5e3a4749b6e7d2bc30c7a49a444 Mon Sep 17 00:00:00 2001 From: Nidhi Date: Wed, 4 Dec 2024 00:01:51 +0530 Subject: [PATCH] chore: Git resource map conversions (#37920) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Description > [!TIP] > _Add a TL;DR when the description is longer than 500 words or extremely technical (helps the content, marketing, and DevRel team)._ > > _Please also include relevant motivation and context. List any dependencies that are required for this change. Add links to Notion, Figma or any other documents that might be relevant to the PR._ Fixes #`Issue Number` _or_ Fixes `Issue URL` > [!WARNING] > _If no issue exists, please create an issue first, and check with the maintainers if the issue is valid._ ## Automation /ok-to-test tags="@tag.Git" ### :mag: Cypress test results > [!IMPORTANT] > 🟣 🟣 🟣 Your tests are running. > Tests running at: > Commit: c78857b0266e6cfd7be488205544de956e88647e > Workflow: `PR Automation test suite` > Tags: `@tag.Git` > Spec: `` >
Tue, 03 Dec 2024 17:36:22 UTC ## Communication Should the DevRel and Marketing teams inform users about this change? - [ ] Yes - [ ] No ## Summary by CodeRabbit ## Release Notes - **New Features** - Enhanced Git resource handling with the ability to save artifacts to Git repositories. - Introduced new methods for managing modified resources and artifact exchange JSON. - Improved tracking of updated entities during export operations, including custom JavaScript libraries and new pages. - New functionality for handling application artifacts and contexts in Git. - **Bug Fixes** - Refined error handling and control flow in auto-commit and migration processes. - **Documentation** - Updated comments and method signatures for clarity and improved understanding. - **Tests** - Added new test cases to validate the conversion processes and resource comparisons. - Enhanced existing tests to utilize new data structures and improve clarity. --- .../appsmith/git/files/FileUtilsCEImpl.java | 219 +++++++++++++- .../com/appsmith/git/files/FileUtilsImpl.java | 6 +- .../git/helpers/DSLTransformerHelper.java | 34 ++- .../git/helpers/FileUtilsImplTest.java | 4 +- .../external/dtos/ModifiedResources.java | 55 +--- .../external/dtos/ce/ModifiedResourcesCE.java | 86 ++++++ .../appsmith/external/git/FileInterface.java | 4 + .../git/models/GitResourceIdentity.java | 4 +- ...tionCollectionExportableServiceCEImpl.java | 18 +- .../git/ApplicationGitFileUtilsCEImpl.java | 142 ++++++++- .../git/ApplicationGitFileUtilsImpl.java | 4 +- .../server/dtos/ce/ApplicationJsonCE.java | 11 + .../dtos/ce/ArtifactExchangeJsonCE.java | 4 + .../ce/MappedExportableResourcesCE_DTO.java | 1 + .../AutoCommitEventHandlerCEImpl.java | 24 +- .../server/helpers/CommonGitFileUtils.java | 7 +- .../helpers/ce/ArtifactGitFileUtilsCE.java | 4 + .../helpers/ce/CommonGitFileUtilsCE.java | 276 ++++++++++++++---- .../CustomJSLibExportableServiceCEImpl.java | 6 + .../NewActionExportableServiceCEImpl.java | 17 +- .../NewPageExportableServiceCEImpl.java | 13 +- .../AutoCommitEventHandlerImplTest.java | 1 + .../ExchangeJsonConversionTests.java | 64 +++- .../ExchangeJsonTestTemplateProviderCE.java | 66 ++++- 24 files changed, 906 insertions(+), 164 deletions(-) create mode 100644 app/server/appsmith-interfaces/src/main/java/com/appsmith/external/dtos/ce/ModifiedResourcesCE.java diff --git a/app/server/appsmith-git/src/main/java/com/appsmith/git/files/FileUtilsCEImpl.java b/app/server/appsmith-git/src/main/java/com/appsmith/git/files/FileUtilsCEImpl.java index d919338cef..b7f3950221 100644 --- a/app/server/appsmith-git/src/main/java/com/appsmith/git/files/FileUtilsCEImpl.java +++ b/app/server/appsmith-git/src/main/java/com/appsmith/git/files/FileUtilsCEImpl.java @@ -6,6 +6,9 @@ import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginException import com.appsmith.external.git.FileInterface; import com.appsmith.external.git.GitExecutor; import com.appsmith.external.git.constants.GitSpan; +import com.appsmith.external.git.models.GitResourceIdentity; +import com.appsmith.external.git.models.GitResourceMap; +import com.appsmith.external.git.models.GitResourceType; import com.appsmith.external.git.operations.FileOperations; import com.appsmith.external.helpers.ObservationHelper; import com.appsmith.external.helpers.Stopwatch; @@ -14,10 +17,12 @@ import com.appsmith.external.models.ArtifactGitReference; import com.appsmith.git.configurations.GitServiceConfig; import com.appsmith.git.constants.CommonConstants; import com.appsmith.git.helpers.DSLTransformerHelper; +import com.fasterxml.jackson.databind.ObjectMapper; import io.micrometer.tracing.Span; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; +import org.apache.commons.io.FilenameUtils; import org.apache.commons.io.IOUtils; import org.eclipse.jgit.api.errors.GitAPIException; import org.json.JSONObject; @@ -28,6 +33,8 @@ import org.springframework.util.StringUtils; import reactor.core.publisher.Mono; import reactor.core.scheduler.Scheduler; import reactor.core.scheduler.Schedulers; +import reactor.util.function.Tuple2; +import reactor.util.function.Tuples; import java.io.BufferedWriter; import java.io.File; @@ -48,7 +55,10 @@ import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; import static com.appsmith.external.git.constants.GitConstants.ACTION_COLLECTION_LIST; import static com.appsmith.external.git.constants.GitConstants.ACTION_LIST; @@ -74,6 +84,7 @@ public class FileUtilsCEImpl implements FileInterface { private final GitExecutor gitExecutor; protected final FileOperations fileOperations; private final ObservationHelper observationHelper; + protected final ObjectMapper objectMapper; private static final String EDIT_MODE_URL_TEMPLATE = "{{editModeUrl}}"; @@ -90,11 +101,23 @@ public class FileUtilsCEImpl implements FileInterface { GitServiceConfig gitServiceConfig, GitExecutor gitExecutor, FileOperations fileOperations, - ObservationHelper observationHelper) { + ObservationHelper observationHelper, + ObjectMapper objectMapper) { this.gitServiceConfig = gitServiceConfig; this.gitExecutor = gitExecutor; this.fileOperations = fileOperations; this.observationHelper = observationHelper; + this.objectMapper = objectMapper; + } + + protected Map getModifiedResourcesTypes() { + return Map.of( + GitResourceType.JSLIB_CONFIG, GitResourceType.JSLIB_CONFIG, + GitResourceType.CONTEXT_CONFIG, GitResourceType.CONTEXT_CONFIG, + GitResourceType.QUERY_CONFIG, GitResourceType.QUERY_CONFIG, + GitResourceType.QUERY_DATA, GitResourceType.QUERY_CONFIG, + GitResourceType.JSOBJECT_CONFIG, GitResourceType.JSOBJECT_CONFIG, + GitResourceType.JSOBJECT_DATA, GitResourceType.JSOBJECT_CONFIG); } /** @@ -215,6 +238,101 @@ public class FileUtilsCEImpl implements FileInterface { .subscribeOn(scheduler); } + @Override + public Mono saveArtifactToGitRepo(Path baseRepoSuffix, GitResourceMap gitResourceMap, String branchName) + throws GitAPIException, IOException { + + // Repo path will be: + // baseRepo : root/orgId/defaultAppId/repoName/{applicationData} + // Checkout to mentioned branch if not already checked-out + return gitExecutor + .resetToLastCommit(baseRepoSuffix, branchName) + .flatMap(isSwitched -> { + Path baseRepo = Paths.get(gitServiceConfig.getGitRootPath()).resolve(baseRepoSuffix); + + try { + updateEntitiesInRepo(gitResourceMap, baseRepo); + } catch (IOException e) { + return Mono.error(e); + } + + return Mono.just(baseRepo); + }) + .subscribeOn(scheduler); + } + + protected Set getExistingFilesInRepo(Path baseRepo) throws IOException { + try (Stream stream = Files.walk(baseRepo).parallel()) { + return stream.filter(path -> { + try { + return Files.isRegularFile(path) || FileUtils.isEmptyDirectory(path.toFile()); + } catch (IOException e) { + log.error("Unable to find file details. Please check the file at file path: {}", path); + log.error("Assuming that it does not exist for now ..."); + return false; + } + }) + .map(baseRepo::relativize) + .map(Path::toString) + .collect(Collectors.toSet()); + } + } + + protected Set updateEntitiesInRepo(GitResourceMap gitResourceMap, Path baseRepo) throws IOException { + ModifiedResources modifiedResources = gitResourceMap.getModifiedResources(); + Map resourceMap = gitResourceMap.getGitResourceMap(); + + Set filesInRepo = getExistingFilesInRepo(baseRepo); + + Set updatedFilesToBeSerialized = resourceMap.keySet().parallelStream() + .map(gitResourceIdentity -> gitResourceIdentity.getFilePath()) + .collect(Collectors.toSet()); + + // Remove all files that need to be serialized from the existing files list, as well as the README file + // What we are left with are all the files to be deleted + filesInRepo.removeAll(updatedFilesToBeSerialized); + filesInRepo.remove("README.md"); + + // Delete all the files because they are no longer needed + // This covers both older structures of storing files and, + // legitimate changes in the artifact that might cause deletions + filesInRepo.stream().parallel().forEach(filePath -> { + try { + Files.deleteIfExists(baseRepo.resolve(filePath)); + } catch (IOException e) { + // We ignore files that could not be deleted and expect to come back to this at a later point + // Just log the path for now + log.error("Unable to delete file at path: {}", filePath); + } + }); + + // Now go through the resource map and based on resource type, check if the resource is modified before + // serialization + // Or simply choose the mechanism for serialization + Map modifiedResourcesTypes = getModifiedResourcesTypes(); + return resourceMap.entrySet().parallelStream() + .map(entry -> { + GitResourceIdentity key = entry.getKey(); + boolean resourceUpdated = true; + if (modifiedResourcesTypes.containsKey(key.getResourceType()) && modifiedResources != null) { + GitResourceType comparisonType = modifiedResourcesTypes.get(key.getResourceType()); + + resourceUpdated = + modifiedResources.isResourceUpdatedNew(comparisonType, key.getResourceIdentifier()); + } + + if (resourceUpdated) { + String filePath = key.getFilePath(); + saveResourceCommon(entry.getValue(), baseRepo.resolve(filePath)); + + return filePath; + } + return null; + }) + .filter(Objects::nonNull) + .collect(Collectors.toSet()); + } + protected Set updateEntitiesInRepo(ApplicationGitReference applicationGitReference, Path baseRepo) { Set validDatasourceFileNames = new HashSet<>(); @@ -434,6 +552,23 @@ public class FileUtilsCEImpl implements FileInterface { return false; } + protected void saveResourceCommon(Object sourceEntity, Path path) { + try { + Files.createDirectories(path.getParent()); + if (sourceEntity instanceof String s) { + writeStringToFile(s, path); + return; + } + if (sourceEntity instanceof JSONObject) { + sourceEntity = objectMapper.readTree(sourceEntity.toString()); + } + fileOperations.writeToFile(sourceEntity, path); + } catch (IOException e) { + log.error("Error while writing resource to file {} with {}", path, e.getMessage()); + log.debug(e.getMessage()); + } + } + /** * This method is used to write actionCollection specific resource to file system. We write the data in two steps * 1. Actual js code @@ -514,9 +649,9 @@ public class FileUtilsCEImpl implements FileInterface { /** * This will reconstruct the application from the repo * - * @param organisationId To which organisation application needs to be rehydrated + * @param organisationId To which organisation application needs to be rehydrated * @param baseApplicationId To which organisation application needs to be rehydrated - * @param branchName for which the application needs to be rehydrate + * @param branchName for which the application needs to be rehydrate * @return application reference from which entire application can be rehydrated */ public Mono reconstructApplicationReferenceFromGitRepo( @@ -672,6 +807,84 @@ public class FileUtilsCEImpl implements FileInterface { directoryPath.resolve(directoryPath.toFile().getName() + CommonConstants.JSON_EXTENSION)); } + protected GitResourceMap fetchGitResourceMap(Path baseRepoPath) throws IOException { + // Extract application metadata from the json + Object metadata = fileOperations.readFile( + baseRepoPath.resolve(CommonConstants.METADATA + CommonConstants.JSON_EXTENSION)); + Integer fileFormatVersion = fileOperations.getFileFormatVersion(metadata); + // Check if fileFormat of the saved files in repo is compatible + if (!isFileFormatCompatible(fileFormatVersion)) { + throw new AppsmithPluginException(AppsmithPluginError.INCOMPATIBLE_FILE_FORMAT); + } + + GitResourceMap gitResourceMap = new GitResourceMap(); + Map resourceMap = gitResourceMap.getGitResourceMap(); + + Set filesInRepo = getExistingFilesInRepo(baseRepoPath); + + filesInRepo.parallelStream() + .filter(path -> !Files.isDirectory(baseRepoPath.resolve(path))) + .forEach(filePath -> { + Tuple2 identity = getGitResourceIdentity(baseRepoPath, filePath); + + resourceMap.put(identity.getT1(), identity.getT2()); + }); + + return gitResourceMap; + } + + protected Tuple2 getGitResourceIdentity(Path baseRepoPath, String filePath) { + Path path = baseRepoPath.resolve(filePath); + GitResourceIdentity identity; + Object contents = fileOperations.readFile(path); + if (!filePath.contains("/")) { + identity = new GitResourceIdentity(GitResourceType.ROOT_CONFIG, filePath, filePath); + } else if (filePath.matches(DATASOURCE_DIRECTORY + "/.*")) { + String gitSyncId = + objectMapper.valueToTree(contents).get("gitSyncId").asText(); + identity = new GitResourceIdentity(GitResourceType.DATASOURCE_CONFIG, gitSyncId, filePath); + } else if (filePath.matches(JS_LIB_DIRECTORY + "/.*")) { + String fileName = FilenameUtils.getBaseName(filePath); + identity = new GitResourceIdentity(GitResourceType.JSLIB_CONFIG, fileName, filePath); + } else if (filePath.matches(PAGE_DIRECTORY + "/[^/]*/[^/]*.json]")) { + String gitSyncId = + objectMapper.valueToTree(contents).get("gitSyncId").asText(); + identity = new GitResourceIdentity(GitResourceType.CONTEXT_CONFIG, gitSyncId, filePath); + } else if (filePath.matches(PAGE_DIRECTORY + "/[^/]*/" + ACTION_DIRECTORY + "/.*/metadata.json")) { + String gitSyncId = + objectMapper.valueToTree(contents).get("gitSyncId").asText(); + identity = new GitResourceIdentity(GitResourceType.QUERY_CONFIG, gitSyncId, filePath); + } else if (filePath.matches(PAGE_DIRECTORY + "/[^/]*/" + ACTION_DIRECTORY + "/.*\\.txt")) { + Object configContents = fileOperations.readFile(path.getParent().resolve("metadata.json")); + String gitSyncId = + objectMapper.valueToTree(configContents).get("gitSyncId").asText(); + identity = new GitResourceIdentity(GitResourceType.QUERY_DATA, gitSyncId, filePath); + } else if (filePath.matches(PAGE_DIRECTORY + "/[^/]*/" + ACTION_COLLECTION_DIRECTORY + "/.*/metadata.json")) { + String gitSyncId = + objectMapper.valueToTree(contents).get("gitSyncId").asText(); + identity = new GitResourceIdentity(GitResourceType.JSOBJECT_CONFIG, gitSyncId, filePath); + } else if (filePath.matches(PAGE_DIRECTORY + "/[^/]*/" + ACTION_COLLECTION_DIRECTORY + "/.*\\.js")) { + Object configContents = fileOperations.readFile(path.getParent().resolve("metadata.json")); + String gitSyncId = + objectMapper.valueToTree(configContents).get("gitSyncId").asText(); + identity = new GitResourceIdentity(GitResourceType.JSOBJECT_DATA, gitSyncId, filePath); + } else if (filePath.matches(PAGE_DIRECTORY + "/[^/]*/widgets/.*\\.json")) { + Pattern pageDirPattern = Pattern.compile("(" + PAGE_DIRECTORY + "/([^/]*))/widgets/.*\\.json"); + Matcher matcher = pageDirPattern.matcher(filePath); + matcher.find(); + String pageDirectory = matcher.group(1); + String pageName = matcher.group(2) + ".json"; + Object configContents = + fileOperations.readFile(baseRepoPath.resolve(pageDirectory).resolve(pageName)); + String gitSyncId = + objectMapper.valueToTree(configContents).get("gitSyncId").asText(); + String widgetId = objectMapper.valueToTree(contents).get("widgetId").asText(); + identity = new GitResourceIdentity(GitResourceType.JSOBJECT_DATA, gitSyncId + "-" + widgetId, filePath); + } else return null; + + return Tuples.of(identity, contents); + } + private ApplicationGitReference fetchApplicationReference(Path baseRepoPath) { ApplicationGitReference applicationGitReference = new ApplicationGitReference(); // Extract application metadata from the json diff --git a/app/server/appsmith-git/src/main/java/com/appsmith/git/files/FileUtilsImpl.java b/app/server/appsmith-git/src/main/java/com/appsmith/git/files/FileUtilsImpl.java index b45aba4a72..34256a0809 100644 --- a/app/server/appsmith-git/src/main/java/com/appsmith/git/files/FileUtilsImpl.java +++ b/app/server/appsmith-git/src/main/java/com/appsmith/git/files/FileUtilsImpl.java @@ -5,6 +5,7 @@ import com.appsmith.external.git.GitExecutor; import com.appsmith.external.git.operations.FileOperations; import com.appsmith.external.helpers.ObservationHelper; import com.appsmith.git.configurations.GitServiceConfig; +import com.fasterxml.jackson.databind.ObjectMapper; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.springframework.context.annotation.Import; @@ -22,7 +23,8 @@ public class FileUtilsImpl extends FileUtilsCEImpl implements FileInterface { GitServiceConfig gitServiceConfig, GitExecutor gitExecutor, FileOperations fileOperations, - ObservationHelper observationHelper) { - super(gitServiceConfig, gitExecutor, fileOperations, observationHelper); + ObservationHelper observationHelper, + ObjectMapper objectMapper) { + super(gitServiceConfig, gitExecutor, fileOperations, observationHelper, objectMapper); } } diff --git a/app/server/appsmith-git/src/main/java/com/appsmith/git/helpers/DSLTransformerHelper.java b/app/server/appsmith-git/src/main/java/com/appsmith/git/helpers/DSLTransformerHelper.java index 9d444b857d..67b7aafc00 100644 --- a/app/server/appsmith-git/src/main/java/com/appsmith/git/helpers/DSLTransformerHelper.java +++ b/app/server/appsmith-git/src/main/java/com/appsmith/git/helpers/DSLTransformerHelper.java @@ -16,6 +16,10 @@ import java.util.TreeMap; import java.util.stream.Collectors; import static com.appsmith.git.constants.CommonConstants.CANVAS_WIDGET; +import static com.appsmith.git.constants.CommonConstants.DELIMITER_PATH; +import static com.appsmith.git.constants.CommonConstants.DELIMITER_POINT; +import static com.appsmith.git.constants.CommonConstants.EMPTY_STRING; +import static com.appsmith.git.constants.CommonConstants.MAIN_CONTAINER; @Component @RequiredArgsConstructor @@ -24,7 +28,7 @@ public class DSLTransformerHelper { public static Map flatten(JSONObject jsonObject) { Map flattenedMap = new HashMap<>(); - flattenObject(jsonObject, CommonConstants.EMPTY_STRING, flattenedMap); + flattenObject(jsonObject, EMPTY_STRING, flattenedMap); return new TreeMap<>(flattenedMap); } @@ -44,8 +48,7 @@ public class DSLTransformerHelper { for (int i = 0; i < children.length(); i++) { JSONObject childObject = children.getJSONObject(i); - String childPrefix = - isCanvasWidget(childObject) ? prefix + widgetName + CommonConstants.DELIMITER_POINT : prefix; + String childPrefix = isCanvasWidget(childObject) ? prefix + widgetName + DELIMITER_POINT : prefix; String widgetType = getWidgetType(jsonObject); flattenObject(childObject, childPrefix, flattenedMap); } @@ -103,13 +106,17 @@ public class DSLTransformerHelper { Map> parentDirectories = new HashMap<>(); paths = paths.stream() - .map(currentPath -> currentPath.replace(CommonConstants.JSON_EXTENSION, CommonConstants.EMPTY_STRING)) + .map(currentPath -> currentPath.replace(CommonConstants.JSON_EXTENSION, EMPTY_STRING)) .collect(Collectors.toList()); for (String path : paths) { - String[] directories = path.split(CommonConstants.DELIMITER_PATH); + String[] directories = path.split(DELIMITER_PATH); int lastDirectoryIndex = directories.length - 1; - if (lastDirectoryIndex > 0 && directories[lastDirectoryIndex].equals(directories[lastDirectoryIndex - 1])) { + if (lastDirectoryIndex <= 0) { + // This is not a valid path anymore, ignore + continue; + } + if (directories[lastDirectoryIndex].equals(directories[lastDirectoryIndex - 1])) { if (lastDirectoryIndex - 2 >= 0) { String parentDirectory = directories[lastDirectoryIndex - 2]; List pathsList = parentDirectories.getOrDefault(parentDirectory, new ArrayList<>()); @@ -143,10 +150,10 @@ public class DSLTransformerHelper { Map jsonMap, Map> pathMapping, JSONObject mainContainer) { // start from the root // Empty page with no widgets - if (!pathMapping.containsKey(CommonConstants.MAIN_CONTAINER)) { + if (!pathMapping.containsKey(MAIN_CONTAINER)) { return mainContainer; } - for (String path : pathMapping.get(CommonConstants.MAIN_CONTAINER)) { + for (String path : pathMapping.get(MAIN_CONTAINER)) { JSONObject child = getChildren(path, jsonMap, pathMapping); JSONArray children = mainContainer.optJSONArray(CommonConstants.CHILDREN); if (children == null) { @@ -179,7 +186,7 @@ public class DSLTransformerHelper { } public static String getWidgetName(String path) { - String[] directories = path.split(CommonConstants.DELIMITER_PATH); + String[] directories = path.split(DELIMITER_PATH); return directories[directories.length - 1]; } @@ -229,15 +236,16 @@ public class DSLTransformerHelper { public static String getPathToWidgetFile(String key, JSONObject jsonObject, String widgetName) { // get path with splitting the name via key - String childPath = key.replace(CommonConstants.MAIN_CONTAINER, CommonConstants.EMPTY_STRING) - .replace(CommonConstants.DELIMITER_POINT, CommonConstants.DELIMITER_PATH); + String childPath = key.replace(MAIN_CONTAINER, EMPTY_STRING).replace(DELIMITER_POINT, DELIMITER_PATH); // Replace the canvas Widget as a child and add it to the same level as parent - childPath = childPath.replaceAll(CANVAS_WIDGET, CommonConstants.EMPTY_STRING); + childPath = childPath.replaceAll(CANVAS_WIDGET, EMPTY_STRING); if (!DSLTransformerHelper.hasChildren(jsonObject) && !DSLTransformerHelper.isTabsWidget(jsonObject)) { // Save the widget as a directory or Save the widget as a file // Only consider widgetName at the end of the childPath to reset // For example, "foobar/bar" should convert into "foobar/" - childPath = childPath.replaceAll(widgetName + "$", CommonConstants.EMPTY_STRING); + childPath = childPath.replaceAll(widgetName + "$", EMPTY_STRING); + } else { + childPath += DELIMITER_PATH; } return childPath; diff --git a/app/server/appsmith-git/src/test/java/com/appsmith/git/helpers/FileUtilsImplTest.java b/app/server/appsmith-git/src/test/java/com/appsmith/git/helpers/FileUtilsImplTest.java index 58b4c3bfc7..e222feff2a 100644 --- a/app/server/appsmith-git/src/test/java/com/appsmith/git/helpers/FileUtilsImplTest.java +++ b/app/server/appsmith-git/src/test/java/com/appsmith/git/helpers/FileUtilsImplTest.java @@ -7,6 +7,7 @@ import com.appsmith.git.configurations.GitServiceConfig; import com.appsmith.git.files.FileUtilsImpl; import com.appsmith.git.files.operations.FileOperationsImpl; import com.appsmith.git.service.GitExecutorImpl; +import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.commons.io.FileUtils; import org.eclipse.jgit.api.errors.GitAPIException; import org.junit.jupiter.api.AfterEach; @@ -42,7 +43,8 @@ public class FileUtilsImplTest { GitServiceConfig gitServiceConfig = new GitServiceConfig(); gitServiceConfig.setGitRootPath(localTestDirectoryPath.toString()); FileOperations fileOperations = new FileOperationsImpl(null, ObservationHelper.NOOP); - fileUtils = new FileUtilsImpl(gitServiceConfig, gitExecutor, fileOperations, ObservationHelper.NOOP); + fileUtils = new FileUtilsImpl( + gitServiceConfig, gitExecutor, fileOperations, ObservationHelper.NOOP, new ObjectMapper()); } @AfterEach diff --git a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/dtos/ModifiedResources.java b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/dtos/ModifiedResources.java index 8157d6c276..166b379c42 100644 --- a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/dtos/ModifiedResources.java +++ b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/dtos/ModifiedResources.java @@ -1,13 +1,7 @@ package com.appsmith.external.dtos; +import com.appsmith.external.dtos.ce.ModifiedResourcesCE; import lombok.Data; -import org.apache.commons.lang3.StringUtils; -import org.springframework.util.CollectionUtils; - -import java.util.HashSet; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; /** * This DTO class is used to store which resources have been updated after the last commit. @@ -17,49 +11,4 @@ import java.util.concurrent.ConcurrentHashMap; * the pages to file system for difference git processes e.g. check git status, commit etc */ @Data -public class ModifiedResources { - // boolean flag to set whether all the resources should be considered as updated or not, it'll be false by default - private boolean isAllModified; - - // a map to store the type of the resources and related entries - Map> modifiedResourceMap = new ConcurrentHashMap<>(); - - /** - * Checks whether the provided resource name should be considered as modified or not. - * It'll return true if the isAllModified flag is set or the resource is present in the modifiedResourceMap - * @param resourceType String, type of the resource e.g. PAGE_LIST - * @param resourceName String, name of the resource e.g. "Home Page" - * @return true if modified, false otherwise - */ - public boolean isResourceUpdated(String resourceType, String resourceName) { - return StringUtils.isNotEmpty(resourceType) - && (isAllModified - || (!CollectionUtils.isEmpty(modifiedResourceMap.get(resourceType)) - && modifiedResourceMap.get(resourceType).contains(resourceName))); - } - - /** - * Adds a new resource to the map. Will create a new set if no set found for the provided resource type. - * @param resourceType String, type of the resource e.g. PAGE_LST - * @param resourceName String, name of the resource e.g. Home Page - */ - public void putResource(String resourceType, String resourceName) { - if (!this.modifiedResourceMap.containsKey(resourceType)) { - this.modifiedResourceMap.put(resourceType, new HashSet<>()); - } - this.modifiedResourceMap.get(resourceType).add(resourceName); - } - - /** - * Adds a set of resources to the map. Will create a new set if no set found for the provided resource type. - * It'll append the resources to the set. - * @param resourceType String, type of the resource e.g. PAGE_LST - * @param resourceNames Set of String, names of the resource e.g. Home Page, About page - */ - public void putResource(String resourceType, Set resourceNames) { - if (!this.modifiedResourceMap.containsKey(resourceType)) { - this.modifiedResourceMap.put(resourceType, new HashSet<>()); - } - this.modifiedResourceMap.get(resourceType).addAll(resourceNames); - } -} +public class ModifiedResources extends ModifiedResourcesCE {} diff --git a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/dtos/ce/ModifiedResourcesCE.java b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/dtos/ce/ModifiedResourcesCE.java new file mode 100644 index 0000000000..9e3e5a3aa8 --- /dev/null +++ b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/dtos/ce/ModifiedResourcesCE.java @@ -0,0 +1,86 @@ +package com.appsmith.external.dtos.ce; + +import com.appsmith.external.git.models.GitResourceType; +import lombok.Data; +import org.apache.commons.lang3.StringUtils; +import org.springframework.util.CollectionUtils; + +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +/** + * This DTO class is used to store which resources have been updated after the last commit. + * Primarily the export process sets this information and git import process uses this information to identify + * which resources need to be written in file system. For example, if a page has not been updated after the last commit, + * the name of the page should not be part of the modifiedResourceMap so that git will skip this page when it writes + * the pages to file system for difference git processes e.g. check git status, commit etc + */ +@Data +public class ModifiedResourcesCE { + // boolean flag to set whether all the resources should be considered as updated or not, it'll be false by default + private boolean isAllModified; + + // a map to store the type of the resources and related entries + Map> modifiedResourceMap = new ConcurrentHashMap<>(); + + Map> modifiedResourceIdentifiers = new ConcurrentHashMap<>(); + + public Map> getModifiedResourceIdentifiers() { + if (this.modifiedResourceIdentifiers.isEmpty()) { + this.modifiedResourceIdentifiers.putAll(Map.of( + GitResourceType.CONTEXT_CONFIG, ConcurrentHashMap.newKeySet(), + GitResourceType.JSLIB_CONFIG, ConcurrentHashMap.newKeySet(), + GitResourceType.QUERY_CONFIG, ConcurrentHashMap.newKeySet(), + GitResourceType.JSOBJECT_CONFIG, ConcurrentHashMap.newKeySet())); + } + return modifiedResourceIdentifiers; + } + + /** + * Checks whether the provided resource name should be considered as modified or not. + * It'll return true if the isAllModified flag is set or the resource is present in the modifiedResourceMap + * @param resourceType String, type of the resource e.g. PAGE_LIST + * @param resourceName String, name of the resource e.g. "Home Page" + * @return true if modified, false otherwise + */ + public boolean isResourceUpdated(String resourceType, String resourceName) { + return StringUtils.isNotEmpty(resourceType) + && (isAllModified + || (!CollectionUtils.isEmpty(modifiedResourceMap.get(resourceType)) + && modifiedResourceMap.get(resourceType).contains(resourceName))); + } + + public boolean isResourceUpdatedNew(GitResourceType resourceType, String resourceIdentifier) { + return StringUtils.isNotEmpty(resourceIdentifier) + && (isAllModified + || (!CollectionUtils.isEmpty(modifiedResourceIdentifiers.get(resourceType)) + && modifiedResourceIdentifiers.get(resourceType).contains(resourceIdentifier))); + } + + /** + * Adds a new resource to the map. Will create a new set if no set found for the provided resource type. + * @param resourceType String, type of the resource e.g. PAGE_LST + * @param resourceName String, name of the resource e.g. Home Page + */ + public void putResource(String resourceType, String resourceName) { + if (!this.modifiedResourceMap.containsKey(resourceType)) { + this.modifiedResourceMap.put(resourceType, new HashSet<>()); + } + this.modifiedResourceMap.get(resourceType).add(resourceName); + } + + /** + * Adds a set of resources to the map. Will create a new set if no set found for the provided resource type. + * It'll append the resources to the set. + * @param resourceType String, type of the resource e.g. PAGE_LST + * @param resourceNames Set of String, names of the resource e.g. Home Page, About page + */ + public void putResource(String resourceType, Set resourceNames) { + if (!this.modifiedResourceMap.containsKey(resourceType)) { + this.modifiedResourceMap.put(resourceType, new HashSet<>()); + } + this.modifiedResourceMap.get(resourceType).addAll(resourceNames); + } +} diff --git a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/git/FileInterface.java b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/git/FileInterface.java index fd17dd1e51..d5422a24ef 100644 --- a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/git/FileInterface.java +++ b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/git/FileInterface.java @@ -1,5 +1,6 @@ package com.appsmith.external.git; +import com.appsmith.external.git.models.GitResourceMap; import com.appsmith.external.models.ApplicationGitReference; import com.appsmith.external.models.ArtifactGitReference; import org.eclipse.jgit.api.errors.GitAPIException; @@ -34,6 +35,9 @@ public interface FileInterface { Path baseRepoSuffix, ArtifactGitReference artifactGitReference, String branchName) throws IOException, GitAPIException; + Mono saveArtifactToGitRepo(Path baseRepoSuffix, GitResourceMap gitResourceMap, String branchName) + throws GitAPIException, IOException; + /** * This method will reconstruct the application from the repo * diff --git a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/git/models/GitResourceIdentity.java b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/git/models/GitResourceIdentity.java index e142821665..2bf12f779f 100644 --- a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/git/models/GitResourceIdentity.java +++ b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/git/models/GitResourceIdentity.java @@ -8,8 +8,6 @@ import lombok.RequiredArgsConstructor; @Data @RequiredArgsConstructor public class GitResourceIdentity { - // TODO @Nidhi should we persist the info from parsing this filePath ? - String filePath; // TODO @Nidhi should we persist this sha against the Appsmith domain to integrate with the isModified logic? String sha; @@ -25,4 +23,6 @@ public class GitResourceIdentity { // root dir files -> fileName @NonNull @EqualsAndHashCode.Include String resourceIdentifier; + + @NonNull String filePath; } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/actioncollections/exportable/ActionCollectionExportableServiceCEImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/actioncollections/exportable/ActionCollectionExportableServiceCEImpl.java index 34f46f26e3..dd4d34a12c 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/actioncollections/exportable/ActionCollectionExportableServiceCEImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/actioncollections/exportable/ActionCollectionExportableServiceCEImpl.java @@ -1,5 +1,6 @@ package com.appsmith.server.actioncollections.exportable; +import com.appsmith.external.git.models.GitResourceType; import com.appsmith.server.acl.AclPermission; import com.appsmith.server.constants.FieldName; import com.appsmith.server.domains.ActionCollection; @@ -11,7 +12,6 @@ import com.appsmith.server.dtos.ExportingMetaDTO; import com.appsmith.server.dtos.MappedExportableResourcesDTO; import com.appsmith.server.exports.exportable.ExportableServiceCE; import com.appsmith.server.exports.exportable.artifactbased.ArtifactBasedExportableService; -import com.appsmith.server.helpers.ImportExportUtils; import com.appsmith.server.solutions.ActionPermission; import lombok.RequiredArgsConstructor; import reactor.core.publisher.Flux; @@ -67,6 +67,7 @@ public class ActionCollectionExportableServiceCEImpl implements ExportableServic // Because the actions will have a reference to the collection Set updatedActionCollectionSet = new HashSet<>(); + Set updatedIdentifiers = new HashSet<>(); actionCollections.forEach(actionCollection -> { ActionCollectionDTO publishedActionCollectionDTO = actionCollection.getPublishedCollection(); ActionCollectionDTO unpublishedActionCollectionDTO = @@ -78,9 +79,12 @@ public class ActionCollectionExportableServiceCEImpl implements ExportableServic // we've replaced page id with page name in previous step String contextNameAtIdReference = artifactBasedExportableService.getContextNameAtIdReference(actionCollectionDTO); - String contextListPath = artifactBasedExportableService.getContextListPath(); - boolean isContextUpdated = ImportExportUtils.isContextNameInUpdatedList( - artifactExchangeJson, contextNameAtIdReference, contextListPath); + String contextGitSyncId = mappedExportableResourcesDTO + .getContextNameToGitSyncIdMap() + .get(contextNameAtIdReference); + boolean isContextUpdated = artifactExchangeJson + .getModifiedResources() + .isResourceUpdatedNew(GitResourceType.CONTEXT_CONFIG, contextGitSyncId); String actionCollectionName = actionCollectionDTO.getUserExecutableName() + NAME_SEPARATOR + contextNameAtIdReference; Instant actionCollectionUpdatedAt = actionCollection.getUpdatedAt(); @@ -92,6 +96,7 @@ public class ActionCollectionExportableServiceCEImpl implements ExportableServic || exportingMetaDTO.getArtifactLastCommittedAt().isBefore(actionCollectionUpdatedAt); if (isActionCollectionUpdated) { updatedActionCollectionSet.add(actionCollectionName); + updatedIdentifiers.add(actionCollection.getGitSyncId()); } actionCollection.sanitiseToExportDBObject(); }); @@ -100,6 +105,11 @@ public class ActionCollectionExportableServiceCEImpl implements ExportableServic artifactExchangeJson .getModifiedResources() .putResource(FieldName.ACTION_COLLECTION_LIST, updatedActionCollectionSet); + artifactExchangeJson + .getModifiedResources() + .getModifiedResourceIdentifiers() + .get(GitResourceType.JSOBJECT_CONFIG) + .addAll(updatedIdentifiers); return actionCollections; }) diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/applications/git/ApplicationGitFileUtilsCEImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/applications/git/ApplicationGitFileUtilsCEImpl.java index dbe2f5dc2a..a5628db2df 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/applications/git/ApplicationGitFileUtilsCEImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/applications/git/ApplicationGitFileUtilsCEImpl.java @@ -19,6 +19,7 @@ import com.appsmith.server.domains.ActionCollection; import com.appsmith.server.domains.Application; import com.appsmith.server.domains.ApplicationPage; import com.appsmith.server.domains.CustomJSLib; +import com.appsmith.server.domains.Layout; import com.appsmith.server.domains.NewAction; import com.appsmith.server.domains.NewPage; import com.appsmith.server.domains.Theme; @@ -32,9 +33,12 @@ import com.appsmith.server.helpers.CollectionUtils; import com.appsmith.server.helpers.ce.ArtifactGitFileUtilsCE; import com.appsmith.server.migrations.JsonSchemaMigration; import com.appsmith.server.newactions.base.NewActionService; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.MapperFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.gson.Gson; import lombok.NonNull; -import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import net.minidev.json.JSONObject; import net.minidev.json.parser.JSONParser; @@ -60,6 +64,11 @@ import java.util.stream.Collectors; import static com.appsmith.external.git.constants.GitConstants.NAME_SEPARATOR; import static com.appsmith.external.helpers.AppsmithBeanUtils.copyNestedNonNullProperties; import static com.appsmith.external.helpers.AppsmithBeanUtils.copyProperties; +import static com.appsmith.git.constants.CommonConstants.DELIMITER_PATH; +import static com.appsmith.git.constants.CommonConstants.JSON_EXTENSION; +import static com.appsmith.git.constants.CommonConstants.MAIN_CONTAINER; +import static com.appsmith.git.constants.CommonConstants.WIDGETS; +import static com.appsmith.git.constants.ce.GitDirectoriesCE.PAGE_DIRECTORY; import static com.appsmith.server.constants.FieldName.ACTION_COLLECTION_LIST; import static com.appsmith.server.constants.FieldName.ACTION_LIST; import static com.appsmith.server.constants.FieldName.CHILDREN; @@ -70,20 +79,36 @@ import static com.appsmith.server.constants.FieldName.EDIT_MODE_THEME; import static com.appsmith.server.constants.FieldName.EXPORTED_APPLICATION; import static com.appsmith.server.constants.FieldName.PAGE_LIST; import static com.appsmith.server.constants.FieldName.WIDGET_ID; +import static com.appsmith.server.constants.ce.FieldNameCE.WIDGET_NAME; import static com.appsmith.server.helpers.ce.CommonGitFileUtilsCE.removeUnwantedFieldsFromBaseDomain; @Slf4j @Component -@RequiredArgsConstructor @Import({FileUtilsImpl.class}) public class ApplicationGitFileUtilsCEImpl implements ArtifactGitFileUtilsCE { private final Gson gson; + private final ObjectMapper objectMapper; private final NewActionService newActionService; private final FileInterface fileUtils; private final JsonSchemaMigration jsonSchemaMigration; private final ActionCollectionService actionCollectionService; + public ApplicationGitFileUtilsCEImpl( + Gson gson, + ObjectMapper objectMapper, + NewActionService newActionService, + FileInterface fileUtils, + JsonSchemaMigration jsonSchemaMigration, + ActionCollectionService actionCollectionService) { + this.gson = gson; + this.objectMapper = objectMapper.copy().disable(MapperFeature.USE_ANNOTATIONS); + this.newActionService = newActionService; + this.fileUtils = fileUtils; + this.jsonSchemaMigration = jsonSchemaMigration; + this.actionCollectionService = actionCollectionService; + } + // Only include the application helper fields in metadata object protected Set getBlockedMetadataFields() { return Set.of( @@ -109,6 +134,11 @@ public class ApplicationGitFileUtilsCEImpl implements ArtifactGitFileUtilsCE { removeUnwantedFieldsFromPage(newPage); - JSONObject dsl = - newPage.getUnpublishedPage().getLayouts().get(0).getDsl(); + PageDTO pageDTO = newPage.getUnpublishedPage(); + JSONObject dsl = pageDTO.getLayouts().get(0).getDsl(); // Get MainContainer widget data, remove the children and club with Canvas.json file JSONObject mainContainer = new JSONObject(dsl); mainContainer.remove(CHILDREN); - newPage.getUnpublishedPage().getLayouts().get(0).setDsl(mainContainer); + pageDTO.getLayouts().get(0).setDsl(mainContainer); // pageName will be used for naming the json file - GitResourceIdentity pageIdentity = - new GitResourceIdentity(GitResourceType.CONTEXT_CONFIG, newPage.getGitSyncId()); + final String pagePathPrefix = PAGE_DIRECTORY + DELIMITER_PATH + pageDTO.getName() + DELIMITER_PATH; + final String pageFilePath = pagePathPrefix + pageDTO.getName() + JSON_EXTENSION; + GitResourceIdentity pageIdentity = new GitResourceIdentity( + GitResourceType.CONTEXT_CONFIG, newPage.getGitSyncId(), pageFilePath); resourceMap.put(pageIdentity, newPage); Map result = DSLTransformerHelper.flatten(new org.json.JSONObject(dsl.toString())); result.forEach((key, jsonObject) -> { String widgetId = newPage.getGitSyncId() + "-" + jsonObject.getString(WIDGET_ID); + String widgetsPath = pagePathPrefix + WIDGETS + DELIMITER_PATH; + String widgetName = jsonObject.getString(WIDGET_NAME); + String subPath = DSLTransformerHelper.getPathToWidgetFile(key, jsonObject, widgetName); + + String widgetPath = widgetsPath + subPath + widgetName + JSON_EXTENSION; GitResourceIdentity widgetIdentity = - new GitResourceIdentity(GitResourceType.WIDGET_CONFIG, widgetId); + new GitResourceIdentity(GitResourceType.WIDGET_CONFIG, widgetId, widgetPath); resourceMap.put(widgetIdentity, jsonObject); }); }); @@ -629,4 +669,80 @@ public class ApplicationGitFileUtilsCEImpl implements ArtifactGitFileUtilsCE resourceMap = gitResourceMap.getGitResourceMap(); + + // exported application + final String applicationFilePath = CommonConstants.APPLICATION + JSON_EXTENSION; + GitResourceIdentity applicationJsonIdentity = + new GitResourceIdentity(GitResourceType.ROOT_CONFIG, applicationFilePath, applicationFilePath); + + Object applicationObject = resourceMap.get(applicationJsonIdentity); + Application application = objectMapper.convertValue(applicationObject, Application.class); + artifactExchangeJson.setArtifact(application); + + // metadata + final String metadataFilePath = CommonConstants.METADATA + JSON_EXTENSION; + GitResourceIdentity metadataIdentity = + new GitResourceIdentity(GitResourceType.ROOT_CONFIG, metadataFilePath, metadataFilePath); + + Object metadataObject = resourceMap.get(metadataIdentity); + ApplicationJson metadata = objectMapper.convertValue(metadataObject, ApplicationJson.class); + copyNestedNonNullProperties(metadata, artifactExchangeJson); + + // pages + List pageList = resourceMap.entrySet().stream() + .filter(entry -> { + GitResourceIdentity key = entry.getKey(); + return GitResourceType.CONTEXT_CONFIG.equals(key.getResourceType()); + }) + .map(Map.Entry::getValue) + .map(pageObject -> objectMapper.convertValue(pageObject, NewPage.class)) + .collect(Collectors.toList()); + artifactExchangeJson.setContextList(pageList); + + // widgets + + pageList.parallelStream().forEach(newPage -> { + Map widgetsData = resourceMap.entrySet().stream() + .filter(entry -> { + GitResourceIdentity key = entry.getKey(); + return GitResourceType.WIDGET_CONFIG.equals(key.getResourceType()) + && key.getResourceIdentifier().startsWith(newPage.getGitSyncId() + "-"); + }) + .collect(Collectors.toMap( + entry -> entry.getKey() + .getFilePath() + .replaceFirst( + PAGE_DIRECTORY + + newPage.getUnpublishedPage() + .getName() + + DELIMITER_PATH + + WIDGETS + + DELIMITER_PATH, + MAIN_CONTAINER + DELIMITER_PATH), + entry -> (org.json.JSONObject) entry.getValue())); + + Layout layout = newPage.getUnpublishedPage().getLayouts().get(0); + org.json.JSONObject mainContainer; + try { + mainContainer = new org.json.JSONObject(objectMapper.writeValueAsString(layout.getDsl())); + + Map> parentDirectories = DSLTransformerHelper.calculateParentDirectories( + widgetsData.keySet().stream().toList()); + org.json.JSONObject nestedDSL = + DSLTransformerHelper.getNestedDSL(widgetsData, parentDirectories, mainContainer); + + JSONParser jsonParser = new JSONParser(); + JSONObject parsedDSL = jsonParser.parse(nestedDSL.toString(), JSONObject.class); + + layout.setDsl(parsedDSL); + } catch (ParseException | JsonProcessingException e) { + throw new RuntimeException(e); + } + }); + } } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/applications/git/ApplicationGitFileUtilsImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/applications/git/ApplicationGitFileUtilsImpl.java index a5535c5f05..fcf0b2682a 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/applications/git/ApplicationGitFileUtilsImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/applications/git/ApplicationGitFileUtilsImpl.java @@ -6,6 +6,7 @@ import com.appsmith.server.actioncollections.base.ActionCollectionService; import com.appsmith.server.helpers.ArtifactGitFileUtils; import com.appsmith.server.migrations.JsonSchemaMigration; import com.appsmith.server.newactions.base.NewActionService; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.Gson; import org.springframework.stereotype.Component; @@ -15,10 +16,11 @@ public class ApplicationGitFileUtilsImpl extends ApplicationGitFileUtilsCEImpl public ApplicationGitFileUtilsImpl( Gson gson, + ObjectMapper objectMapper, NewActionService newActionService, FileInterface fileUtils, JsonSchemaMigration jsonSchemaMigration, ActionCollectionService actionCollectionService) { - super(gson, newActionService, fileUtils, jsonSchemaMigration, actionCollectionService); + super(gson, objectMapper, newActionService, fileUtils, jsonSchemaMigration, actionCollectionService); } } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/dtos/ce/ApplicationJsonCE.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/dtos/ce/ApplicationJsonCE.java index 7f4ec76ee8..348ca06e01 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/dtos/ce/ApplicationJsonCE.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/dtos/ce/ApplicationJsonCE.java @@ -10,6 +10,7 @@ import com.appsmith.server.constants.ArtifactType; import com.appsmith.server.domains.ActionCollection; import com.appsmith.server.domains.Application; import com.appsmith.server.domains.Artifact; +import com.appsmith.server.domains.Context; import com.appsmith.server.domains.CustomJSLib; import com.appsmith.server.domains.NewAction; import com.appsmith.server.domains.NewPage; @@ -123,6 +124,11 @@ public class ApplicationJsonCE implements ArtifactExchangeJsonCE { return this.getExportedApplication(); } + @Override + public void setArtifact(T application) { + this.exportedApplication = (Application) application; + } + @Override public void setThemes(Theme unpublishedTheme, Theme publishedTheme) { this.setEditModeTheme(unpublishedTheme); @@ -138,4 +144,9 @@ public class ApplicationJsonCE implements ArtifactExchangeJsonCE { public List getContextList() { return this.pageList; } + + @Override + public void setContextList(List contextList) { + this.pageList = (List) contextList; + } } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/dtos/ce/ArtifactExchangeJsonCE.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/dtos/ce/ArtifactExchangeJsonCE.java index 47bf907e3d..f22d68d369 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/dtos/ce/ArtifactExchangeJsonCE.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/dtos/ce/ArtifactExchangeJsonCE.java @@ -30,6 +30,8 @@ public interface ArtifactExchangeJsonCE { Artifact getArtifact(); + void setArtifact(T artifact); + default void setThemes(Theme unpublishedTheme, Theme publishedTheme) {} default List getCustomJSLibList() { @@ -68,4 +70,6 @@ public interface ArtifactExchangeJsonCE { @JsonView(Views.Internal.class) List getContextList(); + + void setContextList(List contextList); } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/dtos/ce/MappedExportableResourcesCE_DTO.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/dtos/ce/MappedExportableResourcesCE_DTO.java index 0490dbc2d5..d69a886602 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/dtos/ce/MappedExportableResourcesCE_DTO.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/dtos/ce/MappedExportableResourcesCE_DTO.java @@ -15,6 +15,7 @@ public class MappedExportableResourcesCE_DTO { Map datasourceIdToNameMap = new HashMap<>(); Map datasourceNameToUpdatedAtMap = new HashMap<>(); Map contextIdToNameMap = new HashMap<>(); + Map contextNameToGitSyncIdMap = new HashMap<>(); Map actionIdToNameMap = new HashMap<>(); Map collectionIdToNameMap = new HashMap<>(); } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/git/autocommit/AutoCommitEventHandlerCEImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/git/autocommit/AutoCommitEventHandlerCEImpl.java index 8e9313b90c..bfb09ac1e5 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/git/autocommit/AutoCommitEventHandlerCEImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/git/autocommit/AutoCommitEventHandlerCEImpl.java @@ -4,12 +4,14 @@ import com.appsmith.external.constants.AnalyticsEvents; import com.appsmith.external.dtos.ModifiedResources; import com.appsmith.external.git.GitExecutor; import com.appsmith.external.git.constants.GitConstants.GitCommandConstants; +import com.appsmith.external.git.models.GitResourceType; import com.appsmith.server.configurations.ProjectProperties; import com.appsmith.server.constants.ArtifactType; import com.appsmith.server.constants.FieldName; import com.appsmith.server.domains.Layout; import com.appsmith.server.domains.NewPage; import com.appsmith.server.dtos.ApplicationJson; +import com.appsmith.server.dtos.PageDTO; import com.appsmith.server.events.AutoCommitEvent; import com.appsmith.server.exceptions.AppsmithError; import com.appsmith.server.exceptions.AppsmithException; @@ -28,14 +30,16 @@ import org.springframework.scheduling.annotation.Async; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import reactor.core.scheduler.Schedulers; +import reactor.util.function.Tuple2; +import reactor.util.function.Tuples; import java.nio.file.Path; import java.nio.file.Paths; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; import static com.appsmith.external.git.constants.GitConstants.PAGE_LIST; import static java.lang.Boolean.TRUE; @@ -212,9 +216,17 @@ public class AutoCommitEventHandlerCEImpl implements AutoCommitEventHandlerCE { ApplicationJson to file system conversion will use this field to decide which pages need to be written back to file system. */ - Set pageNamesSet = new HashSet<>(updatedPageNamesList); + + Set pageNamesSet = + updatedPageNamesList.stream().map(Tuple2::getT1).collect(Collectors.toSet()); + Set pageIdentifiersSet = + updatedPageNamesList.stream().map(Tuple2::getT2).collect(Collectors.toSet()); ModifiedResources modifiedResources = new ModifiedResources(); modifiedResources.putResource(PAGE_LIST, pageNamesSet); + modifiedResources + .getModifiedResourceIdentifiers() + .get(GitResourceType.CONTEXT_CONFIG) + .addAll(pageIdentifiersSet); modifiedResources.setAllModified(true); applicationJson.setModifiedResources(modifiedResources); return applicationJson; @@ -236,7 +248,7 @@ public class AutoCommitEventHandlerCEImpl implements AutoCommitEventHandlerCE { * @param latestSchemaVersion latest dsl schema version obtained from RTS * @return list of names of the pages that have been migrated. */ - private Mono> migratePageDsl(List newPageList, Integer latestSchemaVersion) { + private Mono>> migratePageDsl(List newPageList, Integer latestSchemaVersion) { return Flux.fromIterable(newPageList) .filter(newPage -> { // filter the pages which have unpublished page with layouts and where dsl version is not latest @@ -249,8 +261,8 @@ public class AutoCommitEventHandlerCEImpl implements AutoCommitEventHandlerCE { } return false; }) - .map(NewPage::getUnpublishedPage) - .flatMap(pageDTO -> { + .flatMap(newPage -> { + PageDTO pageDTO = newPage.getUnpublishedPage(); Layout layout = pageDTO.getLayouts().get(0); return dslMigrationUtils .migratePageDsl(layout.getDsl()) @@ -258,7 +270,7 @@ public class AutoCommitEventHandlerCEImpl implements AutoCommitEventHandlerCE { layout.setDsl(migratedDsl); return migratedDsl; }) - .thenReturn(pageDTO.getName()); + .thenReturn(Tuples.of(pageDTO.getName(), newPage.getGitSyncId())); }) .collectList(); } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/CommonGitFileUtils.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/CommonGitFileUtils.java index ecb77dd871..90c512060d 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/CommonGitFileUtils.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/CommonGitFileUtils.java @@ -10,6 +10,7 @@ import com.appsmith.server.migrations.JsonSchemaVersions; import com.appsmith.server.newactions.base.NewActionService; import com.appsmith.server.services.AnalyticsService; import com.appsmith.server.services.SessionUserService; +import com.fasterxml.jackson.databind.ObjectMapper; import lombok.extern.slf4j.Slf4j; import org.springframework.context.annotation.Import; import org.springframework.stereotype.Component; @@ -27,7 +28,8 @@ public class CommonGitFileUtils extends CommonGitFileUtilsCE { SessionUserService sessionUserService, NewActionService newActionService, ActionCollectionService actionCollectionService, - JsonSchemaVersions jsonSchemaVersions) { + JsonSchemaVersions jsonSchemaVersions, + ObjectMapper objectMapper) { super( applicationGitFileUtils, fileUtils, @@ -36,6 +38,7 @@ public class CommonGitFileUtils extends CommonGitFileUtilsCE { sessionUserService, newActionService, actionCollectionService, - jsonSchemaVersions); + jsonSchemaVersions, + objectMapper); } } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/ce/ArtifactGitFileUtilsCE.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/ce/ArtifactGitFileUtilsCE.java index 0af8ce9323..f92820c047 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/ce/ArtifactGitFileUtilsCE.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/ce/ArtifactGitFileUtilsCE.java @@ -13,6 +13,8 @@ public interface ArtifactGitFileUtilsCE { T createArtifactReferenceObject(); + ArtifactExchangeJson createArtifactExchangeJsonObject(); + void setArtifactDependentResources(ArtifactExchangeJson artifactExchangeJson, GitResourceMap gitResourceMap); Mono reconstructArtifactExchangeJsonFromFilesInRepository( @@ -24,4 +26,6 @@ public interface ArtifactGitFileUtilsCE { Map getConstantsMap(); Path getRepoSuffixPath(String workspaceId, String artifactId, String repoName, @NonNull String... args); + + void setArtifactDependentPropertiesInJson(GitResourceMap gitResourceMap, ArtifactExchangeJson artifactExchangeJson); } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/ce/CommonGitFileUtilsCE.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/ce/CommonGitFileUtilsCE.java index e0955455d6..486406d642 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/ce/CommonGitFileUtilsCE.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/ce/CommonGitFileUtilsCE.java @@ -7,6 +7,8 @@ import com.appsmith.external.git.models.GitResourceMap; import com.appsmith.external.git.models.GitResourceType; import com.appsmith.external.git.operations.FileOperations; import com.appsmith.external.helpers.Stopwatch; +import com.appsmith.external.models.ActionConfiguration; +import com.appsmith.external.models.ActionDTO; import com.appsmith.external.models.ApplicationGitReference; import com.appsmith.external.models.ArtifactGitReference; import com.appsmith.external.models.BaseDomain; @@ -22,6 +24,7 @@ import com.appsmith.server.domains.CustomJSLib; import com.appsmith.server.domains.GitArtifactMetadata; import com.appsmith.server.domains.NewAction; import com.appsmith.server.domains.Theme; +import com.appsmith.server.dtos.ActionCollectionDTO; import com.appsmith.server.dtos.ApplicationJson; import com.appsmith.server.dtos.ArtifactExchangeJson; import com.appsmith.server.dtos.PageDTO; @@ -32,10 +35,12 @@ import com.appsmith.server.migrations.JsonSchemaVersions; import com.appsmith.server.newactions.base.NewActionService; import com.appsmith.server.services.AnalyticsService; import com.appsmith.server.services.SessionUserService; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.MapperFeature; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.Gson; import com.google.gson.JsonElement; import com.google.gson.JsonObject; -import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.eclipse.jgit.api.errors.GitAPIException; import org.json.JSONObject; @@ -52,19 +57,30 @@ import java.nio.file.Paths; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.stream.Collector; +import java.util.stream.Collectors; import static com.appsmith.external.git.constants.ce.GitConstantsCE.GitCommandConstantsCE.CHECKOUT_BRANCH; import static com.appsmith.external.git.constants.ce.GitConstantsCE.RECONSTRUCT_PAGE; import static com.appsmith.git.constants.CommonConstants.CLIENT_SCHEMA_VERSION; +import static com.appsmith.git.constants.CommonConstants.DELIMITER_PATH; import static com.appsmith.git.constants.CommonConstants.FILE_FORMAT_VERSION; import static com.appsmith.git.constants.CommonConstants.JSON_EXTENSION; +import static com.appsmith.git.constants.CommonConstants.JS_EXTENSION; +import static com.appsmith.git.constants.CommonConstants.METADATA; import static com.appsmith.git.constants.CommonConstants.SERVER_SCHEMA_VERSION; +import static com.appsmith.git.constants.CommonConstants.TEXT_FILE_EXTENSION; import static com.appsmith.git.constants.CommonConstants.THEME; +import static com.appsmith.git.constants.ce.GitDirectoriesCE.ACTION_COLLECTION_DIRECTORY; +import static com.appsmith.git.constants.ce.GitDirectoriesCE.ACTION_DIRECTORY; +import static com.appsmith.git.constants.ce.GitDirectoriesCE.DATASOURCE_DIRECTORY; +import static com.appsmith.git.constants.ce.GitDirectoriesCE.JS_LIB_DIRECTORY; +import static com.appsmith.git.constants.ce.GitDirectoriesCE.PAGE_DIRECTORY; import static com.appsmith.git.files.FileUtilsCEImpl.getJsLibFileName; import static org.springframework.util.StringUtils.hasText; @Slf4j -@RequiredArgsConstructor @Component @Import({FileUtilsImpl.class}) public class CommonGitFileUtilsCE { @@ -83,6 +99,28 @@ public class CommonGitFileUtilsCE { public final int INDEX_LOCK_FILE_STALE_TIME = 300; private final JsonSchemaVersions jsonSchemaVersions; + protected final ObjectMapper objectMapper; + + public CommonGitFileUtilsCE( + ArtifactGitFileUtils applicationGitFileUtils, + FileInterface fileUtils, + FileOperations fileOperations, + AnalyticsService analyticsService, + SessionUserService sessionUserService, + NewActionService newActionService, + ActionCollectionService actionCollectionService, + JsonSchemaVersions jsonSchemaVersions, + ObjectMapper objectMapper) { + this.applicationGitFileUtils = applicationGitFileUtils; + this.fileUtils = fileUtils; + this.fileOperations = fileOperations; + this.analyticsService = analyticsService; + this.sessionUserService = sessionUserService; + this.newActionService = newActionService; + this.actionCollectionService = actionCollectionService; + this.jsonSchemaVersions = jsonSchemaVersions; + this.objectMapper = objectMapper.copy().disable(MapperFeature.USE_ANNOTATIONS); + } private ArtifactGitFileUtils getArtifactBasedFileHelper(ArtifactType artifactType) { if (ArtifactType.APPLICATION.equals(artifactType)) { @@ -120,6 +158,19 @@ public class CommonGitFileUtilsCE { } } + public Mono saveArtifactToLocalRepoNew( + Path baseRepoSuffix, ArtifactExchangeJson artifactExchangeJson, String branchName) + throws IOException, GitAPIException { + + // this should come from the specific files + GitResourceMap gitResourceMap = createGitResourceMap(artifactExchangeJson); + + // Save application to git repo + return fileUtils + .saveArtifactToGitRepo(baseRepoSuffix, gitResourceMap, branchName) + .subscribeOn(Schedulers.boundedElastic()); + } + public Mono saveArtifactToLocalRepoWithAnalytics( Path baseRepoSuffix, ArtifactExchangeJson artifactExchangeJson, String branchName) { @@ -216,8 +267,9 @@ public class CommonGitFileUtilsCE { if (datasourceList != null) { datasourceList.forEach(datasource -> { removeUnwantedFieldsFromDatasource(datasource); + final String filePath = DATASOURCE_DIRECTORY + DELIMITER_PATH + datasource.getName() + JSON_EXTENSION; GitResourceIdentity identity = - new GitResourceIdentity(GitResourceType.DATASOURCE_CONFIG, datasource.getGitSyncId()); + new GitResourceIdentity(GitResourceType.DATASOURCE_CONFIG, datasource.getGitSyncId(), filePath); resourceMap.put(identity, datasource); }); } @@ -230,7 +282,8 @@ public class CommonGitFileUtilsCE { artifactExchangeJson.setThemes(theme, null); // Remove internal fields from the themes removeUnwantedFieldsFromBaseDomain(theme); - GitResourceIdentity identity = new GitResourceIdentity(GitResourceType.ROOT_CONFIG, THEME + JSON_EXTENSION); + final String filePath = THEME + JSON_EXTENSION; + GitResourceIdentity identity = new GitResourceIdentity(GitResourceType.ROOT_CONFIG, filePath, filePath); resourceMap.put(identity, theme); } @@ -240,7 +293,9 @@ public class CommonGitFileUtilsCE { customJSLibList.forEach(jsLib -> { removeUnwantedFieldsFromBaseDomain(jsLib); String jsLibFileName = getJsLibFileName(jsLib.getUidString()); - GitResourceIdentity identity = new GitResourceIdentity(GitResourceType.JSLIB_CONFIG, jsLibFileName); + final String filePath = JS_LIB_DIRECTORY + DELIMITER_PATH + jsLibFileName + JSON_EXTENSION; + GitResourceIdentity identity = + new GitResourceIdentity(GitResourceType.JSLIB_CONFIG, jsLibFileName, filePath); resourceMap.put(identity, jsLib); }); } @@ -265,58 +320,47 @@ public class CommonGitFileUtilsCE { .peek(newAction -> newActionService.generateActionByViewMode(newAction, false)) .forEach(newAction -> { removeUnwantedFieldFromAction(newAction); - String body = newAction.getUnpublishedAction().getActionConfiguration() != null - && newAction - .getUnpublishedAction() - .getActionConfiguration() - .getBody() - != null - ? newAction - .getUnpublishedAction() - .getActionConfiguration() - .getBody() - : ""; + ActionDTO action = newAction.getUnpublishedAction(); + final String actionFileName = action.getValidName().replace(".", "-"); + final String filePathPrefix = PAGE_DIRECTORY + + DELIMITER_PATH + + action.calculateContextId() + + DELIMITER_PATH + + ACTION_DIRECTORY + + DELIMITER_PATH + + actionFileName + + DELIMITER_PATH; + String body = action.getActionConfiguration() != null + && action.getActionConfiguration().getBody() != null + ? action.getActionConfiguration().getBody() + : null; // This is a special case where we are handling REMOTE type plugins based actions such as Twilio // The user configured values are stored in an attribute called formData which is a map unlike the // body if (PluginType.REMOTE.equals(newAction.getPluginType()) - && newAction.getUnpublishedAction().getActionConfiguration() != null - && newAction - .getUnpublishedAction() - .getActionConfiguration() - .getFormData() - != null) { - body = new Gson() - .toJson( - newAction - .getUnpublishedAction() - .getActionConfiguration() - .getFormData(), - Map.class); - newAction - .getUnpublishedAction() - .getActionConfiguration() - .setFormData(null); + && action.getActionConfiguration() != null + && action.getActionConfiguration().getFormData() != null) { + body = new Gson().toJson(action.getActionConfiguration().getFormData(), Map.class); + action.getActionConfiguration().setFormData(null); } // This is a special case where we are handling JS actions as we don't want to commit the body of JS // actions if (PluginType.JS.equals(newAction.getPluginType())) { - if (newAction.getUnpublishedAction().getActionConfiguration() != null) { - newAction - .getUnpublishedAction() - .getActionConfiguration() - .setBody(null); - newAction.getUnpublishedAction().setJsonPathKeys(null); + if (action.getActionConfiguration() != null) { + action.getActionConfiguration().setBody(null); + action.setJsonPathKeys(null); } - } else { + } else if (body != null) { // For the regular actions we save the body field to git repo + final String filePath = filePathPrefix + actionFileName + TEXT_FILE_EXTENSION; GitResourceIdentity actionDataIdentity = - new GitResourceIdentity(GitResourceType.QUERY_DATA, newAction.getGitSyncId()); + new GitResourceIdentity(GitResourceType.QUERY_DATA, newAction.getGitSyncId(), filePath); resourceMap.put(actionDataIdentity, body); } + final String filePath = filePathPrefix + METADATA + JSON_EXTENSION; GitResourceIdentity actionConfigIdentity = - new GitResourceIdentity(GitResourceType.QUERY_CONFIG, newAction.getGitSyncId()); + new GitResourceIdentity(GitResourceType.QUERY_CONFIG, newAction.getGitSyncId(), filePath); resourceMap.put(actionConfigIdentity, newAction); }); } @@ -335,18 +379,29 @@ public class CommonGitFileUtilsCE { actionCollectionService.generateActionCollectionByViewMode(actionCollection, false)) .forEach(actionCollection -> { removeUnwantedFieldFromActionCollection(actionCollection); - String body = actionCollection.getUnpublishedCollection().getBody() != null - ? actionCollection.getUnpublishedCollection().getBody() - : ""; - actionCollection.getUnpublishedCollection().setBody(null); + ActionCollectionDTO collection = actionCollection.getUnpublishedCollection(); + final String filePathPrefix = PAGE_DIRECTORY + + DELIMITER_PATH + + collection.calculateContextId() + + DELIMITER_PATH + + ACTION_COLLECTION_DIRECTORY + + DELIMITER_PATH + + collection.getName() + + DELIMITER_PATH; + String body = collection.getBody(); + collection.setBody(null); - GitResourceIdentity collectionConfigIdentity = - new GitResourceIdentity(GitResourceType.JSOBJECT_CONFIG, actionCollection.getGitSyncId()); + String configFilePath = filePathPrefix + METADATA + JSON_EXTENSION; + GitResourceIdentity collectionConfigIdentity = new GitResourceIdentity( + GitResourceType.JSOBJECT_CONFIG, actionCollection.getGitSyncId(), configFilePath); resourceMap.put(collectionConfigIdentity, actionCollection); - GitResourceIdentity collectionDataIdentity = - new GitResourceIdentity(GitResourceType.JSOBJECT_DATA, actionCollection.getGitSyncId()); - resourceMap.put(collectionDataIdentity, body); + if (body != null) { + String dataFilePath = filePathPrefix + collection.getName() + JS_EXTENSION; + GitResourceIdentity collectionDataIdentity = new GitResourceIdentity( + GitResourceType.JSOBJECT_DATA, actionCollection.getGitSyncId(), dataFilePath); + resourceMap.put(collectionDataIdentity, body); + } }); } @@ -366,6 +421,125 @@ public class CommonGitFileUtilsCE { removeUnwantedFieldsFromBaseDomain(actionCollection); } + public ArtifactExchangeJson createArtifactExchangeJson(GitResourceMap gitResourceMap, ArtifactType artifactType) { + ArtifactGitFileUtils artifactGitFileUtils = getArtifactBasedFileHelper(artifactType); + + ArtifactExchangeJson artifactExchangeJson = artifactGitFileUtils.createArtifactExchangeJsonObject(); + + artifactGitFileUtils.setArtifactDependentPropertiesInJson(gitResourceMap, artifactExchangeJson); + + setArtifactIndependentPropertiesInJson(gitResourceMap, artifactExchangeJson); + + return artifactExchangeJson; + } + + protected void setArtifactIndependentPropertiesInJson( + GitResourceMap gitResourceMap, ArtifactExchangeJson artifactExchangeJson) { + Map resourceMap = gitResourceMap.getGitResourceMap(); + + // datasources + List datasourceList = resourceMap.entrySet().stream() + .filter(entry -> { + GitResourceIdentity key = entry.getKey(); + return GitResourceType.DATASOURCE_CONFIG.equals(key.getResourceType()); + }) + .map(Map.Entry::getValue) + .map(value -> objectMapper.convertValue(value, DatasourceStorage.class)) + .collect(Collectors.toList()); + artifactExchangeJson.setDatasourceList(datasourceList); + + // themes + final String themeFilePath = THEME + JSON_EXTENSION; + GitResourceIdentity themeIdentity = + new GitResourceIdentity(GitResourceType.ROOT_CONFIG, themeFilePath, themeFilePath); + Object themeObject = resourceMap.get(themeIdentity); + Theme theme = objectMapper.convertValue(themeObject, Theme.class); + artifactExchangeJson.setThemes(theme, null); + + // custom js libs + List jsLibList = resourceMap.entrySet().stream() + .filter(entry -> { + GitResourceIdentity key = entry.getKey(); + return GitResourceType.JSLIB_CONFIG.equals(key.getResourceType()); + }) + .map(Map.Entry::getValue) + .map(value -> objectMapper.convertValue(value, CustomJSLib.class)) + .collect(Collectors.toList()); + artifactExchangeJson.setCustomJSLibList(jsLibList); + + // actions + final Set queryTypes = Set.of(GitResourceType.QUERY_CONFIG, GitResourceType.QUERY_DATA); + List actionList = resourceMap.entrySet().stream() + .filter(entry -> { + GitResourceIdentity key = entry.getKey(); + return queryTypes.contains(key.getResourceType()); + }) + .collect(collectByGitSyncId()) + .entrySet() + .parallelStream() + .map(entry -> { + Object config = entry.getValue().get(GitResourceType.QUERY_CONFIG); + NewAction newAction = objectMapper.convertValue(config, NewAction.class); + ActionDTO actionDTO = newAction.getUnpublishedAction(); + Object data = entry.getValue().get(GitResourceType.QUERY_DATA); + ActionConfiguration actionConfiguration = actionDTO.getActionConfiguration(); + if (actionConfiguration == null) { + // This shouldn't happen but safe-guarding just in case + actionConfiguration = new ActionConfiguration(); + } + + if (PluginType.REMOTE.equals(newAction.getPluginType())) { + Map formData = objectMapper.convertValue(data, new TypeReference<>() {}); + actionConfiguration.setFormData(formData); + } else if (data != null) { + String body = String.valueOf(data); + actionConfiguration.setBody(body); + } + + return newAction; + }) + .collect(Collectors.toList()); + artifactExchangeJson.setActionList(actionList); + + // action collections + final Set jsObjectTypes = + Set.of(GitResourceType.JSOBJECT_CONFIG, GitResourceType.JSOBJECT_DATA); + List collectionList = resourceMap.entrySet().stream() + .filter(entry -> { + GitResourceIdentity key = entry.getKey(); + return jsObjectTypes.contains(key.getResourceType()); + }) + .collect(collectByGitSyncId()) + .entrySet() + .parallelStream() + .map(entry -> { + Object config = entry.getValue().get(GitResourceType.JSOBJECT_CONFIG); + ActionCollection actionCollection = objectMapper.convertValue(config, ActionCollection.class); + Object data = entry.getValue().get(GitResourceType.JSOBJECT_DATA); + String body = String.valueOf(data); + actionCollection.getUnpublishedCollection().setBody(body); + + return actionCollection; + }) + .collect(Collectors.toList()); + artifactExchangeJson.setActionCollectionList(collectionList); + } + + private Collector, ?, Map>> + collectByGitSyncId() { + return Collectors.toMap( + entry -> entry.getKey().getResourceIdentifier(), + entry -> { + HashMap map = new HashMap<>(); + map.put(entry.getKey().getResourceType(), entry.getValue()); + return map; + }, + (x, y) -> { + x.putAll(y); + return x; + }); + } + private void setDatasourcesInArtifactReference( ArtifactExchangeJson artifactExchangeJson, ArtifactGitReference artifactGitReference) { Map resourceMap = new HashMap<>(); diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/jslibs/exportable/CustomJSLibExportableServiceCEImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/jslibs/exportable/CustomJSLibExportableServiceCEImpl.java index dde27edae1..8df63d5138 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/jslibs/exportable/CustomJSLibExportableServiceCEImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/jslibs/exportable/CustomJSLibExportableServiceCEImpl.java @@ -1,5 +1,6 @@ package com.appsmith.server.jslibs.exportable; +import com.appsmith.external.git.models.GitResourceType; import com.appsmith.server.constants.FieldName; import com.appsmith.server.domains.Application; import com.appsmith.server.domains.Artifact; @@ -77,6 +78,11 @@ public class CustomJSLibExportableServiceCEImpl implements ExportableServiceCE actionList = tuple.getT1(); Set dbNamesUsedInActions = tuple.getT2(); Set updatedActionSet = new HashSet<>(); + Set updatedIdentities = new HashSet<>(); actionList.forEach(newAction -> { ActionDTO unpublishedActionDTO = newAction.getUnpublishedAction(); ActionDTO publishedActionDTO = newAction.getPublishedAction(); @@ -85,9 +87,12 @@ public class NewActionExportableServiceCEImpl implements ExportableServiceCE updatedPageSet = new HashSet(); + Set updatedPageSet = new HashSet<>(); + Set updatedIdentities = new HashSet<>(); // check the application object for the page reference in the page list // Exclude the deleted pages that are present in view mode because the app is not @@ -79,6 +81,9 @@ public class NewPageExportableServiceCEImpl implements ExportableServiceCE { @@ -114,11 +119,17 @@ public class NewPageExportableServiceCEImpl implements ExportableServiceCE artifactJsonMono = - createArtifactJson(context).cache(); + Mono artifactJsonMono = createArtifactJson(context); + + Mono artifactJsonCloneMono = createArtifactJson(context); Mono> gitResourceMapAndArtifactJsonMono = artifactJsonMono .map(artifactJson -> commonGitFileUtils.createGitResourceMap(artifactJson)) - .zipWith(artifactJsonMono); + .zipWith(artifactJsonCloneMono); StepVerifier.create(gitResourceMapAndArtifactJsonMono) .assertNext(tuple2 -> { @@ -87,8 +101,46 @@ public class ExchangeJsonConversionTests { Class exchangeJsonType = context.getArtifactExchangeJsonType(); - ArtifactExchangeJson artifactExchangeJson = gson.fromJson(artifactJson, exchangeJsonType); + ArtifactExchangeJson artifactExchangeJson = + objectMapper.copy().disable(MapperFeature.USE_ANNOTATIONS).readValue(artifactJson, exchangeJsonType); return jsonSchemaMigration.migrateArtifactExchangeJsonToLatestSchema(artifactExchangeJson, null, null); } + + @TestTemplate + public void testConvertGitResourceMapToArtifactExchangeJson_whenArtifactIsFullyPopulated_returnsCorrespondingJson( + ExchangeJsonContext context) throws IOException { + ArtifactExchangeJson originalArtifactJson = createArtifactJson(context).block(); + + GitResourceMap gitResourceMap = commonGitFileUtils.createGitResourceMap(originalArtifactJson); + + ArtifactExchangeJson artifactExchangeJson = + commonGitFileUtils.createArtifactExchangeJson(gitResourceMap, ArtifactType.APPLICATION); + + assertThat(artifactExchangeJson).isNotNull(); + + templateProvider.assertResourceComparisons(originalArtifactJson, artifactExchangeJson); + } + + @TestTemplate + public void testSerializeArtifactExchangeJson_whenArtifactIsFullyPopulated_returnsCorrespondingBaseRepoPath( + ExchangeJsonContext context) throws IOException, GitAPIException { + ArtifactExchangeJson originalArtifactJson = createArtifactJson(context).block(); + + Mockito.doReturn(Mono.just(true)).when(gitExecutor).resetToLastCommit(Mockito.any(), Mockito.anyString()); + + Files.createDirectories(Path.of("./container-volumes/git-storage/test123")); + + Mono responseMono = + commonGitFileUtils.saveArtifactToLocalRepoNew(Path.of("test123"), originalArtifactJson, "irrelevant"); + + StepVerifier.create(responseMono) + .assertNext(response -> { + Assertions.assertThat(response).isNotNull(); + }) + .verifyComplete(); + + FileUtils.deleteDirectory( + Path.of("./container-volumes/git-storage/test123").toFile()); + } } diff --git a/app/server/appsmith-server/src/test/java/com/appsmith/server/git/resourcemap/templates/providers/ce/ExchangeJsonTestTemplateProviderCE.java b/app/server/appsmith-server/src/test/java/com/appsmith/server/git/resourcemap/templates/providers/ce/ExchangeJsonTestTemplateProviderCE.java index 0724d60170..bbf07f5f62 100644 --- a/app/server/appsmith-server/src/test/java/com/appsmith/server/git/resourcemap/templates/providers/ce/ExchangeJsonTestTemplateProviderCE.java +++ b/app/server/appsmith-server/src/test/java/com/appsmith/server/git/resourcemap/templates/providers/ce/ExchangeJsonTestTemplateProviderCE.java @@ -2,7 +2,12 @@ package com.appsmith.server.git.resourcemap.templates.providers.ce; import com.appsmith.external.git.models.GitResourceIdentity; import com.appsmith.external.git.models.GitResourceType; +import com.appsmith.external.models.DatasourceStorage; import com.appsmith.external.models.PluginType; +import com.appsmith.server.domains.ActionCollection; +import com.appsmith.server.domains.Context; +import com.appsmith.server.domains.CustomJSLib; +import com.appsmith.server.domains.NewAction; import com.appsmith.server.dtos.ApplicationJson; import com.appsmith.server.dtos.ArtifactExchangeJson; import com.appsmith.server.git.resourcemap.templates.contexts.ExchangeJsonContext; @@ -27,7 +32,7 @@ public class ExchangeJsonTestTemplateProviderCE implements TestTemplateInvocatio @Override public Stream provideTestTemplateInvocationContexts( ExtensionContext extensionContext) { - ExchangeJsonContext context = new ExchangeJsonContext("valid-application.json", ApplicationJson.class, 23); + ExchangeJsonContext context = new ExchangeJsonContext("valid-application.json", ApplicationJson.class, 22); return Stream.of(context); } @@ -68,7 +73,14 @@ public class ExchangeJsonTestTemplateProviderCE implements TestTemplateInvocatio List jsObjectDataResources = getResourceListByType(resourceMap, GitResourceType.JSOBJECT_DATA); long resourceMapJsObjectDataCount = jsObjectDataResources.size(); - assertThat(resourceMapJsObjectDataCount).isEqualTo(jsonJsObjectCount); + int jsonJsObjectDataCount = exchangeJson.getActionCollectionList() != null + ? exchangeJson.getActionCollectionList().parallelStream() + .filter(collection -> + collection.getUnpublishedCollection().getBody() != null) + .collect(Collectors.toList()) + .size() + : 0; + assertThat(resourceMapJsObjectDataCount).isEqualTo(jsonJsObjectDataCount); List actionConfigResources = getResourceListByType(resourceMap, GitResourceType.QUERY_CONFIG); long resourceMapActionConfigCount = actionConfigResources.size(); @@ -82,7 +94,17 @@ public class ExchangeJsonTestTemplateProviderCE implements TestTemplateInvocatio long jsonActionDataCount = 0; if (exchangeJson.getActionList() != null) { jsonActionDataCount = exchangeJson.getActionList().stream() - .filter(action -> !PluginType.JS.equals(action.getPluginType())) + .filter(action -> !PluginType.JS.equals(action.getPluginType()) + && action.getUnpublishedAction().getActionConfiguration() != null + && !(action.getUnpublishedAction() + .getActionConfiguration() + .getBody() + == null + || (action.getPluginType().equals(PluginType.REMOTE) + && action.getUnpublishedAction() + .getActionConfiguration() + .getFormData() + == null))) .count(); } assertThat(resourceMapActionDataCount).isEqualTo(jsonActionDataCount); @@ -111,4 +133,42 @@ public class ExchangeJsonTestTemplateProviderCE implements TestTemplateInvocatio .map(Map.Entry::getValue) .collect(Collectors.toList()); } + + public void assertResourceComparisons( + ArtifactExchangeJson originalExchangeJson, ArtifactExchangeJson convertedExchangeJson) { + List datasourceResources = convertedExchangeJson.getDatasourceList(); + long convertedDatasourceCount = datasourceResources.size(); + int jsonDatasourceCount = originalExchangeJson.getDatasourceList() != null + ? originalExchangeJson.getDatasourceList().size() + : 0; + assertThat(convertedDatasourceCount).isEqualTo(jsonDatasourceCount); + + List jsLibResources = convertedExchangeJson.getCustomJSLibList(); + long convertedJsLibCount = jsLibResources.size(); + int jsonJsLibCount = originalExchangeJson.getCustomJSLibList() != null + ? originalExchangeJson.getCustomJSLibList().size() + : 0; + assertThat(convertedJsLibCount).isEqualTo(jsonJsLibCount); + + List contextResources = convertedExchangeJson.getContextList(); + long convertedContextCount = contextResources.size(); + int jsonContextCount = originalExchangeJson.getContextList() != null + ? originalExchangeJson.getContextList().size() + : 0; + assertThat(convertedContextCount).isEqualTo(jsonContextCount); + + List jsObjectResources = convertedExchangeJson.getActionCollectionList(); + long convertedJsObjectCount = jsObjectResources.size(); + int jsonJsObjectCount = originalExchangeJson.getActionCollectionList() != null + ? originalExchangeJson.getActionCollectionList().size() + : 0; + assertThat(convertedJsObjectCount).isEqualTo(jsonJsObjectCount); + + List actionResources = convertedExchangeJson.getActionList(); + long convertedActionCount = actionResources.size(); + int jsonActionCount = originalExchangeJson.getActionList() != null + ? originalExchangeJson.getActionList().size() + : 0; + assertThat(convertedActionCount).isEqualTo(jsonActionCount); + } }