diff --git a/app/server/appsmith-git/src/main/java/com/appsmith/git/constants/GitDirectories.java b/app/server/appsmith-git/src/main/java/com/appsmith/git/constants/GitDirectories.java index db5c21bb98..3d29863957 100644 --- a/app/server/appsmith-git/src/main/java/com/appsmith/git/constants/GitDirectories.java +++ b/app/server/appsmith-git/src/main/java/com/appsmith/git/constants/GitDirectories.java @@ -1,9 +1,5 @@ package com.appsmith.git.constants; -public interface GitDirectories { - String PAGE_DIRECTORY = "pages"; - String ACTION_DIRECTORY = "queries"; - String ACTION_COLLECTION_DIRECTORY = "jsobjects"; - String DATASOURCE_DIRECTORY = "datasources"; - String JS_LIB_DIRECTORY = "jslibs"; -} +import com.appsmith.git.constants.ce.GitDirectoriesCE; + +public interface GitDirectories extends GitDirectoriesCE {} diff --git a/app/server/appsmith-git/src/main/java/com/appsmith/git/constants/ce/GitDirectoriesCE.java b/app/server/appsmith-git/src/main/java/com/appsmith/git/constants/ce/GitDirectoriesCE.java new file mode 100644 index 0000000000..24da45a7f0 --- /dev/null +++ b/app/server/appsmith-git/src/main/java/com/appsmith/git/constants/ce/GitDirectoriesCE.java @@ -0,0 +1,9 @@ +package com.appsmith.git.constants.ce; + +public interface GitDirectoriesCE { + String PAGE_DIRECTORY = "pages"; + String ACTION_DIRECTORY = "queries"; + String ACTION_COLLECTION_DIRECTORY = "jsobjects"; + String DATASOURCE_DIRECTORY = "datasources"; + String JS_LIB_DIRECTORY = "jslibs"; +} diff --git a/app/server/appsmith-git/src/main/java/com/appsmith/git/helpers/FileUtilsImpl.java b/app/server/appsmith-git/src/main/java/com/appsmith/git/helpers/FileUtilsImpl.java index fe03432474..97285327a7 100644 --- a/app/server/appsmith-git/src/main/java/com/appsmith/git/helpers/FileUtilsImpl.java +++ b/app/server/appsmith-git/src/main/java/com/appsmith/git/helpers/FileUtilsImpl.java @@ -1,1147 +1,21 @@ package com.appsmith.git.helpers; -import com.appsmith.external.converters.ISOStringToInstantConverter; -import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginError; -import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginException; import com.appsmith.external.git.FileInterface; import com.appsmith.external.git.GitExecutor; -import com.appsmith.external.helpers.Stopwatch; -import com.appsmith.external.models.ApplicationGitReference; -import com.appsmith.external.models.DatasourceStructure; import com.appsmith.git.configurations.GitServiceConfig; -import com.appsmith.git.constants.CommonConstants; -import com.appsmith.git.converters.GsonDoubleToLongConverter; -import com.appsmith.git.converters.GsonUnorderedToOrderedConverter; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.google.gson.stream.JsonReader; +import com.appsmith.git.helpers.ce.FileUtilsCEImpl; import lombok.Getter; -import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.IOUtils; -import org.eclipse.jgit.api.errors.GitAPIException; -import org.json.JSONArray; -import org.json.JSONObject; import org.springframework.context.annotation.Import; import org.springframework.stereotype.Component; -import org.springframework.util.CollectionUtils; -import org.springframework.util.FileSystemUtils; -import org.springframework.util.StringUtils; -import reactor.core.publisher.Mono; -import reactor.core.scheduler.Scheduler; -import reactor.core.scheduler.Schedulers; - -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.StringWriter; -import java.nio.charset.StandardCharsets; -import java.nio.file.DirectoryNotEmptyException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.attribute.BasicFileAttributes; -import java.nio.file.attribute.FileTime; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.time.Instant; -import java.util.Arrays; -import java.util.Base64; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.regex.Pattern; -import java.util.stream.Stream; - -import static com.appsmith.external.constants.GitConstants.ACTION_COLLECTION_LIST; -import static com.appsmith.external.constants.GitConstants.ACTION_LIST; -import static com.appsmith.external.constants.GitConstants.CUSTOM_JS_LIB_LIST; -import static com.appsmith.external.constants.GitConstants.NAME_SEPARATOR; -import static com.appsmith.external.constants.GitConstants.PAGE_LIST; -import static com.appsmith.git.constants.GitDirectories.ACTION_COLLECTION_DIRECTORY; -import static com.appsmith.git.constants.GitDirectories.ACTION_DIRECTORY; -import static com.appsmith.git.constants.GitDirectories.DATASOURCE_DIRECTORY; -import static com.appsmith.git.constants.GitDirectories.JS_LIB_DIRECTORY; -import static com.appsmith.git.constants.GitDirectories.PAGE_DIRECTORY; @Slf4j @Getter -@RequiredArgsConstructor @Component @Import({GitServiceConfig.class}) -public class FileUtilsImpl implements FileInterface { +public class FileUtilsImpl extends FileUtilsCEImpl implements FileInterface { - private final GitServiceConfig gitServiceConfig; - - private final GitExecutor gitExecutor; - - private static final String EDIT_MODE_URL_TEMPLATE = "{{editModeUrl}}"; - - private static final String VIEW_MODE_URL_TEMPLATE = "{{viewModeUrl}}"; - - private static final Pattern ALLOWED_FILE_EXTENSION_PATTERN = - Pattern.compile("(.*?)\\.(md|MD|git|gitignore|github|yml|yaml)$"); - - private final Scheduler scheduler = Schedulers.boundedElastic(); - - private static final String CANVAS_WIDGET = "(Canvas)[0-9]*."; - - /** - * Application will be stored in the following structure: - * - * For v1: - * repo_name - * application.json - * metadata.json - * datasource - * datasource1Name.json - * datasource2Name.json - * queries (Only requirement here is the filename should be unique) - * action1_page1 - * action2_page2 - * jsobjects (Only requirement here is the filename should be unique) - * jsobject1_page1 - * jsobject2_page2 - * pages - * page1 - * page2 - * - * For v2: - * repo_name - * application.json - * metadata.json - * theme - * publishedTheme.json - * editModeTheme.json - * pages - * page1 - * canvas.json - * queries - * Query1.json - * Query2.json - * jsobjects - * JSObject1.json - * page2 - * page3 - * datasources - * datasource1.json - * datasource2.json - * - * For v3: - * repo_name - * application.json - * metadata.json - * theme - * publishedTheme.json - * editModeTheme.json - * pages - * page1 - * canvas.json - * queries - * Query1.json - * jsobjects - * JSObject1 - * JSObject1.js - * Metadata.json - * page2 - * page3 - * datasources - * datasource1.json - * datasource2.json - * - * For v4: - * repo_name - * application.json - * metadata.json - * theme - * publishedTheme.json - * editModeTheme.json - * pages - * page1 - * canvas.json - * queries - * Query1.json - * jsobjects - * JSObject1 - * JSObject1.js - * Metadata.json - * page2 - * page3 - * datasources - * datasource1.json - * datasource2.json - */ - - /** - * This method will save the complete application in the local repo directory. - * Path to repo will be : ./container-volumes/git-repo/workspaceId/defaultApplicationId/repoName/{application_data} - * @param baseRepoSuffix path suffix used to create a repo path - * @param applicationGitReference application reference object from which entire application can be rehydrated - * @param branchName name of the branch for the current application - * @return repo path where the application is stored - */ - public Mono saveApplicationToGitRepo( - Path baseRepoSuffix, ApplicationGitReference applicationGitReference, String branchName) - throws GitAPIException, IOException { - - // Repo path will be: - // baseRepo : root/orgId/defaultAppId/repoName/{applicationData} - // Checkout to mentioned branch if not already checked-out - Stopwatch processStopwatch = new Stopwatch("FS application save"); - return gitExecutor - .resetToLastCommit(baseRepoSuffix, branchName) - .flatMap(isSwitched -> { - Path baseRepo = Paths.get(gitServiceConfig.getGitRootPath()).resolve(baseRepoSuffix); - - // Gson to pretty format JSON file - // Keep Long type as is by default GSON have behavior to convert to Double - // Convert unordered set to ordered one - Gson gson = new GsonBuilder() - .registerTypeAdapter(Double.class, new GsonDoubleToLongConverter()) - .registerTypeAdapter(Set.class, new GsonUnorderedToOrderedConverter()) - .registerTypeAdapter(Map.class, new GsonUnorderedToOrderedConverter()) - .registerTypeAdapter(Instant.class, new ISOStringToInstantConverter()) - .disableHtmlEscaping() - .setPrettyPrinting() - .create(); - - Set validFileNames = new HashSet<>(); - Map> updatedResources = applicationGitReference.getUpdatedResources(); - - // Remove unwanted directories which was present in v1 of the git file format version - deleteDirectory(baseRepo.resolve(ACTION_DIRECTORY)); - deleteDirectory(baseRepo.resolve(ACTION_COLLECTION_DIRECTORY)); - - // Save application - saveResource( - applicationGitReference.getApplication(), - baseRepo.resolve(CommonConstants.APPLICATION + CommonConstants.JSON_EXTENSION), - gson); - - // Save application metadata - JsonObject metadata = - gson.fromJson(gson.toJson(applicationGitReference.getMetadata()), JsonObject.class); - metadata.addProperty(CommonConstants.FILE_FORMAT_VERSION, CommonConstants.fileFormatVersion); - saveResource( - metadata, - baseRepo.resolve(CommonConstants.METADATA + CommonConstants.JSON_EXTENSION), - gson); - - // Save application theme - saveResource( - applicationGitReference.getTheme(), - baseRepo.resolve(CommonConstants.THEME + CommonConstants.JSON_EXTENSION), - gson); - - // Save pages - Path pageDirectory = baseRepo.resolve(PAGE_DIRECTORY); - Set> pageEntries = - applicationGitReference.getPages().entrySet(); - - Set validPages = new HashSet<>(); - for (Map.Entry pageResource : pageEntries) { - Map validWidgetToParentMap = new HashMap<>(); - final String pageName = pageResource.getKey(); - Path pageSpecificDirectory = pageDirectory.resolve(pageName); - Boolean isResourceUpdated = !CollectionUtils.isEmpty(updatedResources.get(PAGE_LIST)) - ? updatedResources.get(PAGE_LIST).contains(pageName) - : Boolean.FALSE; - if (Boolean.TRUE.equals(isResourceUpdated)) { - // Save page metadata - saveResource( - pageResource.getValue(), - pageSpecificDirectory.resolve(pageName + CommonConstants.JSON_EXTENSION), - gson); - Map result = DSLTransformerHelper.flatten(new JSONObject( - applicationGitReference.getPageDsl().get(pageName))); - result.forEach((key, jsonObject) -> { - // get path with splitting the name via key - String widgetName = key.substring(key.lastIndexOf(CommonConstants.DELIMITER_POINT) + 1); - String childPath = key.replace( - CommonConstants.MAIN_CONTAINER, CommonConstants.EMPTY_STRING) - .replace(CommonConstants.DELIMITER_POINT, CommonConstants.DELIMITER_PATH); - // Replace the canvas Widget as a child and add it to the same level as parent - childPath = childPath.replaceAll(CANVAS_WIDGET, CommonConstants.EMPTY_STRING); - if (!DSLTransformerHelper.hasChildren(jsonObject) - && !DSLTransformerHelper.isTabsWidget(jsonObject)) { - // Save the widget as a directory or Save the widget as a file - childPath = childPath.replace(widgetName, CommonConstants.EMPTY_STRING); - } - Path path = Paths.get( - String.valueOf(pageSpecificDirectory.resolve(CommonConstants.WIDGETS)), - childPath); - validWidgetToParentMap.put( - widgetName, path.toFile().toString()); - saveWidgets(jsonObject, widgetName, path); - }); - // Remove deleted widgets from the file system - deleteWidgets( - pageSpecificDirectory - .resolve(CommonConstants.WIDGETS) - .toFile(), - validWidgetToParentMap); - - // Remove the canvas.json from the file system since the value is stored in the page.json - deleteFile(pageSpecificDirectory.resolve( - CommonConstants.CANVAS + CommonConstants.JSON_EXTENSION)); - } - validPages.add(pageName); - } - scanAndDeleteDirectoryForDeletedResources(validPages, baseRepo.resolve(PAGE_DIRECTORY)); - - // Save JS Libs if there's at least one change - if (updatedResources != null - && !CollectionUtils.isEmpty(updatedResources.get(CUSTOM_JS_LIB_LIST))) { - Path jsLibDirectory = baseRepo.resolve(JS_LIB_DIRECTORY); - Set> jsLibEntries = - applicationGitReference.getJsLibraries().entrySet(); - Set validJsLibs = new HashSet<>(); - jsLibEntries.forEach(jsLibEntry -> { - String uidString = jsLibEntry.getKey(); - boolean isResourceUpdated = - updatedResources.get(CUSTOM_JS_LIB_LIST).contains(uidString); - - String fileNameWithExtension = getJsLibFileName(uidString) + CommonConstants.JSON_EXTENSION; - - Path jsLibSpecificFile = jsLibDirectory.resolve(fileNameWithExtension); - if (isResourceUpdated) { - saveResource(jsLibEntry.getValue(), jsLibSpecificFile, gson); - } - validJsLibs.add(fileNameWithExtension); - }); - scanAndDeleteFileForDeletedResources(validJsLibs, jsLibDirectory); - } - - // Create HashMap for valid actions and actionCollections - HashMap> validActionsMap = new HashMap<>(); - HashMap> validActionCollectionsMap = new HashMap<>(); - validPages.forEach(validPage -> { - validActionsMap.put(validPage, new HashSet<>()); - validActionCollectionsMap.put(validPage, new HashSet<>()); - }); - - // Save actions - for (Map.Entry resource : - applicationGitReference.getActions().entrySet()) { - // queryName_pageName => nomenclature for the keys - // TODO - // queryName => for app level queries, this is not implemented yet - String[] names = resource.getKey().split(NAME_SEPARATOR); - if (names.length > 1 && StringUtils.hasLength(names[1])) { - // For actions, we are referring to validNames to maintain unique file names as just name - // field don't guarantee unique constraint for actions within JSObject - Boolean isResourceUpdated = !CollectionUtils.isEmpty(updatedResources.get(ACTION_LIST)) - ? updatedResources.get(ACTION_LIST).contains(resource.getKey()) - : Boolean.FALSE; - final String queryName = names[0].replace(".", "-"); - final String pageName = names[1]; - Path pageSpecificDirectory = pageDirectory.resolve(pageName); - Path actionSpecificDirectory = pageSpecificDirectory.resolve(ACTION_DIRECTORY); - - if (!validActionsMap.containsKey(pageName)) { - validActionsMap.put(pageName, new HashSet<>()); - } - validActionsMap.get(pageName).add(queryName); - if (Boolean.TRUE.equals(isResourceUpdated)) { - saveActions( - resource.getValue(), - applicationGitReference.getActionBody().containsKey(resource.getKey()) - ? applicationGitReference - .getActionBody() - .get(resource.getKey()) - : null, - queryName, - actionSpecificDirectory.resolve(queryName), - gson); - // Delete the resource from the old file structure v2 - deleteFile(pageSpecificDirectory - .resolve(ACTION_DIRECTORY) - .resolve(queryName + CommonConstants.JSON_EXTENSION)); - } - } - } - - validActionsMap.forEach((pageName, validActionNames) -> { - Path pageSpecificDirectory = pageDirectory.resolve(pageName); - scanAndDeleteDirectoryForDeletedResources( - validActionNames, pageSpecificDirectory.resolve(ACTION_DIRECTORY)); - }); - - // Save JSObjects - for (Map.Entry resource : - applicationGitReference.getActionCollections().entrySet()) { - // JSObjectName_pageName => nomenclature for the keys - // TODO JSObjectName => for app level JSObjects, this is not implemented yet - String[] names = resource.getKey().split(NAME_SEPARATOR); - if (names.length > 1 && StringUtils.hasLength(names[1])) { - final String actionCollectionName = names[0]; - final String pageName = names[1]; - Path pageSpecificDirectory = pageDirectory.resolve(pageName); - Path actionCollectionSpecificDirectory = - pageSpecificDirectory.resolve(ACTION_COLLECTION_DIRECTORY); - - if (!validActionCollectionsMap.containsKey(pageName)) { - validActionCollectionsMap.put(pageName, new HashSet<>()); - } - validActionCollectionsMap.get(pageName).add(actionCollectionName); - Boolean isResourceUpdated = - !CollectionUtils.isEmpty(updatedResources.get(ACTION_COLLECTION_LIST)) - ? updatedResources - .get(ACTION_COLLECTION_LIST) - .contains(resource.getKey()) - : Boolean.FALSE; - if (Boolean.TRUE.equals(isResourceUpdated)) { - saveActionCollection( - resource.getValue(), - applicationGitReference - .getActionCollectionBody() - .get(resource.getKey()), - actionCollectionName, - actionCollectionSpecificDirectory.resolve(actionCollectionName), - gson); - // Delete the resource from the old file structure v2 - deleteFile(actionCollectionSpecificDirectory.resolve( - actionCollectionName + CommonConstants.JSON_EXTENSION)); - } - } - } - - // Verify if the old files are deleted - validActionCollectionsMap.forEach((pageName, validActionCollectionNames) -> { - Path pageSpecificDirectory = pageDirectory.resolve(pageName); - scanAndDeleteDirectoryForDeletedResources( - validActionCollectionNames, pageSpecificDirectory.resolve(ACTION_COLLECTION_DIRECTORY)); - }); - - // Save datasources ref - for (Map.Entry resource : - applicationGitReference.getDatasources().entrySet()) { - saveResource( - resource.getValue(), - baseRepo.resolve(DATASOURCE_DIRECTORY) - .resolve(resource.getKey() + CommonConstants.JSON_EXTENSION), - gson); - validFileNames.add(resource.getKey() + CommonConstants.JSON_EXTENSION); - } - // Scan datasource directory and delete any unwanted files if present - if (!applicationGitReference.getDatasources().isEmpty()) { - scanAndDeleteFileForDeletedResources(validFileNames, baseRepo.resolve(DATASOURCE_DIRECTORY)); - } - processStopwatch.stopAndLogTimeInMillis(); - return Mono.just(baseRepo); - }) - .subscribeOn(scheduler); - } - - /** - * This method will be used to store the DB resource to JSON file - * @param sourceEntity resource extracted from DB to be stored in file - * @param path file path where the resource to be stored - * @param gson - * @return if the file operation is successful - */ - private boolean saveResource(Object sourceEntity, Path path, Gson gson) { - try { - Files.createDirectories(path.getParent()); - return writeToFile(sourceEntity, path, gson); - } catch (IOException e) { - log.error("Error while writing resource to file {} with {}", path, e.getMessage()); - log.debug(e.getMessage()); - } - return false; - } - - private void saveWidgets(JSONObject sourceEntity, String resourceName, Path path) { - try { - Files.createDirectories(path); - writeStringToFile(sourceEntity.toString(4), path.resolve(resourceName + CommonConstants.JSON_EXTENSION)); - } catch (IOException e) { - log.debug("Error while writings widgets data to file, {}", e.getMessage()); - } - } - - /** - * This method is used to write actionCollection specific resource to file system. We write the data in two steps - * 1. Actual js code - * 2. Metadata of the actionCollection - * @param sourceEntity the metadata of the action collection - * @param body actual js code written by the user - * @param resourceName name of the action collection - * @param path file path where the resource will be stored - * @param gson - * @return if the file operation is successful - */ - private boolean saveActionCollection(Object sourceEntity, String body, String resourceName, Path path, Gson gson) { - try { - Files.createDirectories(path); - // Write the js Object body to .js file to make conflict handling easier - Path bodyPath = path.resolve(resourceName + CommonConstants.JS_EXTENSION); - writeStringToFile(body, bodyPath); - - // Write metadata for the jsObject - Path metadataPath = path.resolve(CommonConstants.METADATA + CommonConstants.JSON_EXTENSION); - return writeToFile(sourceEntity, metadataPath, gson); - } catch (IOException e) { - log.debug(e.getMessage()); - } - return false; - } - - /** - * This method is used to write action specific resource to file system. We write the data in two steps - * * 1. Actual query written by the user - * * 2. Metadata of the actios - * @param sourceEntity the metadata of the action - * @param body actual query written by the user - * @param resourceName name of the action - * @param path file path where the resource will be stored - * @param gson - * @return if the file operation is successful - */ - private boolean saveActions(Object sourceEntity, String body, String resourceName, Path path, Gson gson) { - try { - Files.createDirectories(path); - // Write the user written query to .txt file to make conflict handling easier - // Body will be null if the action is of type JS - if (StringUtils.hasLength(body)) { - Path bodyPath = path.resolve(resourceName + CommonConstants.TEXT_FILE_EXTENSION); - writeStringToFile(body, bodyPath); - } - - // Write metadata for the actions - Path metadataPath = path.resolve(CommonConstants.METADATA + CommonConstants.JSON_EXTENSION); - return writeToFile(sourceEntity, metadataPath, gson); - } catch (IOException e) { - log.error("Error while reading file {} with message {} with cause", path, e.getMessage(), e.getCause()); - } - return false; - } - - private boolean writeStringToFile(String data, Path path) throws IOException { - try (BufferedWriter fileWriter = Files.newBufferedWriter(path, StandardCharsets.UTF_8)) { - fileWriter.write(data); - return true; - } - } - - private boolean writeToFile(Object sourceEntity, Path path, Gson gson) throws IOException { - try (BufferedWriter fileWriter = Files.newBufferedWriter(path, StandardCharsets.UTF_8)) { - gson.toJson(sourceEntity, fileWriter); - return true; - } - } - - /** - * This method will delete the JSON resource available in local git directory on subsequent commit made after the - * deletion of respective resource from DB - * @param validResources resources those are still available in DB - * @param resourceDirectory directory which needs to be scanned for possible file deletion operations - */ - public void scanAndDeleteFileForDeletedResources(Set validResources, Path resourceDirectory) { - // Scan resource directory and delete any unwanted file if present - // unwanted file : corresponding resource from DB has been deleted - if (resourceDirectory.toFile().exists()) { - try (Stream paths = Files.walk(resourceDirectory)) { - paths.filter(pathLocal -> Files.isRegularFile(pathLocal) - && !validResources.contains( - pathLocal.getFileName().toString())) - .forEach(this::deleteFile); - } catch (IOException e) { - log.error("Error while scanning directory: {}, with error {}", resourceDirectory, e.getMessage()); - } - } - } - - /** - * This method will delete the JSON resource directory available in local git directory on subsequent commit made after the - * deletion of respective resource from DB - * @param validResources resources those are still available in DB - * @param resourceDirectory directory which needs to be scanned for possible file deletion operations - */ - public void scanAndDeleteDirectoryForDeletedResources(Set validResources, Path resourceDirectory) { - // Scan resource directory and delete any unwanted directory if present - // unwanted directory : corresponding resource from DB has been deleted - if (resourceDirectory.toFile().exists()) { - try (Stream paths = Files.walk(resourceDirectory, 1)) { - paths.filter(path -> Files.isDirectory(path) - && !path.equals(resourceDirectory) - && !validResources.contains(path.getFileName().toString())) - .forEach(this::deleteDirectory); - } catch (IOException e) { - log.error("Error while scanning directory {} with error {}", resourceDirectory, e.getMessage()); - } - } - } - - /** - * This method will delete the directory and all its contents - * @param directory - */ - private void deleteDirectory(Path directory) { - if (directory.toFile().exists()) { - try { - FileUtils.deleteDirectory(directory.toFile()); - } catch (IOException e) { - log.error("Unable to delete directory for path {} with message {}", directory, e.getMessage()); - } - } - } - - /** - * This method will delete the file from local repo - * @param filePath file that needs to be deleted - */ - private void deleteFile(Path filePath) { - try { - Files.deleteIfExists(filePath); - } catch (DirectoryNotEmptyException e) { - log.error("Unable to delete non-empty directory at {} with cause", filePath, e.getMessage()); - } catch (IOException e) { - log.error("Unable to delete file {} with {}", filePath, e.getMessage()); - } - } - - /** - * This will reconstruct the application from the repo - * @param organisationId To which organisation application needs to be rehydrated - * @param defaultApplicationId To which organisation application needs to be rehydrated - * @param branchName for which the application needs to be rehydrate - * @return application reference from which entire application can be rehydrated - */ - public Mono reconstructApplicationReferenceFromGitRepo( - String organisationId, String defaultApplicationId, String repoName, String branchName) { - - Stopwatch processStopwatch = new Stopwatch("FS reconstruct application"); - Path baseRepoSuffix = Paths.get(organisationId, defaultApplicationId, repoName); - - // Checkout to mentioned branch if not already checked-out - return gitExecutor - .checkoutToBranch(baseRepoSuffix, branchName) - .map(isSwitched -> { - Path baseRepoPath = - Paths.get(gitServiceConfig.getGitRootPath()).resolve(baseRepoSuffix); - - // Instance creator is required while de-serialising using Gson as key instance can't be invoked - // with - // no-args constructor - Gson gson = new GsonBuilder() - .registerTypeAdapter( - DatasourceStructure.Key.class, new DatasourceStructure.KeyInstanceCreator()) - .create(); - - ApplicationGitReference applicationGitReference = fetchApplicationReference(baseRepoPath, gson); - processStopwatch.stopAndLogTimeInMillis(); - return applicationGitReference; - }) - .subscribeOn(scheduler); - } - - /** - * This is used to initialize repo with Readme file when the application is connected to remote repo - * - * @param baseRepoSuffix path suffix used to create a repo path this includes the readme.md as well - * @param viewModeUrl URL to deployed version of the application view only mode - * @param editModeUrl URL to deployed version of the application edit mode - * @return Path to the base repo - * @throws IOException - */ - @Override - public Mono initializeReadme(Path baseRepoSuffix, String viewModeUrl, String editModeUrl) throws IOException { - return Mono.fromCallable(() -> { - ClassLoader classLoader = getClass().getClassLoader(); - InputStream inputStream = classLoader.getResourceAsStream(gitServiceConfig.getReadmeTemplatePath()); - - StringWriter stringWriter = new StringWriter(); - IOUtils.copy(inputStream, stringWriter, "UTF-8"); - String data = stringWriter - .toString() - .replace(EDIT_MODE_URL_TEMPLATE, editModeUrl) - .replace(VIEW_MODE_URL_TEMPLATE, viewModeUrl); - - File file = new File(Paths.get(gitServiceConfig.getGitRootPath()) - .resolve(baseRepoSuffix) - .toFile() - .toString()); - FileUtils.writeStringToFile(file, data, "UTF-8", true); - - // Remove readme.md from the path - return file.toPath().getParent(); - }) - .subscribeOn(scheduler); - } - - @Override - public Mono deleteLocalRepo(Path baseRepoSuffix) { - // Remove the complete directory from path: baseRepo/workspaceId/defaultApplicationId - File file = Paths.get(gitServiceConfig.getGitRootPath()) - .resolve(baseRepoSuffix) - .getParent() - .toFile(); - while (file.exists()) { - FileSystemUtils.deleteRecursively(file); - } - return Mono.just(Boolean.TRUE); - } - - @Override - public Mono checkIfDirectoryIsEmpty(Path baseRepoSuffix) { - return Mono.fromCallable(() -> { - File[] files = Paths.get(gitServiceConfig.getGitRootPath()) - .resolve(baseRepoSuffix) - .toFile() - .listFiles(); - for (File file : files) { - if (!ALLOWED_FILE_EXTENSION_PATTERN.matcher(file.getName()).matches() - && !file.getName().equals("LICENSE")) { - // Remove the cloned repo from the file system since the repo doesnt satisfy the criteria - while (file.exists()) { - FileSystemUtils.deleteRecursively(file); - } - return false; - } - } - return true; - }); - } - - /** - * This method will be used to read and dehydrate the json file present from the local git repo - * @param filePath file on which the read operation will be performed - * @param gson - * @return resource stored in the JSON file - */ - public static Object readFile(Path filePath, Gson gson) { - - Object file; - try (JsonReader reader = new JsonReader(new FileReader(filePath.toFile()))) { - file = gson.fromJson(reader, Object.class); - } catch (Exception e) { - log.error("Error while reading file {} with message {} with cause", filePath, e.getMessage(), e.getCause()); - return null; - } - return file; - } - - /** - * This method will be used to read and dehydrate the json files present from the local git repo - * @param directoryPath directory path for files on which read operation will be performed - * @param gson - * @return resources stored in the directory - */ - private Map readFiles(Path directoryPath, Gson gson, String keySuffix) { - Map resource = new HashMap<>(); - File directory = directoryPath.toFile(); - if (directory.isDirectory()) { - Arrays.stream(Objects.requireNonNull(directory.listFiles())).forEach(file -> { - try (JsonReader reader = new JsonReader(new FileReader(file))) { - resource.put(file.getName() + keySuffix, gson.fromJson(reader, Object.class)); - } catch (Exception e) { - log.error( - "Error while reading file {} with message {} with cause", - file.toPath(), - e.getMessage(), - e.getCause()); - } - }); - } - return resource; - } - - /** - * This method will read the content of the file as a plain text and does not apply the gson to json transformation - * @param filePath file path for files on which read operation will be performed - * @return content of the file in the path - */ - private String readFileAsString(Path filePath) { - String data = CommonConstants.EMPTY_STRING; - try { - data = FileUtils.readFileToString(filePath.toFile(), "UTF-8"); - } catch (IOException e) { - log.error("Error while reading the file from git repo {} ", e.getMessage()); - } - return data; - } - - /** - * This method is to read the content for action and actionCollection or any nested resources which has the new structure - v3 - * Where the user written JS Object code and the metadata is split into to different files - * @param directoryPath file path for files on which read operation will be performed - * @return resources stored in the directory - */ - private Map readActionCollection( - Path directoryPath, Gson gson, String keySuffix, Map actionCollectionBodyMap) { - Map resource = new HashMap<>(); - File directory = directoryPath.toFile(); - if (directory.isDirectory()) { - for (File dirFile : Objects.requireNonNull(directory.listFiles())) { - String resourceName = dirFile.getName(); - Path resourcePath = - directoryPath.resolve(resourceName).resolve(resourceName + CommonConstants.JS_EXTENSION); - String body = CommonConstants.EMPTY_STRING; - if (resourcePath.toFile().exists()) { - body = readFileAsString(resourcePath); - } - Object file = readFile( - directoryPath - .resolve(resourceName) - .resolve(CommonConstants.METADATA + CommonConstants.JSON_EXTENSION), - gson); - actionCollectionBodyMap.put(resourceName + keySuffix, body); - resource.put(resourceName + keySuffix, file); - } - } - return resource; - } - - /** - * This method is to read the content for action and actionCollection or any nested resources which has the new structure - v4 - * Where the user queries and the metadata is split into to different files - * @param directoryPath directory path for files on which read operation will be performed - * @param gson - * @return resources stored in the directory - */ - private Map readAction( - Path directoryPath, Gson gson, String keySuffix, Map actionCollectionBodyMap) { - Map resource = new HashMap<>(); - File directory = directoryPath.toFile(); - if (directory.isDirectory()) { - for (File dirFile : Objects.requireNonNull(directory.listFiles())) { - String resourceName = dirFile.getName(); - String body = CommonConstants.EMPTY_STRING; - Path queryPath = - directoryPath.resolve(resourceName).resolve(resourceName + CommonConstants.TEXT_FILE_EXTENSION); - if (queryPath.toFile().exists()) { - body = readFileAsString(queryPath); - } - Object file = readFile( - directoryPath - .resolve(resourceName) - .resolve(CommonConstants.METADATA + CommonConstants.JSON_EXTENSION), - gson); - actionCollectionBodyMap.put(resourceName + keySuffix, body); - resource.put(resourceName + keySuffix, file); - } - } - return resource; - } - - private Object readPageMetadata(Path directoryPath, Gson gson) { - return readFile(directoryPath.resolve(directoryPath.toFile().getName() + CommonConstants.JSON_EXTENSION), gson); - } - - private ApplicationGitReference fetchApplicationReference(Path baseRepoPath, Gson gson) { - ApplicationGitReference applicationGitReference = new ApplicationGitReference(); - // Extract application metadata from the json - Object metadata = - readFile(baseRepoPath.resolve(CommonConstants.METADATA + CommonConstants.JSON_EXTENSION), gson); - Integer fileFormatVersion = getFileFormatVersion(metadata); - // Check if fileFormat of the saved files in repo is compatible - if (!isFileFormatCompatible(fileFormatVersion)) { - throw new AppsmithPluginException(AppsmithPluginError.INCOMPATIBLE_FILE_FORMAT); - } - // Extract application data from the json - applicationGitReference.setApplication( - readFile(baseRepoPath.resolve(CommonConstants.APPLICATION + CommonConstants.JSON_EXTENSION), gson)); - applicationGitReference.setTheme( - readFile(baseRepoPath.resolve(CommonConstants.THEME + CommonConstants.JSON_EXTENSION), gson)); - Path pageDirectory = baseRepoPath.resolve(PAGE_DIRECTORY); - // Reconstruct application from given file format - switch (fileFormatVersion) { - case 1: - // Extract actions - applicationGitReference.setActions( - readFiles(baseRepoPath.resolve(ACTION_DIRECTORY), gson, CommonConstants.EMPTY_STRING)); - // Extract actionCollections - applicationGitReference.setActionCollections(readFiles( - baseRepoPath.resolve(ACTION_COLLECTION_DIRECTORY), gson, CommonConstants.EMPTY_STRING)); - // Extract pages - applicationGitReference.setPages(readFiles(pageDirectory, gson, CommonConstants.EMPTY_STRING)); - // Extract datasources - applicationGitReference.setDatasources( - readFiles(baseRepoPath.resolve(DATASOURCE_DIRECTORY), gson, CommonConstants.EMPTY_STRING)); - break; - - case 2: - case 3: - case 4: - updateGitApplicationReference( - baseRepoPath, gson, applicationGitReference, pageDirectory, fileFormatVersion); - break; - - case 5: - updateGitApplicationReferenceV2( - baseRepoPath, gson, applicationGitReference, pageDirectory, fileFormatVersion); - break; - - default: - } - applicationGitReference.setMetadata(metadata); - - Path jsLibDirectory = baseRepoPath.resolve(JS_LIB_DIRECTORY); - Map jsLibrariesMap = readFiles(jsLibDirectory, gson, CommonConstants.EMPTY_STRING); - applicationGitReference.setJsLibraries(jsLibrariesMap); - - return applicationGitReference; - } - - @Deprecated - private void updateGitApplicationReference( - Path baseRepoPath, - Gson gson, - ApplicationGitReference applicationGitReference, - Path pageDirectory, - int fileFormatVersion) { - // Extract pages and nested actions and actionCollections - File directory = pageDirectory.toFile(); - Map pageMap = new HashMap<>(); - Map actionMap = new HashMap<>(); - Map actionBodyMap = new HashMap<>(); - Map actionCollectionMap = new HashMap<>(); - Map actionCollectionBodyMap = new HashMap<>(); - if (directory.isDirectory()) { - // Loop through all the directories and nested directories inside the pages directory to extract - // pages, actions and actionCollections from the JSON files - for (File page : Objects.requireNonNull(directory.listFiles())) { - pageMap.put( - page.getName(), - readFile(page.toPath().resolve(CommonConstants.CANVAS + CommonConstants.JSON_EXTENSION), gson)); - - if (fileFormatVersion >= 4) { - actionMap.putAll( - readAction(page.toPath().resolve(ACTION_DIRECTORY), gson, page.getName(), actionBodyMap)); - } else { - actionMap.putAll(readFiles(page.toPath().resolve(ACTION_DIRECTORY), gson, page.getName())); - } - - if (fileFormatVersion >= 3) { - actionCollectionMap.putAll(readActionCollection( - page.toPath().resolve(ACTION_COLLECTION_DIRECTORY), - gson, - page.getName(), - actionCollectionBodyMap)); - } else { - actionCollectionMap.putAll( - readFiles(page.toPath().resolve(ACTION_COLLECTION_DIRECTORY), gson, page.getName())); - } - } - } - applicationGitReference.setActions(actionMap); - applicationGitReference.setActionBody(actionBodyMap); - applicationGitReference.setActionCollections(actionCollectionMap); - applicationGitReference.setActionCollectionBody(actionCollectionBodyMap); - applicationGitReference.setPages(pageMap); - // Extract datasources - applicationGitReference.setDatasources( - readFiles(baseRepoPath.resolve(DATASOURCE_DIRECTORY), gson, CommonConstants.EMPTY_STRING)); - } - - private Integer getFileFormatVersion(Object metadata) { - if (metadata == null) { - return 1; - } - Gson gson = new Gson(); - JsonObject json = gson.fromJson(gson.toJson(metadata), JsonObject.class); - JsonElement fileFormatVersion = json.get(CommonConstants.FILE_FORMAT_VERSION); - return fileFormatVersion.getAsInt(); - } - - public static boolean isFileFormatCompatible(int savedFileFormat) { - return savedFileFormat <= CommonConstants.fileFormatVersion; - } - - private void updateGitApplicationReferenceV2( - Path baseRepoPath, - Gson gson, - ApplicationGitReference applicationGitReference, - Path pageDirectory, - int fileFormatVersion) { - // Extract pages and nested actions and actionCollections - File directory = pageDirectory.toFile(); - Map pageMap = new HashMap<>(); - Map pageDsl = new HashMap<>(); - Map actionMap = new HashMap<>(); - Map actionBodyMap = new HashMap<>(); - Map actionCollectionMap = new HashMap<>(); - Map actionCollectionBodyMap = new HashMap<>(); - if (directory.isDirectory()) { - // Loop through all the directories and nested directories inside the pages directory to extract - // pages, actions and actionCollections from the JSON files - for (File page : Objects.requireNonNull(directory.listFiles())) { - if (page.isDirectory()) { - pageMap.put(page.getName(), readPageMetadata(page.toPath(), gson)); - - JSONObject mainContainer = getMainContainer(pageMap.get(page.getName()), gson); - - // Read widgets data recursively from the widgets directory - Map widgetsData = readWidgetsData( - page.toPath().resolve(CommonConstants.WIDGETS).toString()); - // Construct the nested DSL from the widgets data - Map> parentDirectories = DSLTransformerHelper.calculateParentDirectories( - widgetsData.keySet().stream().toList()); - JSONObject nestedDSL = - DSLTransformerHelper.getNestedDSL(widgetsData, parentDirectories, mainContainer); - pageDsl.put(page.getName(), nestedDSL.toString()); - actionMap.putAll( - readAction(page.toPath().resolve(ACTION_DIRECTORY), gson, page.getName(), actionBodyMap)); - actionCollectionMap.putAll(readActionCollection( - page.toPath().resolve(ACTION_COLLECTION_DIRECTORY), - gson, - page.getName(), - actionCollectionBodyMap)); - } - } - } - applicationGitReference.setActions(actionMap); - applicationGitReference.setActionBody(actionBodyMap); - applicationGitReference.setActionCollections(actionCollectionMap); - applicationGitReference.setActionCollectionBody(actionCollectionBodyMap); - applicationGitReference.setPages(pageMap); - applicationGitReference.setPageDsl(pageDsl); - // Extract datasources - applicationGitReference.setDatasources( - readFiles(baseRepoPath.resolve(DATASOURCE_DIRECTORY), gson, CommonConstants.EMPTY_STRING)); - } - - private Map readWidgetsData(String directoryPath) { - Map jsonMap = new HashMap<>(); - File directory = new File(directoryPath); - - if (!directory.isDirectory()) { - log.error("Error reading directory: {}", directoryPath); - return jsonMap; - } - - try { - readFilesRecursively(directory, jsonMap, directoryPath); - } catch (IOException exception) { - log.error("Error reading directory: {}, error message {}", directoryPath, exception.getMessage()); - } - - return jsonMap; - } - - private void readFilesRecursively(File directory, Map jsonMap, String rootPath) - throws IOException { - File[] files = directory.listFiles(); - if (files == null) { - return; - } - - for (File file : files) { - if (file.isFile()) { - String filePath = file.getAbsolutePath(); - String relativePath = filePath.replace(rootPath, CommonConstants.EMPTY_STRING); - relativePath = CommonConstants.DELIMITER_PATH - + CommonConstants.MAIN_CONTAINER - + relativePath.substring(relativePath.indexOf("//") + 1); - try { - String fileContent = new String(Files.readAllBytes(file.toPath())); - JSONObject jsonObject = new JSONObject(fileContent); - jsonMap.put(relativePath, jsonObject); - } catch (IOException exception) { - log.error("Error reading file: {}, error message {}", filePath, exception.getMessage()); - } - } else if (file.isDirectory()) { - readFilesRecursively(file, jsonMap, rootPath); - } - } - } - - private void deleteWidgets(File directory, Map validWidgetToParentMap) { - File[] files = directory.listFiles(); - if (files == null) { - return; - } - - for (File file : files) { - if (file.isDirectory()) { - deleteWidgets(file, validWidgetToParentMap); - } - - String name = file.getName().replace(CommonConstants.JSON_EXTENSION, CommonConstants.EMPTY_STRING); - // If input widget was inside a container before, but the user moved it out of the container - // then we need to delete the widget from the container directory - // The check here is to validate if the parent is correct or not - if (!validWidgetToParentMap.containsKey(name)) { - if (file.isDirectory()) { - deleteDirectory(file.toPath()); - } else { - deleteFile(file.toPath()); - } - } else if (!file.getParentFile().getPath().equals(validWidgetToParentMap.get(name)) - && !file.getPath().equals(validWidgetToParentMap.get(name))) { - if (file.isDirectory()) { - deleteDirectory(file.toPath()); - } else { - deleteFile(file.toPath()); - } - } - } - } - - private JSONObject getMainContainer(Object pageJson, Gson gson) { - JSONObject pageJSON = new JSONObject(gson.toJson(pageJson)); - JSONArray layouts = pageJSON.getJSONObject("unpublishedPage").getJSONArray("layouts"); - return layouts.getJSONObject(0).getJSONObject("dsl"); - } - - @Override - public Mono deleteIndexLockFile(Path path, int validTimeInSeconds) { - // Check the time created of the index.lock file - // If the File is stale for more than validTime, then delete the file - try { - BasicFileAttributes attr = Files.readAttributes(path, BasicFileAttributes.class); - FileTime fileTime = attr.creationTime(); - Instant now = Instant.now(); - Instant validCreateTime = now.minusSeconds(validTimeInSeconds); - if (fileTime.toInstant().isBefore(validCreateTime)) { - // Add base repo path - path = Paths.get(path + ".lock"); - deleteFile(path); - return Mono.just(now.minusMillis(fileTime.toMillis()).getEpochSecond()); - } else { - return Mono.just(0L); - } - } catch (IOException ex) { - log.error("Error reading index.lock file: {}", ex.getMessage()); - return Mono.just(0L); - } - } - - /** - * We use UID string for custom js lib. UID strings are in this format: {libname}_{url to the lib src}. - * This method converts this uid string into a valid file name so that there is no unsupported character in the - * file name for any OS. - * This method returns a string in the format: {libname}_{base64 encoded hash of uid string} - * @param uidString UID string value of a JS lib - * @return String - */ - public static String getJsLibFileName(String uidString) { - int firstUnderscoreIndex = uidString.indexOf('_'); // this finds the first occurrence of "_" - String prefix; - if (firstUnderscoreIndex != -1) { - prefix = uidString.substring(0, firstUnderscoreIndex); // we're getting the prefix from the uidString - } else { - prefix = "jslib"; - } - - StringBuilder stringBuilder = new StringBuilder(prefix); - stringBuilder.append("_"); - try { - MessageDigest digest = MessageDigest.getInstance("SHA-256"); - byte[] hash = digest.digest(uidString.getBytes(StandardCharsets.UTF_8)); - stringBuilder.append(Base64.getUrlEncoder().withoutPadding().encodeToString(hash)); - } catch (NoSuchAlgorithmException e) { - throw new RuntimeException("Failed to hash URL string", e); - } - return stringBuilder.toString(); + public FileUtilsImpl(GitServiceConfig gitServiceConfig, GitExecutor gitExecutor) { + super(gitServiceConfig, gitExecutor); } } diff --git a/app/server/appsmith-git/src/main/java/com/appsmith/git/helpers/ce/FileUtilsCEImpl.java b/app/server/appsmith-git/src/main/java/com/appsmith/git/helpers/ce/FileUtilsCEImpl.java new file mode 100644 index 0000000000..889a228c5d --- /dev/null +++ b/app/server/appsmith-git/src/main/java/com/appsmith/git/helpers/ce/FileUtilsCEImpl.java @@ -0,0 +1,1154 @@ +package com.appsmith.git.helpers.ce; + +import com.appsmith.external.converters.ISOStringToInstantConverter; +import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginError; +import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginException; +import com.appsmith.external.git.FileInterface; +import com.appsmith.external.git.GitExecutor; +import com.appsmith.external.helpers.Stopwatch; +import com.appsmith.external.models.ApplicationGitReference; +import com.appsmith.external.models.DatasourceStructure; +import com.appsmith.git.configurations.GitServiceConfig; +import com.appsmith.git.constants.CommonConstants; +import com.appsmith.git.converters.GsonDoubleToLongConverter; +import com.appsmith.git.converters.GsonUnorderedToOrderedConverter; +import com.appsmith.git.helpers.DSLTransformerHelper; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.stream.JsonReader; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.eclipse.jgit.api.errors.GitAPIException; +import org.json.JSONArray; +import org.json.JSONObject; +import org.springframework.context.annotation.Import; +import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; +import org.springframework.util.FileSystemUtils; +import org.springframework.util.StringUtils; +import reactor.core.publisher.Mono; +import reactor.core.scheduler.Scheduler; +import reactor.core.scheduler.Schedulers; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.StringWriter; +import java.nio.charset.StandardCharsets; +import java.nio.file.DirectoryNotEmptyException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.BasicFileAttributes; +import java.nio.file.attribute.FileTime; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.time.Instant; +import java.util.Arrays; +import java.util.Base64; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.regex.Pattern; +import java.util.stream.Stream; + +import static com.appsmith.external.constants.GitConstants.ACTION_COLLECTION_LIST; +import static com.appsmith.external.constants.GitConstants.ACTION_LIST; +import static com.appsmith.external.constants.GitConstants.CUSTOM_JS_LIB_LIST; +import static com.appsmith.external.constants.GitConstants.NAME_SEPARATOR; +import static com.appsmith.external.constants.GitConstants.PAGE_LIST; +import static com.appsmith.git.constants.GitDirectories.ACTION_COLLECTION_DIRECTORY; +import static com.appsmith.git.constants.GitDirectories.ACTION_DIRECTORY; +import static com.appsmith.git.constants.GitDirectories.DATASOURCE_DIRECTORY; +import static com.appsmith.git.constants.GitDirectories.JS_LIB_DIRECTORY; +import static com.appsmith.git.constants.GitDirectories.PAGE_DIRECTORY; + +@Slf4j +@Getter +@RequiredArgsConstructor +@Component +@Import({GitServiceConfig.class}) +public class FileUtilsCEImpl implements FileInterface { + + private final GitServiceConfig gitServiceConfig; + + private final GitExecutor gitExecutor; + + private static final String EDIT_MODE_URL_TEMPLATE = "{{editModeUrl}}"; + + private static final String VIEW_MODE_URL_TEMPLATE = "{{viewModeUrl}}"; + + private static final Pattern ALLOWED_FILE_EXTENSION_PATTERN = + Pattern.compile("(.*?)\\.(md|MD|git|gitignore|github|yml|yaml)$"); + + private final Scheduler scheduler = Schedulers.boundedElastic(); + + private static final String CANVAS_WIDGET = "(Canvas)[0-9]*."; + + /** + * Application will be stored in the following structure: + * + * For v1: + * repo_name + * application.json + * metadata.json + * datasource + * datasource1Name.json + * datasource2Name.json + * queries (Only requirement here is the filename should be unique) + * action1_page1 + * action2_page2 + * jsobjects (Only requirement here is the filename should be unique) + * jsobject1_page1 + * jsobject2_page2 + * pages + * page1 + * page2 + * + * For v2: + * repo_name + * application.json + * metadata.json + * theme + * publishedTheme.json + * editModeTheme.json + * pages + * page1 + * canvas.json + * queries + * Query1.json + * Query2.json + * jsobjects + * JSObject1.json + * page2 + * page3 + * datasources + * datasource1.json + * datasource2.json + * + * For v3: + * repo_name + * application.json + * metadata.json + * theme + * publishedTheme.json + * editModeTheme.json + * pages + * page1 + * canvas.json + * queries + * Query1.json + * jsobjects + * JSObject1 + * JSObject1.js + * Metadata.json + * page2 + * page3 + * datasources + * datasource1.json + * datasource2.json + * + * For v4: + * repo_name + * application.json + * metadata.json + * theme + * publishedTheme.json + * editModeTheme.json + * pages + * page1 + * canvas.json + * queries + * Query1.json + * jsobjects + * JSObject1 + * JSObject1.js + * Metadata.json + * page2 + * page3 + * datasources + * datasource1.json + * datasource2.json + */ + + /** + * This method will save the complete application in the local repo directory. + * Path to repo will be : ./container-volumes/git-repo/workspaceId/defaultApplicationId/repoName/{application_data} + * + * @param baseRepoSuffix path suffix used to create a repo path + * @param applicationGitReference application reference object from which entire application can be rehydrated + * @param branchName name of the branch for the current application + * @return repo path where the application is stored + */ + public Mono saveApplicationToGitRepo( + Path baseRepoSuffix, ApplicationGitReference applicationGitReference, String branchName) + throws GitAPIException, IOException { + + // Repo path will be: + // baseRepo : root/orgId/defaultAppId/repoName/{applicationData} + // Checkout to mentioned branch if not already checked-out + Stopwatch processStopwatch = new Stopwatch("FS application save"); + return gitExecutor + .resetToLastCommit(baseRepoSuffix, branchName) + .flatMap(isSwitched -> { + Path baseRepo = Paths.get(gitServiceConfig.getGitRootPath()).resolve(baseRepoSuffix); + + // Gson to pretty format JSON file + // Keep Long type as is by default GSON have behavior to convert to Double + // Convert unordered set to ordered one + Gson gson = new GsonBuilder() + .registerTypeAdapter(Double.class, new GsonDoubleToLongConverter()) + .registerTypeAdapter(Set.class, new GsonUnorderedToOrderedConverter()) + .registerTypeAdapter(Map.class, new GsonUnorderedToOrderedConverter()) + .registerTypeAdapter(Instant.class, new ISOStringToInstantConverter()) + .disableHtmlEscaping() + .setPrettyPrinting() + .create(); + + updateEntitiesInRepo(applicationGitReference, baseRepo, gson); + + processStopwatch.stopAndLogTimeInMillis(); + return Mono.just(baseRepo); + }) + .subscribeOn(scheduler); + } + + protected Set updateEntitiesInRepo( + ApplicationGitReference applicationGitReference, Path baseRepo, Gson gson) { + + Set validDatasourceFileNames = new HashSet<>(); + Map> updatedResources = applicationGitReference.getUpdatedResources(); + + // Remove unwanted directories which was present in v1 of the git file format version + deleteDirectory(baseRepo.resolve(ACTION_DIRECTORY)); + deleteDirectory(baseRepo.resolve(ACTION_COLLECTION_DIRECTORY)); + + // Save application + saveResource( + applicationGitReference.getApplication(), + baseRepo.resolve(CommonConstants.APPLICATION + CommonConstants.JSON_EXTENSION), + gson); + + // Save application metadata + JsonObject metadata = gson.fromJson(gson.toJson(applicationGitReference.getMetadata()), JsonObject.class); + metadata.addProperty(CommonConstants.FILE_FORMAT_VERSION, CommonConstants.fileFormatVersion); + saveResource(metadata, baseRepo.resolve(CommonConstants.METADATA + CommonConstants.JSON_EXTENSION), gson); + + // Save application theme + saveResource( + applicationGitReference.getTheme(), + baseRepo.resolve(CommonConstants.THEME + CommonConstants.JSON_EXTENSION), + gson); + + // Save pages + Path pageDirectory = baseRepo.resolve(PAGE_DIRECTORY); + Set> pageEntries = + applicationGitReference.getPages().entrySet(); + + Set validPages = new HashSet<>(); + for (Map.Entry pageResource : pageEntries) { + Map validWidgetToParentMap = new HashMap<>(); + final String pageName = pageResource.getKey(); + Path pageSpecificDirectory = pageDirectory.resolve(pageName); + Boolean isResourceUpdated = !CollectionUtils.isEmpty(updatedResources.get(PAGE_LIST)) + ? updatedResources.get(PAGE_LIST).contains(pageName) + : Boolean.FALSE; + if (Boolean.TRUE.equals(isResourceUpdated)) { + // Save page metadata + saveResource( + pageResource.getValue(), + pageSpecificDirectory.resolve(pageName + CommonConstants.JSON_EXTENSION), + gson); + Map result = DSLTransformerHelper.flatten( + new JSONObject(applicationGitReference.getPageDsl().get(pageName))); + result.forEach((key, jsonObject) -> { + // get path with splitting the name via key + String widgetName = key.substring(key.lastIndexOf(CommonConstants.DELIMITER_POINT) + 1); + String childPath = key.replace(CommonConstants.MAIN_CONTAINER, CommonConstants.EMPTY_STRING) + .replace(CommonConstants.DELIMITER_POINT, CommonConstants.DELIMITER_PATH); + // Replace the canvas Widget as a child and add it to the same level as parent + childPath = childPath.replaceAll(CANVAS_WIDGET, CommonConstants.EMPTY_STRING); + if (!DSLTransformerHelper.hasChildren(jsonObject) + && !DSLTransformerHelper.isTabsWidget(jsonObject)) { + // Save the widget as a directory or Save the widget as a file + childPath = childPath.replace(widgetName, CommonConstants.EMPTY_STRING); + } + Path path = Paths.get( + String.valueOf(pageSpecificDirectory.resolve(CommonConstants.WIDGETS)), childPath); + validWidgetToParentMap.put(widgetName, path.toFile().toString()); + saveWidgets(jsonObject, widgetName, path); + }); + // Remove deleted widgets from the file system + deleteWidgets( + pageSpecificDirectory.resolve(CommonConstants.WIDGETS).toFile(), validWidgetToParentMap); + + // Remove the canvas.json from the file system since the value is stored in the page.json + deleteFile(pageSpecificDirectory.resolve(CommonConstants.CANVAS + CommonConstants.JSON_EXTENSION)); + } + validPages.add(pageName); + } + scanAndDeleteDirectoryForDeletedResources(validPages, baseRepo.resolve(PAGE_DIRECTORY)); + + // Save JS Libs if there's at least one change + if (updatedResources != null && !CollectionUtils.isEmpty(updatedResources.get(CUSTOM_JS_LIB_LIST))) { + Path jsLibDirectory = baseRepo.resolve(JS_LIB_DIRECTORY); + Set> jsLibEntries = + applicationGitReference.getJsLibraries().entrySet(); + Set validJsLibs = new HashSet<>(); + jsLibEntries.forEach(jsLibEntry -> { + String uidString = jsLibEntry.getKey(); + boolean isResourceUpdated = + updatedResources.get(CUSTOM_JS_LIB_LIST).contains(uidString); + + String fileNameWithExtension = getJsLibFileName(uidString) + CommonConstants.JSON_EXTENSION; + + Path jsLibSpecificFile = jsLibDirectory.resolve(fileNameWithExtension); + if (isResourceUpdated) { + saveResource(jsLibEntry.getValue(), jsLibSpecificFile, gson); + } + validJsLibs.add(fileNameWithExtension); + }); + scanAndDeleteFileForDeletedResources(validJsLibs, jsLibDirectory); + } + + // Create HashMap for valid actions and actionCollections + HashMap> validActionsMap = new HashMap<>(); + HashMap> validActionCollectionsMap = new HashMap<>(); + HashMap> validModuleInstancesMap = new HashMap<>(); + validPages.forEach(validPage -> { + validActionsMap.put(validPage, new HashSet<>()); + validActionCollectionsMap.put(validPage, new HashSet<>()); + validModuleInstancesMap.put(validPage, new HashSet<>()); + }); + + // Save actions + for (Map.Entry resource : + applicationGitReference.getActions().entrySet()) { + // queryName_pageName => nomenclature for the keys + // TODO queryName => for app level queries, this is not implemented yet + String[] names = resource.getKey().split(NAME_SEPARATOR); + if (names.length > 1 && StringUtils.hasLength(names[1])) { + // For actions, we are referring to validNames to maintain unique file names as just name + // field don't guarantee unique constraint for actions within JSObject + Boolean isResourceUpdated = !CollectionUtils.isEmpty(updatedResources.get(ACTION_LIST)) + ? updatedResources.get(ACTION_LIST).contains(resource.getKey()) + : Boolean.FALSE; + final String queryName = names[0].replace(".", "-"); + final String pageName = names[1]; + Path pageSpecificDirectory = pageDirectory.resolve(pageName); + Path actionSpecificDirectory = pageSpecificDirectory.resolve(ACTION_DIRECTORY); + + if (!validActionsMap.containsKey(pageName)) { + validActionsMap.put(pageName, new HashSet<>()); + } + validActionsMap.get(pageName).add(queryName); + if (Boolean.TRUE.equals(isResourceUpdated)) { + saveActions( + resource.getValue(), + applicationGitReference.getActionBody().containsKey(resource.getKey()) + ? applicationGitReference.getActionBody().get(resource.getKey()) + : null, + queryName, + actionSpecificDirectory.resolve(queryName), + gson); + // Delete the resource from the old file structure v2 + deleteFile(pageSpecificDirectory + .resolve(ACTION_DIRECTORY) + .resolve(queryName + CommonConstants.JSON_EXTENSION)); + } + } + } + + validActionsMap.forEach((pageName, validActionNames) -> { + Path pageSpecificDirectory = pageDirectory.resolve(pageName); + scanAndDeleteDirectoryForDeletedResources( + validActionNames, pageSpecificDirectory.resolve(ACTION_DIRECTORY)); + }); + + // Save JSObjects + for (Map.Entry resource : + applicationGitReference.getActionCollections().entrySet()) { + // JSObjectName_pageName => nomenclature for the keys + // TODO JSObjectName => for app level JSObjects, this is not implemented yet + String[] names = resource.getKey().split(NAME_SEPARATOR); + if (names.length > 1 && StringUtils.hasLength(names[1])) { + final String actionCollectionName = names[0]; + final String pageName = names[1]; + Path pageSpecificDirectory = pageDirectory.resolve(pageName); + Path actionCollectionSpecificDirectory = pageSpecificDirectory.resolve(ACTION_COLLECTION_DIRECTORY); + + if (!validActionCollectionsMap.containsKey(pageName)) { + validActionCollectionsMap.put(pageName, new HashSet<>()); + } + validActionCollectionsMap.get(pageName).add(actionCollectionName); + Boolean isResourceUpdated = !CollectionUtils.isEmpty(updatedResources.get(ACTION_COLLECTION_LIST)) + ? updatedResources.get(ACTION_COLLECTION_LIST).contains(resource.getKey()) + : Boolean.FALSE; + if (Boolean.TRUE.equals(isResourceUpdated)) { + saveActionCollection( + resource.getValue(), + applicationGitReference.getActionCollectionBody().get(resource.getKey()), + actionCollectionName, + actionCollectionSpecificDirectory.resolve(actionCollectionName), + gson); + // Delete the resource from the old file structure v2 + deleteFile(actionCollectionSpecificDirectory.resolve( + actionCollectionName + CommonConstants.JSON_EXTENSION)); + } + } + } + + // Verify if the old files are deleted + validActionCollectionsMap.forEach((pageName, validActionCollectionNames) -> { + Path pageSpecificDirectory = pageDirectory.resolve(pageName); + scanAndDeleteDirectoryForDeletedResources( + validActionCollectionNames, pageSpecificDirectory.resolve(ACTION_COLLECTION_DIRECTORY)); + }); + + // Save datasources ref + for (Map.Entry resource : + applicationGitReference.getDatasources().entrySet()) { + saveResource( + resource.getValue(), + baseRepo.resolve(DATASOURCE_DIRECTORY).resolve(resource.getKey() + CommonConstants.JSON_EXTENSION), + gson); + validDatasourceFileNames.add(resource.getKey() + CommonConstants.JSON_EXTENSION); + } + // Scan datasource directory and delete any unwanted files if present + if (!applicationGitReference.getDatasources().isEmpty()) { + scanAndDeleteFileForDeletedResources(validDatasourceFileNames, baseRepo.resolve(DATASOURCE_DIRECTORY)); + } + + return validPages; + } + + /** + * This method will be used to store the DB resource to JSON file + * + * @param sourceEntity resource extracted from DB to be stored in file + * @param path file path where the resource to be stored + * @param gson + * @return if the file operation is successful + */ + protected boolean saveResource(Object sourceEntity, Path path, Gson gson) { + try { + Files.createDirectories(path.getParent()); + return writeToFile(sourceEntity, path, gson); + } catch (IOException e) { + log.error("Error while writing resource to file {} with {}", path, e.getMessage()); + log.debug(e.getMessage()); + } + return false; + } + + private void saveWidgets(JSONObject sourceEntity, String resourceName, Path path) { + try { + Files.createDirectories(path); + writeStringToFile(sourceEntity.toString(4), path.resolve(resourceName + CommonConstants.JSON_EXTENSION)); + } catch (IOException e) { + log.debug("Error while writings widgets data to file, {}", e.getMessage()); + } + } + + /** + * This method is used to write actionCollection specific resource to file system. We write the data in two steps + * 1. Actual js code + * 2. Metadata of the actionCollection + * + * @param sourceEntity the metadata of the action collection + * @param body actual js code written by the user + * @param resourceName name of the action collection + * @param path file path where the resource will be stored + * @param gson + * @return if the file operation is successful + */ + private boolean saveActionCollection(Object sourceEntity, String body, String resourceName, Path path, Gson gson) { + try { + Files.createDirectories(path); + if (StringUtils.hasText(body)) { + // Write the js Object body to .js file to make conflict handling easier + Path bodyPath = path.resolve(resourceName + CommonConstants.JS_EXTENSION); + writeStringToFile(body, bodyPath); + } + + // Write metadata for the jsObject + Path metadataPath = path.resolve(CommonConstants.METADATA + CommonConstants.JSON_EXTENSION); + return writeToFile(sourceEntity, metadataPath, gson); + } catch (IOException e) { + log.debug(e.getMessage()); + } + return false; + } + + /** + * This method is used to write action specific resource to file system. We write the data in two steps + * * 1. Actual query written by the user + * * 2. Metadata of the actios + * + * @param sourceEntity the metadata of the action + * @param body actual query written by the user + * @param resourceName name of the action + * @param path file path where the resource will be stored + * @param gson + * @return if the file operation is successful + */ + private boolean saveActions(Object sourceEntity, String body, String resourceName, Path path, Gson gson) { + try { + Files.createDirectories(path); + // Write the user written query to .txt file to make conflict handling easier + // Body will be null if the action is of type JS + if (StringUtils.hasLength(body)) { + Path bodyPath = path.resolve(resourceName + CommonConstants.TEXT_FILE_EXTENSION); + writeStringToFile(body, bodyPath); + } + + // Write metadata for the actions + Path metadataPath = path.resolve(CommonConstants.METADATA + CommonConstants.JSON_EXTENSION); + return writeToFile(sourceEntity, metadataPath, gson); + } catch (IOException e) { + log.error("Error while reading file {} with message {} with cause", path, e.getMessage(), e.getCause()); + } + return false; + } + + private boolean writeStringToFile(String data, Path path) throws IOException { + try (BufferedWriter fileWriter = Files.newBufferedWriter(path, StandardCharsets.UTF_8)) { + fileWriter.write(data); + return true; + } + } + + private boolean writeToFile(Object sourceEntity, Path path, Gson gson) throws IOException { + try (BufferedWriter fileWriter = Files.newBufferedWriter(path, StandardCharsets.UTF_8)) { + gson.toJson(sourceEntity, fileWriter); + return true; + } + } + + /** + * This method will delete the JSON resource available in local git directory on subsequent commit made after the + * deletion of respective resource from DB + * + * @param validResources resources those are still available in DB + * @param resourceDirectory directory which needs to be scanned for possible file deletion operations + */ + public void scanAndDeleteFileForDeletedResources(Set validResources, Path resourceDirectory) { + // Scan resource directory and delete any unwanted file if present + // unwanted file : corresponding resource from DB has been deleted + if (resourceDirectory.toFile().exists()) { + try (Stream paths = Files.walk(resourceDirectory)) { + paths.filter(pathLocal -> Files.isRegularFile(pathLocal) + && !validResources.contains( + pathLocal.getFileName().toString())) + .forEach(this::deleteFile); + } catch (IOException e) { + log.error("Error while scanning directory: {}, with error {}", resourceDirectory, e.getMessage()); + } + } + } + + /** + * This method will delete the JSON resource directory available in local git directory on subsequent commit made after the + * deletion of respective resource from DB + * + * @param validResources resources those are still available in DB + * @param resourceDirectory directory which needs to be scanned for possible file deletion operations + */ + public void scanAndDeleteDirectoryForDeletedResources(Set validResources, Path resourceDirectory) { + // Scan resource directory and delete any unwanted directory if present + // unwanted directory : corresponding resource from DB has been deleted + if (resourceDirectory.toFile().exists()) { + try (Stream paths = Files.walk(resourceDirectory, 1)) { + paths.filter(path -> Files.isDirectory(path) + && !path.equals(resourceDirectory) + && !validResources.contains(path.getFileName().toString())) + .forEach(this::deleteDirectory); + } catch (IOException e) { + log.error("Error while scanning directory {} with error {}", resourceDirectory, e.getMessage()); + } + } + } + + /** + * This method will delete the directory and all its contents + * + * @param directory + */ + private void deleteDirectory(Path directory) { + if (directory.toFile().exists()) { + try { + FileUtils.deleteDirectory(directory.toFile()); + } catch (IOException e) { + log.error("Unable to delete directory for path {} with message {}", directory, e.getMessage()); + } + } + } + + /** + * This method will delete the file from local repo + * + * @param filePath file that needs to be deleted + */ + private void deleteFile(Path filePath) { + try { + Files.deleteIfExists(filePath); + } catch (DirectoryNotEmptyException e) { + log.error("Unable to delete non-empty directory at {} with cause", filePath, e.getMessage()); + } catch (IOException e) { + log.error("Unable to delete file {} with {}", filePath, e.getMessage()); + } + } + + /** + * This will reconstruct the application from the repo + * + * @param organisationId To which organisation application needs to be rehydrated + * @param defaultApplicationId To which organisation application needs to be rehydrated + * @param branchName for which the application needs to be rehydrate + * @return application reference from which entire application can be rehydrated + */ + public Mono reconstructApplicationReferenceFromGitRepo( + String organisationId, String defaultApplicationId, String repoName, String branchName) { + + Stopwatch processStopwatch = new Stopwatch("FS reconstruct application"); + Path baseRepoSuffix = Paths.get(organisationId, defaultApplicationId, repoName); + + // Checkout to mentioned branch if not already checked-out + return gitExecutor + .checkoutToBranch(baseRepoSuffix, branchName) + .map(isSwitched -> { + Path baseRepoPath = + Paths.get(gitServiceConfig.getGitRootPath()).resolve(baseRepoSuffix); + + // Instance creator is required while de-serialising using Gson as key instance can't be invoked + // with no-args constructor + Gson gson = new GsonBuilder() + .registerTypeAdapter( + DatasourceStructure.Key.class, new DatasourceStructure.KeyInstanceCreator()) + .create(); + + ApplicationGitReference applicationGitReference = fetchApplicationReference(baseRepoPath, gson); + processStopwatch.stopAndLogTimeInMillis(); + return applicationGitReference; + }) + .subscribeOn(scheduler); + } + + /** + * This is used to initialize repo with Readme file when the application is connected to remote repo + * + * @param baseRepoSuffix path suffix used to create a repo path this includes the readme.md as well + * @param viewModeUrl URL to deployed version of the application view only mode + * @param editModeUrl URL to deployed version of the application edit mode + * @return Path to the base repo + * @throws IOException + */ + @Override + public Mono initializeReadme(Path baseRepoSuffix, String viewModeUrl, String editModeUrl) throws IOException { + return Mono.fromCallable(() -> { + ClassLoader classLoader = getClass().getClassLoader(); + InputStream inputStream = classLoader.getResourceAsStream(gitServiceConfig.getReadmeTemplatePath()); + + StringWriter stringWriter = new StringWriter(); + IOUtils.copy(inputStream, stringWriter, "UTF-8"); + String data = stringWriter + .toString() + .replace(EDIT_MODE_URL_TEMPLATE, editModeUrl) + .replace(VIEW_MODE_URL_TEMPLATE, viewModeUrl); + + File file = new File(Paths.get(gitServiceConfig.getGitRootPath()) + .resolve(baseRepoSuffix) + .toFile() + .toString()); + FileUtils.writeStringToFile(file, data, "UTF-8", true); + + // Remove readme.md from the path + return file.toPath().getParent(); + }) + .subscribeOn(scheduler); + } + + @Override + public Mono deleteLocalRepo(Path baseRepoSuffix) { + // Remove the complete directory from path: baseRepo/workspaceId/defaultApplicationId + File file = Paths.get(gitServiceConfig.getGitRootPath()) + .resolve(baseRepoSuffix) + .getParent() + .toFile(); + while (file.exists()) { + FileSystemUtils.deleteRecursively(file); + } + return Mono.just(Boolean.TRUE); + } + + @Override + public Mono checkIfDirectoryIsEmpty(Path baseRepoSuffix) { + return Mono.fromCallable(() -> { + File[] files = Paths.get(gitServiceConfig.getGitRootPath()) + .resolve(baseRepoSuffix) + .toFile() + .listFiles(); + for (File file : files) { + if (!ALLOWED_FILE_EXTENSION_PATTERN.matcher(file.getName()).matches() + && !file.getName().equals("LICENSE")) { + // Remove the cloned repo from the file system since the repo doesnt satisfy the criteria + while (file.exists()) { + FileSystemUtils.deleteRecursively(file); + } + return false; + } + } + return true; + }); + } + + /** + * This method will be used to read and dehydrate the json file present from the local git repo + * + * @param filePath file on which the read operation will be performed + * @param gson + * @return resource stored in the JSON file + */ + public static Object readFile(Path filePath, Gson gson) { + + Object file; + try (JsonReader reader = new JsonReader(new FileReader(filePath.toFile()))) { + file = gson.fromJson(reader, Object.class); + } catch (Exception e) { + log.error("Error while reading file {} with message {} with cause", filePath, e.getMessage(), e.getCause()); + return null; + } + return file; + } + + /** + * This method will be used to read and dehydrate the json files present from the local git repo + * + * @param directoryPath directory path for files on which read operation will be performed + * @param gson + * @return resources stored in the directory + */ + protected Map readFiles(Path directoryPath, Gson gson, String keySuffix) { + Map resource = new HashMap<>(); + File directory = directoryPath.toFile(); + if (directory.isDirectory()) { + Arrays.stream(Objects.requireNonNull(directory.listFiles())).forEach(file -> { + try (JsonReader reader = new JsonReader(new FileReader(file))) { + resource.put(file.getName() + keySuffix, gson.fromJson(reader, Object.class)); + } catch (Exception e) { + log.error( + "Error while reading file {} with message {} with cause", + file.toPath(), + e.getMessage(), + e.getCause()); + } + }); + } + return resource; + } + + /** + * This method will read the content of the file as a plain text and does not apply the gson to json transformation + * + * @param filePath file path for files on which read operation will be performed + * @return content of the file in the path + */ + private String readFileAsString(Path filePath) { + String data = CommonConstants.EMPTY_STRING; + try { + data = FileUtils.readFileToString(filePath.toFile(), "UTF-8"); + } catch (IOException e) { + log.error("Error while reading the file from git repo {} ", e.getMessage()); + } + return data; + } + + /** + * This method is to read the content for action and actionCollection or any nested resources which has the new structure - v3 + * Where the user written JS Object code and the metadata is split into to different files + * + * @param directoryPath file path for files on which read operation will be performed + * @return resources stored in the directory + */ + private Map readActionCollection( + Path directoryPath, Gson gson, String keySuffix, Map actionCollectionBodyMap) { + Map resource = new HashMap<>(); + File directory = directoryPath.toFile(); + if (directory.isDirectory()) { + for (File dirFile : Objects.requireNonNull(directory.listFiles())) { + String resourceName = dirFile.getName(); + Path resourcePath = + directoryPath.resolve(resourceName).resolve(resourceName + CommonConstants.JS_EXTENSION); + String body = CommonConstants.EMPTY_STRING; + if (resourcePath.toFile().exists()) { + body = readFileAsString(resourcePath); + } + Object file = readFile( + directoryPath + .resolve(resourceName) + .resolve(CommonConstants.METADATA + CommonConstants.JSON_EXTENSION), + gson); + actionCollectionBodyMap.put(resourceName + keySuffix, body); + resource.put(resourceName + keySuffix, file); + } + } + return resource; + } + + /** + * This method is to read the content for action and actionCollection or any nested resources which has the new structure - v4 + * Where the user queries and the metadata is split into to different files + * + * @param directoryPath directory path for files on which read operation will be performed + * @param gson + * @return resources stored in the directory + */ + private Map readAction( + Path directoryPath, Gson gson, String keySuffix, Map actionCollectionBodyMap) { + Map resource = new HashMap<>(); + File directory = directoryPath.toFile(); + if (directory.isDirectory()) { + for (File dirFile : Objects.requireNonNull(directory.listFiles())) { + String resourceName = dirFile.getName(); + String body = CommonConstants.EMPTY_STRING; + Path queryPath = + directoryPath.resolve(resourceName).resolve(resourceName + CommonConstants.TEXT_FILE_EXTENSION); + if (queryPath.toFile().exists()) { + body = readFileAsString(queryPath); + } + Object file = readFile( + directoryPath + .resolve(resourceName) + .resolve(CommonConstants.METADATA + CommonConstants.JSON_EXTENSION), + gson); + actionCollectionBodyMap.put(resourceName + keySuffix, body); + resource.put(resourceName + keySuffix, file); + } + } + return resource; + } + + private Object readPageMetadata(Path directoryPath, Gson gson) { + return readFile(directoryPath.resolve(directoryPath.toFile().getName() + CommonConstants.JSON_EXTENSION), gson); + } + + private ApplicationGitReference fetchApplicationReference(Path baseRepoPath, Gson gson) { + ApplicationGitReference applicationGitReference = new ApplicationGitReference(); + // Extract application metadata from the json + Object metadata = + readFile(baseRepoPath.resolve(CommonConstants.METADATA + CommonConstants.JSON_EXTENSION), gson); + Integer fileFormatVersion = getFileFormatVersion(metadata); + // Check if fileFormat of the saved files in repo is compatible + if (!isFileFormatCompatible(fileFormatVersion)) { + throw new AppsmithPluginException(AppsmithPluginError.INCOMPATIBLE_FILE_FORMAT); + } + // Extract application data from the json + applicationGitReference.setApplication( + readFile(baseRepoPath.resolve(CommonConstants.APPLICATION + CommonConstants.JSON_EXTENSION), gson)); + applicationGitReference.setTheme( + readFile(baseRepoPath.resolve(CommonConstants.THEME + CommonConstants.JSON_EXTENSION), gson)); + Path pageDirectory = baseRepoPath.resolve(PAGE_DIRECTORY); + // Reconstruct application from given file format + switch (fileFormatVersion) { + case 1: + // Extract actions + applicationGitReference.setActions( + readFiles(baseRepoPath.resolve(ACTION_DIRECTORY), gson, CommonConstants.EMPTY_STRING)); + // Extract actionCollections + applicationGitReference.setActionCollections(readFiles( + baseRepoPath.resolve(ACTION_COLLECTION_DIRECTORY), gson, CommonConstants.EMPTY_STRING)); + // Extract pages + applicationGitReference.setPages(readFiles(pageDirectory, gson, CommonConstants.EMPTY_STRING)); + // Extract datasources + applicationGitReference.setDatasources( + readFiles(baseRepoPath.resolve(DATASOURCE_DIRECTORY), gson, CommonConstants.EMPTY_STRING)); + break; + + case 2: + case 3: + case 4: + updateGitApplicationReference( + baseRepoPath, gson, applicationGitReference, pageDirectory, fileFormatVersion); + break; + + case 5: + updateGitApplicationReferenceV2( + baseRepoPath, gson, applicationGitReference, pageDirectory, fileFormatVersion); + break; + + default: + } + applicationGitReference.setMetadata(metadata); + + Path jsLibDirectory = baseRepoPath.resolve(JS_LIB_DIRECTORY); + Map jsLibrariesMap = readFiles(jsLibDirectory, gson, CommonConstants.EMPTY_STRING); + applicationGitReference.setJsLibraries(jsLibrariesMap); + + return applicationGitReference; + } + + @Deprecated + private void updateGitApplicationReference( + Path baseRepoPath, + Gson gson, + ApplicationGitReference applicationGitReference, + Path pageDirectory, + int fileFormatVersion) { + // Extract pages and nested actions and actionCollections + File directory = pageDirectory.toFile(); + Map pageMap = new HashMap<>(); + Map actionMap = new HashMap<>(); + Map actionBodyMap = new HashMap<>(); + Map actionCollectionMap = new HashMap<>(); + Map actionCollectionBodyMap = new HashMap<>(); + if (directory.isDirectory()) { + // Loop through all the directories and nested directories inside the pages directory to extract + // pages, actions and actionCollections from the JSON files + for (File page : Objects.requireNonNull(directory.listFiles())) { + pageMap.put( + page.getName(), + readFile(page.toPath().resolve(CommonConstants.CANVAS + CommonConstants.JSON_EXTENSION), gson)); + + if (fileFormatVersion >= 4) { + actionMap.putAll( + readAction(page.toPath().resolve(ACTION_DIRECTORY), gson, page.getName(), actionBodyMap)); + } else { + actionMap.putAll(readFiles(page.toPath().resolve(ACTION_DIRECTORY), gson, page.getName())); + } + + if (fileFormatVersion >= 3) { + actionCollectionMap.putAll(readActionCollection( + page.toPath().resolve(ACTION_COLLECTION_DIRECTORY), + gson, + page.getName(), + actionCollectionBodyMap)); + } else { + actionCollectionMap.putAll( + readFiles(page.toPath().resolve(ACTION_COLLECTION_DIRECTORY), gson, page.getName())); + } + } + } + applicationGitReference.setActions(actionMap); + applicationGitReference.setActionBody(actionBodyMap); + applicationGitReference.setActionCollections(actionCollectionMap); + applicationGitReference.setActionCollectionBody(actionCollectionBodyMap); + applicationGitReference.setPages(pageMap); + // Extract datasources + applicationGitReference.setDatasources( + readFiles(baseRepoPath.resolve(DATASOURCE_DIRECTORY), gson, CommonConstants.EMPTY_STRING)); + } + + private Integer getFileFormatVersion(Object metadata) { + if (metadata == null) { + return 1; + } + Gson gson = new Gson(); + JsonObject json = gson.fromJson(gson.toJson(metadata), JsonObject.class); + JsonElement fileFormatVersion = json.get(CommonConstants.FILE_FORMAT_VERSION); + return fileFormatVersion.getAsInt(); + } + + public static boolean isFileFormatCompatible(int savedFileFormat) { + return savedFileFormat <= CommonConstants.fileFormatVersion; + } + + protected void updateGitApplicationReferenceV2( + Path baseRepoPath, + Gson gson, + ApplicationGitReference applicationGitReference, + Path pageDirectory, + int fileFormatVersion) { + // Extract pages and nested actions and actionCollections + File directory = pageDirectory.toFile(); + Map pageMap = new HashMap<>(); + Map pageDsl = new HashMap<>(); + Map actionMap = new HashMap<>(); + Map actionBodyMap = new HashMap<>(); + Map actionCollectionMap = new HashMap<>(); + Map moduleInstanceMap = new HashMap<>(); + Map actionCollectionBodyMap = new HashMap<>(); + if (directory.isDirectory()) { + // Loop through all the directories and nested directories inside the pages directory to extract + // pages, actions and actionCollections from the JSON files + for (File page : Objects.requireNonNull(directory.listFiles())) { + if (page.isDirectory()) { + pageMap.put(page.getName(), readPageMetadata(page.toPath(), gson)); + + JSONObject mainContainer = getMainContainer(pageMap.get(page.getName()), gson); + + // Read widgets data recursively from the widgets directory + Map widgetsData = readWidgetsData( + page.toPath().resolve(CommonConstants.WIDGETS).toString()); + // Construct the nested DSL from the widgets data + Map> parentDirectories = DSLTransformerHelper.calculateParentDirectories( + widgetsData.keySet().stream().toList()); + JSONObject nestedDSL = + DSLTransformerHelper.getNestedDSL(widgetsData, parentDirectories, mainContainer); + pageDsl.put(page.getName(), nestedDSL.toString()); + actionMap.putAll( + readAction(page.toPath().resolve(ACTION_DIRECTORY), gson, page.getName(), actionBodyMap)); + actionCollectionMap.putAll(readActionCollection( + page.toPath().resolve(ACTION_COLLECTION_DIRECTORY), + gson, + page.getName(), + actionCollectionBodyMap)); + } + } + } + applicationGitReference.setActions(actionMap); + applicationGitReference.setActionBody(actionBodyMap); + applicationGitReference.setActionCollections(actionCollectionMap); + applicationGitReference.setActionCollectionBody(actionCollectionBodyMap); + applicationGitReference.setPages(pageMap); + applicationGitReference.setPageDsl(pageDsl); + // Extract datasources + applicationGitReference.setDatasources( + readFiles(baseRepoPath.resolve(DATASOURCE_DIRECTORY), gson, CommonConstants.EMPTY_STRING)); + } + + private Map readWidgetsData(String directoryPath) { + Map jsonMap = new HashMap<>(); + File directory = new File(directoryPath); + + if (!directory.isDirectory()) { + log.error("Error reading directory: {}", directoryPath); + return jsonMap; + } + + try { + readFilesRecursively(directory, jsonMap, directoryPath); + } catch (IOException exception) { + log.error("Error reading directory: {}, error message {}", directoryPath, exception.getMessage()); + } + + return jsonMap; + } + + private void readFilesRecursively(File directory, Map jsonMap, String rootPath) + throws IOException { + File[] files = directory.listFiles(); + if (files == null) { + return; + } + + for (File file : files) { + if (file.isFile()) { + String filePath = file.getAbsolutePath(); + String relativePath = filePath.replace(rootPath, CommonConstants.EMPTY_STRING); + relativePath = CommonConstants.DELIMITER_PATH + + CommonConstants.MAIN_CONTAINER + + relativePath.substring(relativePath.indexOf("//") + 1); + try { + String fileContent = new String(Files.readAllBytes(file.toPath())); + JSONObject jsonObject = new JSONObject(fileContent); + jsonMap.put(relativePath, jsonObject); + } catch (IOException exception) { + log.error("Error reading file: {}, error message {}", filePath, exception.getMessage()); + } + } else if (file.isDirectory()) { + readFilesRecursively(file, jsonMap, rootPath); + } + } + } + + private void deleteWidgets(File directory, Map validWidgetToParentMap) { + File[] files = directory.listFiles(); + if (files == null) { + return; + } + + for (File file : files) { + if (file.isDirectory()) { + deleteWidgets(file, validWidgetToParentMap); + } + + String name = file.getName().replace(CommonConstants.JSON_EXTENSION, CommonConstants.EMPTY_STRING); + // If input widget was inside a container before, but the user moved it out of the container + // then we need to delete the widget from the container directory + // The check here is to validate if the parent is correct or not + if (!validWidgetToParentMap.containsKey(name)) { + if (file.isDirectory()) { + deleteDirectory(file.toPath()); + } else { + deleteFile(file.toPath()); + } + } else if (!file.getParentFile().getPath().equals(validWidgetToParentMap.get(name)) + && !file.getPath().equals(validWidgetToParentMap.get(name))) { + if (file.isDirectory()) { + deleteDirectory(file.toPath()); + } else { + deleteFile(file.toPath()); + } + } + } + } + + private JSONObject getMainContainer(Object pageJson, Gson gson) { + JSONObject pageJSON = new JSONObject(gson.toJson(pageJson)); + JSONArray layouts = pageJSON.getJSONObject("unpublishedPage").getJSONArray("layouts"); + return layouts.getJSONObject(0).getJSONObject("dsl"); + } + + @Override + public Mono deleteIndexLockFile(Path path, int validTimeInSeconds) { + // Check the time created of the index.lock file + // If the File is stale for more than validTime, then delete the file + try { + BasicFileAttributes attr = Files.readAttributes(path, BasicFileAttributes.class); + FileTime fileTime = attr.creationTime(); + Instant now = Instant.now(); + Instant validCreateTime = now.minusSeconds(validTimeInSeconds); + if (fileTime.toInstant().isBefore(validCreateTime)) { + // Add base repo path + path = Paths.get(path + ".lock"); + deleteFile(path); + return Mono.just(now.minusMillis(fileTime.toMillis()).getEpochSecond()); + } else { + return Mono.just(0L); + } + } catch (IOException ex) { + log.error("Error reading index.lock file: {}", ex.getMessage()); + return Mono.just(0L); + } + } + + /** + * We use UID string for custom js lib. UID strings are in this format: {libname}_{url to the lib src}. + * This method converts this uid string into a valid file name so that there is no unsupported character in the + * file name for any OS. + * This method returns a string in the format: {libname}_{base64 encoded hash of uid string} + * + * @param uidString UID string value of a JS lib + * @return String + */ + public static String getJsLibFileName(String uidString) { + int firstUnderscoreIndex = uidString.indexOf('_'); // this finds the first occurrence of "_" + String prefix; + if (firstUnderscoreIndex != -1) { + prefix = uidString.substring(0, firstUnderscoreIndex); // we're getting the prefix from the uidString + } else { + prefix = "jslib"; + } + + StringBuilder stringBuilder = new StringBuilder(prefix); + stringBuilder.append("_"); + try { + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + byte[] hash = digest.digest(uidString.getBytes(StandardCharsets.UTF_8)); + stringBuilder.append(Base64.getUrlEncoder().withoutPadding().encodeToString(hash)); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException("Failed to hash URL string", e); + } + return stringBuilder.toString(); + } +} diff --git a/app/server/appsmith-git/src/main/java/com/appsmith/git/service/GitExecutorImpl.java b/app/server/appsmith-git/src/main/java/com/appsmith/git/service/GitExecutorImpl.java index 869630eb42..f4309bff23 100644 --- a/app/server/appsmith-git/src/main/java/com/appsmith/git/service/GitExecutorImpl.java +++ b/app/server/appsmith-git/src/main/java/com/appsmith/git/service/GitExecutorImpl.java @@ -1,913 +1,16 @@ package com.appsmith.git.service; -import com.appsmith.external.constants.AnalyticsEvents; -import com.appsmith.external.constants.ErrorReferenceDocUrl; -import com.appsmith.external.dtos.GitBranchDTO; -import com.appsmith.external.dtos.GitLogDTO; -import com.appsmith.external.dtos.GitStatusDTO; -import com.appsmith.external.dtos.MergeStatusDTO; import com.appsmith.external.git.GitExecutor; -import com.appsmith.external.helpers.Stopwatch; import com.appsmith.git.configurations.GitServiceConfig; -import com.appsmith.git.constants.AppsmithBotAsset; -import com.appsmith.git.constants.CommonConstants; -import com.appsmith.git.constants.Constraint; -import com.appsmith.git.constants.GitDirectories; -import com.appsmith.git.helpers.RepositoryHelper; -import com.appsmith.git.helpers.SshTransportConfigCallback; -import com.appsmith.git.helpers.StopwatchHelpers; -import lombok.RequiredArgsConstructor; +import com.appsmith.git.service.ce.GitExecutorCEImpl; import lombok.extern.slf4j.Slf4j; -import org.eclipse.jgit.api.CreateBranchCommand; -import org.eclipse.jgit.api.Git; -import org.eclipse.jgit.api.ListBranchCommand; -import org.eclipse.jgit.api.MergeCommand; -import org.eclipse.jgit.api.MergeResult; -import org.eclipse.jgit.api.RebaseCommand; -import org.eclipse.jgit.api.RebaseResult; -import org.eclipse.jgit.api.ResetCommand; -import org.eclipse.jgit.api.Status; -import org.eclipse.jgit.api.TransportConfigCallback; -import org.eclipse.jgit.api.errors.CheckoutConflictException; -import org.eclipse.jgit.api.errors.GitAPIException; -import org.eclipse.jgit.lib.BranchTrackingStatus; -import org.eclipse.jgit.lib.PersonIdent; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.lib.StoredConfig; -import org.eclipse.jgit.merge.MergeStrategy; -import org.eclipse.jgit.revwalk.RevCommit; -import org.eclipse.jgit.transport.RefSpec; -import org.eclipse.jgit.util.StringUtils; import org.springframework.stereotype.Component; -import org.springframework.util.FileSystemUtils; -import reactor.core.publisher.Mono; -import reactor.core.scheduler.Scheduler; -import reactor.core.scheduler.Schedulers; -import java.io.File; -import java.io.IOException; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.time.Duration; -import java.time.ZoneId; -import java.time.ZoneOffset; -import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Date; -import java.util.HashSet; -import java.util.List; -import java.util.Optional; -import java.util.Set; - -import static com.appsmith.git.constants.CommonConstants.FILE_MIGRATION_MESSAGE; - -@RequiredArgsConstructor @Component @Slf4j -public class GitExecutorImpl implements GitExecutor { +public class GitExecutorImpl extends GitExecutorCEImpl implements GitExecutor { - private final RepositoryHelper repositoryHelper = new RepositoryHelper(); - - private final GitServiceConfig gitServiceConfig; - - public static final DateTimeFormatter ISO_FORMATTER = - DateTimeFormatter.ISO_INSTANT.withZone(ZoneId.from(ZoneOffset.UTC)); - - private final Scheduler scheduler = Schedulers.boundedElastic(); - - private static final String SUCCESS_MERGE_STATUS = "This branch has no conflicts with the base branch."; - - /** - * This method will handle the git-commit functionality. Under the hood it checks if the repo has already been - * initialised and will be initialised if git repo is not present - * @param path parent path to repo - * @param commitMessage message which will be registered for this commit - * @param authorName author details - * @param authorEmail author details - * @param doAmend To amend with the previous commit - * @return if the commit was successful - */ - @Override - public Mono commitApplication( - Path path, - String commitMessage, - String authorName, - String authorEmail, - boolean isSuffixedPath, - boolean doAmend) { - - final String finalAuthorName = - StringUtils.isEmptyOrNull(authorName) ? AppsmithBotAsset.APPSMITH_BOT_USERNAME : authorName; - final String finalAuthorEmail = - StringUtils.isEmptyOrNull(authorEmail) ? AppsmithBotAsset.APPSMITH_BOT_EMAIL : authorEmail; - return Mono.fromCallable(() -> { - log.debug("Trying to commit to local repo path, {}", path); - Path repoPath = path; - if (Boolean.TRUE.equals(isSuffixedPath)) { - repoPath = createRepoPath(repoPath); - } - Stopwatch processStopwatch = - StopwatchHelpers.startStopwatch(repoPath, AnalyticsEvents.GIT_COMMIT.getEventName()); - // Just need to open a repository here and make a commit - try (Git git = Git.open(repoPath.toFile())) { - // Stage all the files added and modified - git.add().addFilepattern(".").call(); - // Stage modified and deleted files - git.add().setUpdate(true).addFilepattern(".").call(); - - // Commit the changes - git.commit() - .setMessage(commitMessage) - // Only make a commit if there are any updates - .setAllowEmpty(false) - .setAuthor(finalAuthorName, finalAuthorEmail) - .setCommitter(finalAuthorName, finalAuthorEmail) - .setAmend(doAmend) - .call(); - processStopwatch.stopAndLogTimeInMillis(); - return "Committed successfully!"; - } - }) - .timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS)) - .subscribeOn(scheduler); - } - - /** - * Method to create a new repository to provided path - * @param repoPath path where new repo needs to be created - * @return if the operation was successful - */ - @Override - public boolean createNewRepository(Path repoPath) throws GitAPIException { - // create new repo to the mentioned path - log.debug("Trying to create new repository: {}", repoPath); - Git.init().setDirectory(repoPath.toFile()).call(); - return true; - } - - /** - * Method to get the commit history - * @param repoSuffix Path used to generate the repo url specific to the application for which the commit history is requested - * @return list of git commits - */ - @Override - public Mono> getCommitHistory(Path repoSuffix) { - return Mono.fromCallable(() -> { - log.debug(Thread.currentThread().getName() + ": get commit history for " + repoSuffix); - List commitLogs = new ArrayList<>(); - Path repoPath = createRepoPath(repoSuffix); - Stopwatch processStopwatch = StopwatchHelpers.startStopwatch( - repoPath, AnalyticsEvents.GIT_COMMIT_HISTORY.getEventName()); - try (Git git = Git.open(repoPath.toFile())) { - Iterable gitLogs = git.log() - .setMaxCount(Constraint.MAX_COMMIT_LOGS) - .call(); - gitLogs.forEach(revCommit -> { - PersonIdent author = revCommit.getAuthorIdent(); - GitLogDTO gitLog = new GitLogDTO( - revCommit.getName(), - author.getName(), - author.getEmailAddress(), - revCommit.getFullMessage(), - ISO_FORMATTER.format(new Date(revCommit.getCommitTime() * 1000L).toInstant())); - processStopwatch.stopAndLogTimeInMillis(); - commitLogs.add(gitLog); - }); - return commitLogs; - } - }) - .timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS)) - .subscribeOn(scheduler); - } - - @Override - public Path createRepoPath(Path suffix) { - return Paths.get(gitServiceConfig.getGitRootPath()).resolve(suffix); - } - - /** - * Method to push changes to remote repo - * @param repoSuffix Path used to generate the repo url specific to the application which needs to be pushed to remote - * @param remoteUrl remote repo url - * @param publicKey - * @param privateKey - * @return Success message - */ - @Override - public Mono pushApplication( - Path repoSuffix, String remoteUrl, String publicKey, String privateKey, String branchName) { - // We can safely assume that repo has been already initialised either in commit or clone flow and can directly - // open the repo - return Mono.fromCallable(() -> { - log.debug(Thread.currentThread().getName() + ": pushing changes to remote " + remoteUrl); - // open the repo - Path baseRepoPath = createRepoPath(repoSuffix); - Stopwatch processStopwatch = - StopwatchHelpers.startStopwatch(baseRepoPath, AnalyticsEvents.GIT_PUSH.getEventName()); - try (Git git = Git.open(baseRepoPath.toFile())) { - TransportConfigCallback transportConfigCallback = - new SshTransportConfigCallback(privateKey, publicKey); - - StringBuilder result = new StringBuilder("Pushed successfully with status : "); - git.push() - .setTransportConfigCallback(transportConfigCallback) - .setRemote(remoteUrl) - .call() - .forEach(pushResult -> pushResult - .getRemoteUpdates() - .forEach(remoteRefUpdate -> { - result.append(remoteRefUpdate.getStatus()) - .append(","); - if (!StringUtils.isEmptyOrNull(remoteRefUpdate.getMessage())) { - result.append(remoteRefUpdate.getMessage()) - .append(","); - } - })); - // We can support username and password in future if needed - // pushCommand.setCredentialsProvider(new UsernamePasswordCredentialsProvider("username", - // "password")); - processStopwatch.stopAndLogTimeInMillis(); - return result.substring(0, result.length() - 1); - } - }) - .timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS)) - .subscribeOn(scheduler); - } - - /** Clone the repo to the file path : container-volume/orgId/defaultAppId/repo/applicationData - * - * @param repoSuffix combination of orgId, defaultId and repoName - * @param remoteUrl ssh url of the git repo(we support cloning via ssh url only with deploy key) - * @param privateKey generated by us and specific to the defaultApplication - * @param publicKey generated by us and specific to the defaultApplication - * @return defaultBranchName of the repo - * */ - @Override - public Mono cloneApplication(Path repoSuffix, String remoteUrl, String privateKey, String publicKey) { - - Stopwatch processStopwatch = - StopwatchHelpers.startStopwatch(repoSuffix, AnalyticsEvents.GIT_CLONE.getEventName()); - return Mono.fromCallable(() -> { - log.debug(Thread.currentThread().getName() + ": Cloning the repo from the remote " + remoteUrl); - final TransportConfigCallback transportConfigCallback = - new SshTransportConfigCallback(privateKey, publicKey); - File file = Paths.get(gitServiceConfig.getGitRootPath()) - .resolve(repoSuffix) - .toFile(); - while (file.exists()) { - FileSystemUtils.deleteRecursively(file); - } - - Git git = Git.cloneRepository() - .setURI(remoteUrl) - .setTransportConfigCallback(transportConfigCallback) - .setDirectory(file) - .call(); - String branchName = git.getRepository().getBranch(); - - repositoryHelper.updateRemoteBranchTrackingConfig(branchName, git); - git.close(); - processStopwatch.stopAndLogTimeInMillis(); - return branchName; - }) - .timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS)) - .subscribeOn(scheduler); - } - - @Override - public Mono createAndCheckoutToBranch(Path repoSuffix, String branchName) { - // We can safely assume that repo has been already initialised either in commit or clone flow and can directly - // open the repo - Stopwatch processStopwatch = - StopwatchHelpers.startStopwatch(repoSuffix, AnalyticsEvents.GIT_CREATE_BRANCH.getEventName()); - return Mono.fromCallable(() -> { - log.debug(Thread.currentThread().getName() + ": Creating branch " + branchName + "for the repo " - + repoSuffix); - // open the repo - Path baseRepoPath = createRepoPath(repoSuffix); - try (Git git = Git.open(baseRepoPath.toFile())) { - // Create and checkout to new branch - git.checkout() - .setCreateBranch(Boolean.TRUE) - .setName(branchName) - .setUpstreamMode(CreateBranchCommand.SetupUpstreamMode.TRACK) - .call(); - - repositoryHelper.updateRemoteBranchTrackingConfig(branchName, git); - processStopwatch.stopAndLogTimeInMillis(); - return git.getRepository().getBranch(); - } - }) - .timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS)) - .subscribeOn(scheduler); - } - - @Override - public Mono deleteBranch(Path repoSuffix, String branchName) { - // We can safely assume that repo has been already initialised either in commit or clone flow and can directly - // open the repo - Stopwatch processStopwatch = - StopwatchHelpers.startStopwatch(repoSuffix, AnalyticsEvents.GIT_DELETE_BRANCH.getEventName()); - return Mono.fromCallable(() -> { - log.debug(Thread.currentThread().getName() + ": Deleting branch " + branchName + "for the repo " - + repoSuffix); - // open the repo - Path baseRepoPath = createRepoPath(repoSuffix); - try (Git git = Git.open(baseRepoPath.toFile())) { - // Create and checkout to new branch - List deleteBranchList = git.branchDelete() - .setBranchNames(branchName) - .setForce(Boolean.TRUE) - .call(); - processStopwatch.stopAndLogTimeInMillis(); - if (deleteBranchList.isEmpty()) { - return Boolean.FALSE; - } - return Boolean.TRUE; - } - }) - .timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS)) - .subscribeOn(scheduler); - } - - @Override - public Mono checkoutToBranch(Path repoSuffix, String branchName) { - - Stopwatch processStopwatch = - StopwatchHelpers.startStopwatch(repoSuffix, AnalyticsEvents.GIT_CHECKOUT.getEventName()); - return Mono.fromCallable(() -> { - log.debug(Thread.currentThread().getName() + ": Switching to the branch " + branchName); - // We can safely assume that repo has been already initialised either in commit or clone flow and - // can directly - // open the repo - Path baseRepoPath = createRepoPath(repoSuffix); - try (Git git = Git.open(baseRepoPath.toFile())) { - if (StringUtils.equalsIgnoreCase( - branchName, git.getRepository().getBranch())) { - return Boolean.TRUE; - } - // Create and checkout to new branch - String checkedOutBranch = git.checkout() - .setCreateBranch(Boolean.FALSE) - .setName(branchName) - .setUpstreamMode(CreateBranchCommand.SetupUpstreamMode.SET_UPSTREAM) - .call() - .getName(); - processStopwatch.stopAndLogTimeInMillis(); - return StringUtils.equalsIgnoreCase(checkedOutBranch, "refs/heads/" + branchName); - } catch (Exception e) { - throw new Exception(e); - } - }) - .timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS)) - .subscribeOn(scheduler); - } - - @Override - public Mono pullApplication( - Path repoSuffix, String remoteUrl, String branchName, String privateKey, String publicKey) - throws IOException { - - Stopwatch processStopwatch = - StopwatchHelpers.startStopwatch(repoSuffix, AnalyticsEvents.GIT_PULL.getEventName()); - TransportConfigCallback transportConfigCallback = new SshTransportConfigCallback(privateKey, publicKey); - - try (Git git = Git.open(createRepoPath(repoSuffix).toFile())) { - return Mono.fromCallable(() -> { - log.debug(Thread.currentThread().getName() + ": Pull changes from remote " + remoteUrl - + " for the branch " + branchName); - // checkout the branch on which the merge command is run - git.checkout() - .setName(branchName) - .setCreateBranch(false) - .call(); - MergeResult mergeResult = git.pull() - .setRemoteBranchName(branchName) - .setTransportConfigCallback(transportConfigCallback) - .setFastForward(MergeCommand.FastForwardMode.FF) - .call() - .getMergeResult(); - MergeStatusDTO mergeStatus = new MergeStatusDTO(); - Long count = - Arrays.stream(mergeResult.getMergedCommits()).count(); - if (mergeResult.getMergeStatus().isSuccessful()) { - mergeStatus.setMergeAble(true); - mergeStatus.setStatus(count + " commits merged from origin/" + branchName); - } else { - // If there are conflicts add the conflicting file names to the response structure - mergeStatus.setMergeAble(false); - List mergeConflictFiles = new ArrayList<>(); - if (!Optional.ofNullable(mergeResult.getConflicts()).isEmpty()) { - mergeConflictFiles.addAll( - mergeResult.getConflicts().keySet()); - } - mergeStatus.setConflictingFiles(mergeConflictFiles); - // On merge conflicts abort the merge => git merge --abort - git.getRepository().writeMergeCommitMsg(null); - git.getRepository().writeMergeHeads(null); - processStopwatch.stopAndLogTimeInMillis(); - throw new org.eclipse.jgit.errors.CheckoutConflictException(mergeConflictFiles.toString()); - } - processStopwatch.stopAndLogTimeInMillis(); - return mergeStatus; - }) - .onErrorResume(error -> { - try { - return resetToLastCommit(git).flatMap(ignore -> Mono.error(error)); - } catch (GitAPIException e) { - log.error("Error for hard resetting to latest commit {0}", e); - return Mono.error(e); - } - }) - .timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS)) - .subscribeOn(scheduler); - } - } - - @Override - public Mono> listBranches(Path repoSuffix) { - Path baseRepoPath = createRepoPath(repoSuffix); - return Mono.fromCallable(() -> { - log.debug(Thread.currentThread().getName() + ": Get branches for the application " + repoSuffix); - Git git = Git.open(baseRepoPath.toFile()); - List refList = git.branchList() - .setListMode(ListBranchCommand.ListMode.ALL) - .call(); - - List branchList = new ArrayList<>(); - GitBranchDTO gitBranchDTO = new GitBranchDTO(); - if (refList.isEmpty()) { - gitBranchDTO.setBranchName(git.getRepository().getBranch()); - branchList.add(gitBranchDTO); - } else { - for (Ref ref : refList) { - // if (!ref.getName().equals(defaultBranch)) { - gitBranchDTO = new GitBranchDTO(); - gitBranchDTO.setBranchName(ref.getName() - .replace("refs/", "") - .replace("heads/", "") - .replace("remotes/", "")); - branchList.add(gitBranchDTO); - } - } - git.close(); - return branchList; - }) - .timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS)) - .subscribeOn(scheduler); - } - - @Override - public Mono getRemoteDefaultBranch(Path repoSuffix, String remoteUrl, String privateKey, String publicKey) { - Path baseRepoPath = createRepoPath(repoSuffix); - return Mono.fromCallable(() -> { - TransportConfigCallback transportConfigCallback = - new SshTransportConfigCallback(privateKey, publicKey); - Git git = Git.open(baseRepoPath.toFile()); - - return git.lsRemote() - .setRemote(remoteUrl) - .setTransportConfigCallback(transportConfigCallback) - .callAsMap() - .get("HEAD") - .getTarget() - .getName() - .replace("refs/heads/", ""); - }) - .timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS)) - .subscribeOn(scheduler); - } - - /** - * This method will handle the git-status functionality - * - * @param repoPath Path to actual repo - * @param branchName branch name for which the status is required - * @return Map of file names those are modified, conflicted etc. - */ - @Override - public Mono getStatus(Path repoPath, String branchName) { - Stopwatch processStopwatch = - StopwatchHelpers.startStopwatch(repoPath, AnalyticsEvents.GIT_STATUS.getEventName()); - return Mono.fromCallable(() -> { - try (Git git = Git.open(repoPath.toFile())) { - log.debug(Thread.currentThread().getName() + ": Get status for repo " + repoPath + ", branch " - + branchName); - Status status = git.status().call(); - GitStatusDTO response = new GitStatusDTO(); - Set modifiedAssets = new HashSet<>(); - modifiedAssets.addAll(status.getModified()); - modifiedAssets.addAll(status.getAdded()); - modifiedAssets.addAll(status.getRemoved()); - modifiedAssets.addAll(status.getUncommittedChanges()); - modifiedAssets.addAll(status.getUntracked()); - response.setAdded(status.getAdded()); - response.setRemoved(status.getRemoved()); - - Set queriesModified = new HashSet<>(); - Set jsObjectsModified = new HashSet<>(); - Set pagesModified = new HashSet<>(); - int modifiedPages = 0; - int modifiedQueries = 0; - int modifiedJSObjects = 0; - int modifiedDatasources = 0; - int modifiedJSLibs = 0; - for (String x : modifiedAssets) { - // begins with pages and filename and parent name should be same or contains widgets - if (x.contains(CommonConstants.WIDGETS)) { - if (!pagesModified.contains(getPageName(x))) { - pagesModified.add(getPageName(x)); - modifiedPages++; - } - } else if (!x.contains(CommonConstants.WIDGETS) - && x.startsWith(GitDirectories.PAGE_DIRECTORY) - && !x.contains(GitDirectories.ACTION_DIRECTORY) - && !x.contains(GitDirectories.ACTION_COLLECTION_DIRECTORY)) { - if (!pagesModified.contains(getPageName(x))) { - pagesModified.add(getPageName(x)); - modifiedPages++; - } - } else if (x.contains(GitDirectories.ACTION_DIRECTORY + CommonConstants.DELIMITER_PATH)) { - String queryName = - x.split(GitDirectories.ACTION_DIRECTORY + CommonConstants.DELIMITER_PATH)[1]; - int position = queryName.indexOf(CommonConstants.DELIMITER_PATH); - if (position != -1) { - queryName = queryName.substring(0, position); - String pageName = x.split(CommonConstants.DELIMITER_PATH)[1]; - if (!queriesModified.contains(pageName + queryName)) { - queriesModified.add(pageName + queryName); - modifiedQueries++; - } - } - } else if (x.contains( - GitDirectories.ACTION_COLLECTION_DIRECTORY + CommonConstants.DELIMITER_PATH) - && !x.endsWith(CommonConstants.JSON_EXTENSION)) { - String queryName = x.substring(x.lastIndexOf(CommonConstants.DELIMITER_PATH) + 1); - String pageName = x.split(CommonConstants.DELIMITER_PATH)[1]; - if (!jsObjectsModified.contains(pageName + queryName)) { - jsObjectsModified.add(pageName + queryName); - modifiedJSObjects++; - } - } else if (x.contains( - GitDirectories.DATASOURCE_DIRECTORY + CommonConstants.DELIMITER_PATH)) { - modifiedDatasources++; - } else if (x.contains(GitDirectories.JS_LIB_DIRECTORY + CommonConstants.DELIMITER_PATH)) { - modifiedJSLibs++; - // remove this code in future when all the older format js libs are migrated to new - // format - - if (x.contains("js.json")) { - /* - As this updated filename has color(:), it means this is the older format js - lib file that we're going to rename with the format without colon. - Hence, we need to show a message to user saying this might be a system level change. - */ - response.setMigrationMessage(FILE_MIGRATION_MESSAGE); - } - } - } - response.setModified(modifiedAssets); - response.setConflicting(status.getConflicting()); - response.setIsClean(status.isClean()); - response.setModifiedPages(modifiedPages); - response.setModifiedQueries(modifiedQueries); - response.setModifiedJSObjects(modifiedJSObjects); - response.setModifiedDatasources(modifiedDatasources); - response.setModifiedJSLibs(modifiedJSLibs); - - BranchTrackingStatus trackingStatus = BranchTrackingStatus.of(git.getRepository(), branchName); - if (trackingStatus != null) { - response.setAheadCount(trackingStatus.getAheadCount()); - response.setBehindCount(trackingStatus.getBehindCount()); - response.setRemoteBranch(trackingStatus.getRemoteTrackingBranch()); - } else { - log.debug( - "Remote tracking details not present for branch: {}, repo: {}", - branchName, - repoPath); - response.setAheadCount(0); - response.setBehindCount(0); - response.setRemoteBranch("untracked"); - } - - // Remove modified changes from current branch so that checkout to other branches will be - // possible - if (!status.isClean()) { - return resetToLastCommit(git).map(ref -> { - processStopwatch.stopAndLogTimeInMillis(); - return response; - }); - } - processStopwatch.stopAndLogTimeInMillis(); - return Mono.just(response); - } - }) - .timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS)) - .flatMap(response -> response) - .subscribeOn(scheduler); - } - - private String getPageName(String path) { - String[] pathArray = path.split(CommonConstants.DELIMITER_PATH); - return pathArray[1]; - } - - @Override - public Mono mergeBranch(Path repoSuffix, String sourceBranch, String destinationBranch) { - return Mono.fromCallable(() -> { - Stopwatch processStopwatch = - StopwatchHelpers.startStopwatch(repoSuffix, AnalyticsEvents.GIT_MERGE.getEventName()); - log.debug(Thread.currentThread().getName() + ": Merge branch " + sourceBranch + " on " - + destinationBranch); - try (Git git = Git.open(createRepoPath(repoSuffix).toFile())) { - try { - // checkout the branch on which the merge command is run - git.checkout() - .setName(destinationBranch) - .setCreateBranch(false) - .call(); - - MergeResult mergeResult = git.merge() - .include(git.getRepository().findRef(sourceBranch)) - .setStrategy(MergeStrategy.RECURSIVE) - .call(); - processStopwatch.stopAndLogTimeInMillis(); - return mergeResult.getMergeStatus().name(); - } catch (GitAPIException e) { - // On merge conflicts abort the merge => git merge --abort - git.getRepository().writeMergeCommitMsg(null); - git.getRepository().writeMergeHeads(null); - processStopwatch.stopAndLogTimeInMillis(); - throw new Exception(e); - } - } - }) - .onErrorResume(error -> { - try { - return resetToLastCommit(repoSuffix, destinationBranch).thenReturn(error.getMessage()); - } catch (GitAPIException | IOException e) { - log.error("Error while hard resetting to latest commit {0}", e); - return Mono.error(e); - } - }) - .timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS)) - .subscribeOn(scheduler); - } - - @Override - public Mono fetchRemote( - Path repoSuffix, - String publicKey, - String privateKey, - boolean isRepoPath, - String branchName, - boolean isFetchAll) { - Stopwatch processStopwatch = - StopwatchHelpers.startStopwatch(repoSuffix, AnalyticsEvents.GIT_FETCH.getEventName()); - Path repoPath = Boolean.TRUE.equals(isRepoPath) ? repoSuffix : createRepoPath(repoSuffix); - return Mono.fromCallable(() -> { - TransportConfigCallback config = new SshTransportConfigCallback(privateKey, publicKey); - try (Git git = Git.open(repoPath.toFile())) { - String fetchMessages; - if (Boolean.TRUE.equals(isFetchAll)) { - fetchMessages = git.fetch() - .setRemoveDeletedRefs(true) - .setTransportConfigCallback(config) - .call() - .getMessages(); - } else { - RefSpec ref = - new RefSpec("refs/heads/" + branchName + ":refs/remotes/origin/" + branchName); - fetchMessages = git.fetch() - .setRefSpecs(ref) - .setRemoveDeletedRefs(true) - .setTransportConfigCallback(config) - .call() - .getMessages(); - } - processStopwatch.stopAndLogTimeInMillis(); - return fetchMessages; - } - }) - .onErrorResume(error -> { - log.error(error.getMessage()); - return Mono.error(error); - }) - .timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS)) - .subscribeOn(scheduler); - } - - @Override - public Mono isMergeBranch(Path repoSuffix, String sourceBranch, String destinationBranch) { - Stopwatch processStopwatch = - StopwatchHelpers.startStopwatch(repoSuffix, AnalyticsEvents.GIT_MERGE_CHECK.getEventName()); - return Mono.fromCallable(() -> { - log.debug( - Thread.currentThread().getName() - + ": Check mergeability for repo {} with src: {}, dest: {}", - repoSuffix, - sourceBranch, - destinationBranch); - - try (Git git = Git.open(createRepoPath(repoSuffix).toFile())) { - - // checkout the branch on which the merge command is run - try { - git.checkout() - .setName(destinationBranch) - .setCreateBranch(false) - .call(); - } catch (GitAPIException e) { - if (e instanceof CheckoutConflictException) { - MergeStatusDTO mergeStatus = new MergeStatusDTO(); - mergeStatus.setMergeAble(false); - mergeStatus.setConflictingFiles(((CheckoutConflictException) e).getConflictingPaths()); - processStopwatch.stopAndLogTimeInMillis(); - return mergeStatus; - } - } - - MergeResult mergeResult = git.merge() - .include(git.getRepository().findRef(sourceBranch)) - .setFastForward(MergeCommand.FastForwardMode.NO_FF) - .setCommit(false) - .call(); - - MergeStatusDTO mergeStatus = new MergeStatusDTO(); - if (mergeResult.getMergeStatus().isSuccessful()) { - mergeStatus.setMergeAble(true); - mergeStatus.setMessage(SUCCESS_MERGE_STATUS); - } else { - // If there aer conflicts add the conflicting file names to the response structure - mergeStatus.setMergeAble(false); - List mergeConflictFiles = - new ArrayList<>(mergeResult.getConflicts().keySet()); - mergeStatus.setConflictingFiles(mergeConflictFiles); - StringBuilder errorMessage = new StringBuilder(); - if (mergeResult.getMergeStatus().equals(MergeResult.MergeStatus.CONFLICTING)) { - errorMessage.append("Conflicts"); - } else { - errorMessage.append(mergeResult.getMergeStatus().toString()); - } - errorMessage - .append(" while merging branch: ") - .append(destinationBranch) - .append(" <= ") - .append(sourceBranch); - mergeStatus.setMessage(errorMessage.toString()); - mergeStatus.setReferenceDoc(ErrorReferenceDocUrl.GIT_MERGE_CONFLICT.getDocUrl()); - } - mergeStatus.setStatus(mergeResult.getMergeStatus().name()); - return mergeStatus; - } - }) - .flatMap(status -> { - try { - // Revert uncommitted changes if any - return resetToLastCommit(repoSuffix, destinationBranch).map(ignore -> { - processStopwatch.stopAndLogTimeInMillis(); - return status; - }); - } catch (GitAPIException | IOException e) { - log.error("Error for hard resetting to latest commit {0}", e); - return Mono.error(e); - } - }) - .timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS)) - .subscribeOn(scheduler); - } - - public Mono checkoutRemoteBranch(Path repoSuffix, String branchName) { - // We can safely assume that repo has been already initialised either in commit or clone flow and can directly - // open the repo - return Mono.fromCallable(() -> { - log.debug(Thread.currentThread().getName() + ": Checking out remote branch origin/" + branchName - + " for the repo " + repoSuffix); - // open the repo - Path baseRepoPath = createRepoPath(repoSuffix); - try (Git git = Git.open(baseRepoPath.toFile())) { - // Create and checkout to new branch - git.checkout() - .setCreateBranch(Boolean.TRUE) - .setName(branchName) - .setUpstreamMode(CreateBranchCommand.SetupUpstreamMode.TRACK) - .setStartPoint("origin/" + branchName) - .call(); - - StoredConfig config = git.getRepository().getConfig(); - config.setString("branch", branchName, "remote", "origin"); - config.setString("branch", branchName, "merge", "refs/heads/" + branchName); - config.save(); - return git.getRepository().getBranch(); - } - }) - .timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS)) - .subscribeOn(scheduler); - } - - @Override - public Mono testConnection(String publicKey, String privateKey, String remoteUrl) { - return Mono.fromCallable(() -> { - TransportConfigCallback transportConfigCallback = - new SshTransportConfigCallback(privateKey, publicKey); - Git.lsRemoteRepository() - .setTransportConfigCallback(transportConfigCallback) - .setRemote(remoteUrl) - .setHeads(true) - .setTags(true) - .call(); - return true; - }) - .timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS)) - .subscribeOn(scheduler); - } - - private Mono resetToLastCommit(Git git) throws GitAPIException { - Stopwatch processStopwatch = StopwatchHelpers.startStopwatch( - git.getRepository().getDirectory().toPath().getParent(), AnalyticsEvents.GIT_RESET.getEventName()); - return Mono.fromCallable(() -> { - // Remove tracked files - Ref ref = git.reset().setMode(ResetCommand.ResetType.HARD).call(); - // Remove untracked files - git.clean().setForce(true).setCleanDirectories(true).call(); - processStopwatch.stopAndLogTimeInMillis(); - return ref; - }) - .timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS)) - .subscribeOn(scheduler); - } - - public Mono resetToLastCommit(Path repoSuffix, String branchName) throws GitAPIException, IOException { - try (Git git = Git.open(createRepoPath(repoSuffix).toFile())) { - return this.resetToLastCommit(git) - .flatMap(ref -> checkoutToBranch(repoSuffix, branchName)) - .flatMap(checkedOut -> { - try { - return resetToLastCommit(git).thenReturn(true); - } catch (GitAPIException e) { - log.error(e.getMessage()); - return Mono.error(e); - } - }); - } - } - - public Mono resetHard(Path repoSuffix, String branchName) { - return this.checkoutToBranch(repoSuffix, branchName) - .flatMap(aBoolean -> { - try (Git git = Git.open(createRepoPath(repoSuffix).toFile())) { - Ref ref = git.reset() - .setMode(ResetCommand.ResetType.HARD) - .setRef("HEAD~1") - .call(); - return Mono.just(true); - } catch (GitAPIException | IOException e) { - log.error("Error while resetting the commit, {}", e.getMessage()); - } - return Mono.just(false); - }) - .timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS)) - .subscribeOn(scheduler); - } - - public Mono rebaseBranch(Path repoSuffix, String branchName) { - return this.checkoutToBranch(repoSuffix, branchName) - .flatMap(isCheckedOut -> { - try (Git git = Git.open(createRepoPath(repoSuffix).toFile())) { - RebaseResult result = - git.rebase().setUpstream("origin/" + branchName).call(); - if (result.getStatus().isSuccessful()) { - return Mono.just(true); - } else { - log.error( - "Error while rebasing the branch, {}, {}", - result.getStatus().name(), - result.getConflicts()); - git.rebase() - .setUpstream("origin/" + branchName) - .setOperation(RebaseCommand.Operation.ABORT) - .call(); - return Mono.error(new Exception("Error while rebasing the branch, " - + result.getStatus().name())); - } - } catch (GitAPIException | IOException e) { - log.error("Error while rebasing the branch, {}", e.getMessage()); - return Mono.error(e); - } - }) - .timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS)) - .subscribeOn(scheduler); - } - - @Override - public Mono getBranchTrackingStatus(Path repoPath, String branchName) { - return Mono.fromCallable(() -> { - try (Git git = Git.open(createRepoPath(repoPath).toFile())) { - return BranchTrackingStatus.of(git.getRepository(), branchName); - } - }) - .timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS)) - .subscribeOn(scheduler); + public GitExecutorImpl(GitServiceConfig gitServiceConfig) { + super(gitServiceConfig); } } diff --git a/app/server/appsmith-git/src/main/java/com/appsmith/git/service/ce/GitExecutorCEImpl.java b/app/server/appsmith-git/src/main/java/com/appsmith/git/service/ce/GitExecutorCEImpl.java new file mode 100644 index 0000000000..2bbaa24f2d --- /dev/null +++ b/app/server/appsmith-git/src/main/java/com/appsmith/git/service/ce/GitExecutorCEImpl.java @@ -0,0 +1,917 @@ +package com.appsmith.git.service.ce; + +import com.appsmith.external.constants.AnalyticsEvents; +import com.appsmith.external.constants.ErrorReferenceDocUrl; +import com.appsmith.external.dtos.GitBranchDTO; +import com.appsmith.external.dtos.GitLogDTO; +import com.appsmith.external.dtos.GitStatusDTO; +import com.appsmith.external.dtos.MergeStatusDTO; +import com.appsmith.external.git.GitExecutor; +import com.appsmith.external.helpers.Stopwatch; +import com.appsmith.git.configurations.GitServiceConfig; +import com.appsmith.git.constants.AppsmithBotAsset; +import com.appsmith.git.constants.CommonConstants; +import com.appsmith.git.constants.Constraint; +import com.appsmith.git.constants.GitDirectories; +import com.appsmith.git.helpers.RepositoryHelper; +import com.appsmith.git.helpers.SshTransportConfigCallback; +import com.appsmith.git.helpers.StopwatchHelpers; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.eclipse.jgit.api.CreateBranchCommand; +import org.eclipse.jgit.api.Git; +import org.eclipse.jgit.api.ListBranchCommand; +import org.eclipse.jgit.api.MergeCommand; +import org.eclipse.jgit.api.MergeResult; +import org.eclipse.jgit.api.RebaseCommand; +import org.eclipse.jgit.api.RebaseResult; +import org.eclipse.jgit.api.ResetCommand; +import org.eclipse.jgit.api.Status; +import org.eclipse.jgit.api.TransportConfigCallback; +import org.eclipse.jgit.api.errors.CheckoutConflictException; +import org.eclipse.jgit.api.errors.GitAPIException; +import org.eclipse.jgit.lib.BranchTrackingStatus; +import org.eclipse.jgit.lib.PersonIdent; +import org.eclipse.jgit.lib.Ref; +import org.eclipse.jgit.lib.StoredConfig; +import org.eclipse.jgit.merge.MergeStrategy; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.transport.RefSpec; +import org.eclipse.jgit.util.StringUtils; +import org.springframework.stereotype.Component; +import org.springframework.util.FileSystemUtils; +import reactor.core.publisher.Mono; +import reactor.core.scheduler.Scheduler; +import reactor.core.scheduler.Schedulers; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Duration; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; + +import static com.appsmith.git.constants.CommonConstants.FILE_MIGRATION_MESSAGE; + +@RequiredArgsConstructor +@Component +@Slf4j +public class GitExecutorCEImpl implements GitExecutor { + + private final RepositoryHelper repositoryHelper = new RepositoryHelper(); + + private final GitServiceConfig gitServiceConfig; + + public static final DateTimeFormatter ISO_FORMATTER = + DateTimeFormatter.ISO_INSTANT.withZone(ZoneId.from(ZoneOffset.UTC)); + + private final Scheduler scheduler = Schedulers.boundedElastic(); + + private static final String SUCCESS_MERGE_STATUS = "This branch has no conflicts with the base branch."; + + /** + * This method will handle the git-commit functionality. Under the hood it checks if the repo has already been + * initialised and will be initialised if git repo is not present + * @param path parent path to repo + * @param commitMessage message which will be registered for this commit + * @param authorName author details + * @param authorEmail author details + * @param doAmend To amend with the previous commit + * @return if the commit was successful + */ + @Override + public Mono commitApplication( + Path path, + String commitMessage, + String authorName, + String authorEmail, + boolean isSuffixedPath, + boolean doAmend) { + + final String finalAuthorName = + StringUtils.isEmptyOrNull(authorName) ? AppsmithBotAsset.APPSMITH_BOT_USERNAME : authorName; + final String finalAuthorEmail = + StringUtils.isEmptyOrNull(authorEmail) ? AppsmithBotAsset.APPSMITH_BOT_EMAIL : authorEmail; + return Mono.fromCallable(() -> { + log.debug("Trying to commit to local repo path, {}", path); + Path repoPath = path; + if (Boolean.TRUE.equals(isSuffixedPath)) { + repoPath = createRepoPath(repoPath); + } + Stopwatch processStopwatch = + StopwatchHelpers.startStopwatch(repoPath, AnalyticsEvents.GIT_COMMIT.getEventName()); + // Just need to open a repository here and make a commit + try (Git git = Git.open(repoPath.toFile())) { + // Stage all the files added and modified + git.add().addFilepattern(".").call(); + // Stage modified and deleted files + git.add().setUpdate(true).addFilepattern(".").call(); + + // Commit the changes + git.commit() + .setMessage(commitMessage) + // Only make a commit if there are any updates + .setAllowEmpty(false) + .setAuthor(finalAuthorName, finalAuthorEmail) + .setCommitter(finalAuthorName, finalAuthorEmail) + .setAmend(doAmend) + .call(); + processStopwatch.stopAndLogTimeInMillis(); + return "Committed successfully!"; + } + }) + .timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS)) + .subscribeOn(scheduler); + } + + /** + * Method to create a new repository to provided path + * @param repoPath path where new repo needs to be created + * @return if the operation was successful + */ + @Override + public boolean createNewRepository(Path repoPath) throws GitAPIException { + // create new repo to the mentioned path + log.debug("Trying to create new repository: {}", repoPath); + Git.init().setDirectory(repoPath.toFile()).call(); + return true; + } + + /** + * Method to get the commit history + * @param repoSuffix Path used to generate the repo url specific to the application for which the commit history is requested + * @return list of git commits + */ + @Override + public Mono> getCommitHistory(Path repoSuffix) { + return Mono.fromCallable(() -> { + log.debug(Thread.currentThread().getName() + ": get commit history for " + repoSuffix); + List commitLogs = new ArrayList<>(); + Path repoPath = createRepoPath(repoSuffix); + Stopwatch processStopwatch = StopwatchHelpers.startStopwatch( + repoPath, AnalyticsEvents.GIT_COMMIT_HISTORY.getEventName()); + try (Git git = Git.open(repoPath.toFile())) { + Iterable gitLogs = git.log() + .setMaxCount(Constraint.MAX_COMMIT_LOGS) + .call(); + gitLogs.forEach(revCommit -> { + PersonIdent author = revCommit.getAuthorIdent(); + GitLogDTO gitLog = new GitLogDTO( + revCommit.getName(), + author.getName(), + author.getEmailAddress(), + revCommit.getFullMessage(), + ISO_FORMATTER.format(new Date(revCommit.getCommitTime() * 1000L).toInstant())); + processStopwatch.stopAndLogTimeInMillis(); + commitLogs.add(gitLog); + }); + return commitLogs; + } + }) + .timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS)) + .subscribeOn(scheduler); + } + + @Override + public Path createRepoPath(Path suffix) { + return Paths.get(gitServiceConfig.getGitRootPath()).resolve(suffix); + } + + /** + * Method to push changes to remote repo + * @param repoSuffix Path used to generate the repo url specific to the application which needs to be pushed to remote + * @param remoteUrl remote repo url + * @param publicKey + * @param privateKey + * @return Success message + */ + @Override + public Mono pushApplication( + Path repoSuffix, String remoteUrl, String publicKey, String privateKey, String branchName) { + // We can safely assume that repo has been already initialised either in commit or clone flow and can directly + // open the repo + return Mono.fromCallable(() -> { + log.debug(Thread.currentThread().getName() + ": pushing changes to remote " + remoteUrl); + // open the repo + Path baseRepoPath = createRepoPath(repoSuffix); + Stopwatch processStopwatch = + StopwatchHelpers.startStopwatch(baseRepoPath, AnalyticsEvents.GIT_PUSH.getEventName()); + try (Git git = Git.open(baseRepoPath.toFile())) { + TransportConfigCallback transportConfigCallback = + new SshTransportConfigCallback(privateKey, publicKey); + + StringBuilder result = new StringBuilder("Pushed successfully with status : "); + git.push() + .setTransportConfigCallback(transportConfigCallback) + .setRemote(remoteUrl) + .call() + .forEach(pushResult -> pushResult + .getRemoteUpdates() + .forEach(remoteRefUpdate -> { + result.append(remoteRefUpdate.getStatus()) + .append(","); + if (!StringUtils.isEmptyOrNull(remoteRefUpdate.getMessage())) { + result.append(remoteRefUpdate.getMessage()) + .append(","); + } + })); + // We can support username and password in future if needed + // pushCommand.setCredentialsProvider(new UsernamePasswordCredentialsProvider("username", + // "password")); + processStopwatch.stopAndLogTimeInMillis(); + return result.substring(0, result.length() - 1); + } + }) + .timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS)) + .subscribeOn(scheduler); + } + + /** Clone the repo to the file path : container-volume/orgId/defaultAppId/repo/applicationData + * + * @param repoSuffix combination of orgId, defaultId and repoName + * @param remoteUrl ssh url of the git repo(we support cloning via ssh url only with deploy key) + * @param privateKey generated by us and specific to the defaultApplication + * @param publicKey generated by us and specific to the defaultApplication + * @return defaultBranchName of the repo + * */ + @Override + public Mono cloneApplication(Path repoSuffix, String remoteUrl, String privateKey, String publicKey) { + + Stopwatch processStopwatch = + StopwatchHelpers.startStopwatch(repoSuffix, AnalyticsEvents.GIT_CLONE.getEventName()); + return Mono.fromCallable(() -> { + log.debug(Thread.currentThread().getName() + ": Cloning the repo from the remote " + remoteUrl); + final TransportConfigCallback transportConfigCallback = + new SshTransportConfigCallback(privateKey, publicKey); + File file = Paths.get(gitServiceConfig.getGitRootPath()) + .resolve(repoSuffix) + .toFile(); + while (file.exists()) { + FileSystemUtils.deleteRecursively(file); + } + Git git = Git.cloneRepository() + .setURI(remoteUrl) + .setTransportConfigCallback(transportConfigCallback) + .setDirectory(file) + .call(); + String branchName = git.getRepository().getBranch(); + + repositoryHelper.updateRemoteBranchTrackingConfig(branchName, git); + git.close(); + processStopwatch.stopAndLogTimeInMillis(); + return branchName; + }) + .timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS)) + .subscribeOn(scheduler); + } + + @Override + public Mono createAndCheckoutToBranch(Path repoSuffix, String branchName) { + // We can safely assume that repo has been already initialised either in commit or clone flow and can directly + // open the repo + Stopwatch processStopwatch = + StopwatchHelpers.startStopwatch(repoSuffix, AnalyticsEvents.GIT_CREATE_BRANCH.getEventName()); + return Mono.fromCallable(() -> { + log.debug(Thread.currentThread().getName() + ": Creating branch " + branchName + "for the repo " + + repoSuffix); + // open the repo + Path baseRepoPath = createRepoPath(repoSuffix); + try (Git git = Git.open(baseRepoPath.toFile())) { + // Create and checkout to new branch + git.checkout() + .setCreateBranch(Boolean.TRUE) + .setName(branchName) + .setUpstreamMode(CreateBranchCommand.SetupUpstreamMode.TRACK) + .call(); + + repositoryHelper.updateRemoteBranchTrackingConfig(branchName, git); + processStopwatch.stopAndLogTimeInMillis(); + return git.getRepository().getBranch(); + } + }) + .timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS)) + .subscribeOn(scheduler); + } + + @Override + public Mono deleteBranch(Path repoSuffix, String branchName) { + // We can safely assume that repo has been already initialised either in commit or clone flow and can directly + // open the repo + Stopwatch processStopwatch = + StopwatchHelpers.startStopwatch(repoSuffix, AnalyticsEvents.GIT_DELETE_BRANCH.getEventName()); + return Mono.fromCallable(() -> { + log.debug(Thread.currentThread().getName() + ": Deleting branch " + branchName + "for the repo " + + repoSuffix); + // open the repo + Path baseRepoPath = createRepoPath(repoSuffix); + try (Git git = Git.open(baseRepoPath.toFile())) { + // Create and checkout to new branch + List deleteBranchList = git.branchDelete() + .setBranchNames(branchName) + .setForce(Boolean.TRUE) + .call(); + processStopwatch.stopAndLogTimeInMillis(); + if (deleteBranchList.isEmpty()) { + return Boolean.FALSE; + } + return Boolean.TRUE; + } + }) + .timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS)) + .subscribeOn(scheduler); + } + + @Override + public Mono checkoutToBranch(Path repoSuffix, String branchName) { + + Stopwatch processStopwatch = + StopwatchHelpers.startStopwatch(repoSuffix, AnalyticsEvents.GIT_CHECKOUT.getEventName()); + return Mono.fromCallable(() -> { + log.debug(Thread.currentThread().getName() + ": Switching to the branch " + branchName); + // We can safely assume that repo has been already initialised either in commit or clone flow and + // can directly + // open the repo + Path baseRepoPath = createRepoPath(repoSuffix); + try (Git git = Git.open(baseRepoPath.toFile())) { + if (StringUtils.equalsIgnoreCase( + branchName, git.getRepository().getBranch())) { + return Boolean.TRUE; + } + // Create and checkout to new branch + String checkedOutBranch = git.checkout() + .setCreateBranch(Boolean.FALSE) + .setName(branchName) + .setUpstreamMode(CreateBranchCommand.SetupUpstreamMode.SET_UPSTREAM) + .call() + .getName(); + processStopwatch.stopAndLogTimeInMillis(); + return StringUtils.equalsIgnoreCase(checkedOutBranch, "refs/heads/" + branchName); + } catch (Exception e) { + throw new Exception(e); + } + }) + .timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS)) + .subscribeOn(scheduler); + } + + @Override + public Mono pullApplication( + Path repoSuffix, String remoteUrl, String branchName, String privateKey, String publicKey) + throws IOException { + + Stopwatch processStopwatch = + StopwatchHelpers.startStopwatch(repoSuffix, AnalyticsEvents.GIT_PULL.getEventName()); + TransportConfigCallback transportConfigCallback = new SshTransportConfigCallback(privateKey, publicKey); + + try (Git git = Git.open(createRepoPath(repoSuffix).toFile())) { + return Mono.fromCallable(() -> { + log.debug(Thread.currentThread().getName() + ": Pull changes from remote " + remoteUrl + + " for the branch " + branchName); + // checkout the branch on which the merge command is run + git.checkout() + .setName(branchName) + .setCreateBranch(false) + .call(); + MergeResult mergeResult = git.pull() + .setRemoteBranchName(branchName) + .setTransportConfigCallback(transportConfigCallback) + .setFastForward(MergeCommand.FastForwardMode.FF) + .call() + .getMergeResult(); + MergeStatusDTO mergeStatus = new MergeStatusDTO(); + Long count = + Arrays.stream(mergeResult.getMergedCommits()).count(); + if (mergeResult.getMergeStatus().isSuccessful()) { + mergeStatus.setMergeAble(true); + mergeStatus.setStatus(count + " commits merged from origin/" + branchName); + } else { + // If there are conflicts add the conflicting file names to the response structure + mergeStatus.setMergeAble(false); + List mergeConflictFiles = new ArrayList<>(); + if (!Optional.ofNullable(mergeResult.getConflicts()).isEmpty()) { + mergeConflictFiles.addAll( + mergeResult.getConflicts().keySet()); + } + mergeStatus.setConflictingFiles(mergeConflictFiles); + // On merge conflicts abort the merge => git merge --abort + git.getRepository().writeMergeCommitMsg(null); + git.getRepository().writeMergeHeads(null); + processStopwatch.stopAndLogTimeInMillis(); + throw new org.eclipse.jgit.errors.CheckoutConflictException(mergeConflictFiles.toString()); + } + processStopwatch.stopAndLogTimeInMillis(); + return mergeStatus; + }) + .onErrorResume(error -> { + try { + return resetToLastCommit(git).flatMap(ignore -> Mono.error(error)); + } catch (GitAPIException e) { + log.error("Error for hard resetting to latest commit {0}", e); + return Mono.error(e); + } + }) + .timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS)) + .subscribeOn(scheduler); + } + } + + @Override + public Mono> listBranches(Path repoSuffix) { + Path baseRepoPath = createRepoPath(repoSuffix); + return Mono.fromCallable(() -> { + log.debug(Thread.currentThread().getName() + ": Get branches for the application " + repoSuffix); + Git git = Git.open(baseRepoPath.toFile()); + List refList = git.branchList() + .setListMode(ListBranchCommand.ListMode.ALL) + .call(); + + List branchList = new ArrayList<>(); + GitBranchDTO gitBranchDTO = new GitBranchDTO(); + if (refList.isEmpty()) { + gitBranchDTO.setBranchName(git.getRepository().getBranch()); + branchList.add(gitBranchDTO); + } else { + for (Ref ref : refList) { + // if (!ref.getName().equals(defaultBranch)) { + gitBranchDTO = new GitBranchDTO(); + gitBranchDTO.setBranchName(ref.getName() + .replace("refs/", "") + .replace("heads/", "") + .replace("remotes/", "")); + branchList.add(gitBranchDTO); + } + } + git.close(); + return branchList; + }) + .timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS)) + .subscribeOn(scheduler); + } + + @Override + public Mono getRemoteDefaultBranch(Path repoSuffix, String remoteUrl, String privateKey, String publicKey) { + Path baseRepoPath = createRepoPath(repoSuffix); + return Mono.fromCallable(() -> { + TransportConfigCallback transportConfigCallback = + new SshTransportConfigCallback(privateKey, publicKey); + Git git = Git.open(baseRepoPath.toFile()); + + return git.lsRemote() + .setRemote(remoteUrl) + .setTransportConfigCallback(transportConfigCallback) + .callAsMap() + .get("HEAD") + .getTarget() + .getName() + .replace("refs/heads/", ""); + }) + .timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS)) + .subscribeOn(scheduler); + } + + /** + * This method will handle the git-status functionality + * + * @param repoPath Path to actual repo + * @param branchName branch name for which the status is required + * @return Map of file names those are modified, conflicted etc. + */ + @Override + public Mono getStatus(Path repoPath, String branchName) { + Stopwatch processStopwatch = + StopwatchHelpers.startStopwatch(repoPath, AnalyticsEvents.GIT_STATUS.getEventName()); + return Mono.fromCallable(() -> { + try (Git git = Git.open(repoPath.toFile())) { + log.debug(Thread.currentThread().getName() + ": Get status for repo " + repoPath + ", branch " + + branchName); + Status status = git.status().call(); + GitStatusDTO response = new GitStatusDTO(); + Set modifiedAssets = new HashSet<>(); + modifiedAssets.addAll(status.getModified()); + modifiedAssets.addAll(status.getAdded()); + modifiedAssets.addAll(status.getRemoved()); + modifiedAssets.addAll(status.getUncommittedChanges()); + modifiedAssets.addAll(status.getUntracked()); + response.setAdded(status.getAdded()); + response.setRemoved(status.getRemoved()); + + populateModifiedEntities(status, response, modifiedAssets); + + BranchTrackingStatus trackingStatus = BranchTrackingStatus.of(git.getRepository(), branchName); + if (trackingStatus != null) { + response.setAheadCount(trackingStatus.getAheadCount()); + response.setBehindCount(trackingStatus.getBehindCount()); + response.setRemoteBranch(trackingStatus.getRemoteTrackingBranch()); + } else { + log.debug( + "Remote tracking details not present for branch: {}, repo: {}", + branchName, + repoPath); + response.setAheadCount(0); + response.setBehindCount(0); + response.setRemoteBranch("untracked"); + } + + // Remove modified changes from current branch so that checkout to other branches will be + // possible + if (!status.isClean()) { + return resetToLastCommit(git).map(ref -> { + processStopwatch.stopAndLogTimeInMillis(); + return response; + }); + } + processStopwatch.stopAndLogTimeInMillis(); + return Mono.just(response); + } + }) + .timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS)) + .flatMap(response -> response) + .subscribeOn(scheduler); + } + + protected void populateModifiedEntities(Status status, GitStatusDTO response, Set modifiedAssets) { + Set queriesModified = new HashSet<>(); + Set jsObjectsModified = new HashSet<>(); + Set pagesModified = new HashSet<>(); + int modifiedPages = 0; + int modifiedQueries = 0; + int modifiedJSObjects = 0; + int modifiedDatasources = 0; + int modifiedJSLibs = 0; + for (String x : modifiedAssets) { + // begins with pages and filename and parent name should be same or contains widgets + if (x.contains(CommonConstants.WIDGETS)) { + if (!pagesModified.contains(getPageName(x))) { + pagesModified.add(getPageName(x)); + modifiedPages++; + } + } else if (isAModifiedPage(x)) { + if (!pagesModified.contains(getPageName(x))) { + pagesModified.add(getPageName(x)); + modifiedPages++; + } + } else if (x.contains(GitDirectories.ACTION_DIRECTORY + CommonConstants.DELIMITER_PATH)) { + String queryName = x.split(GitDirectories.ACTION_DIRECTORY + CommonConstants.DELIMITER_PATH)[1]; + int position = queryName.indexOf(CommonConstants.DELIMITER_PATH); + if (position != -1) { + queryName = queryName.substring(0, position); + String pageName = x.split(CommonConstants.DELIMITER_PATH)[1]; + if (!queriesModified.contains(pageName + queryName)) { + queriesModified.add(pageName + queryName); + modifiedQueries++; + } + } + } else if (x.contains(GitDirectories.ACTION_COLLECTION_DIRECTORY + CommonConstants.DELIMITER_PATH) + && !x.endsWith(CommonConstants.JSON_EXTENSION)) { + String queryName = x.substring(x.lastIndexOf(CommonConstants.DELIMITER_PATH) + 1); + String pageName = x.split(CommonConstants.DELIMITER_PATH)[1]; + if (!jsObjectsModified.contains(pageName + queryName)) { + jsObjectsModified.add(pageName + queryName); + modifiedJSObjects++; + } + } else if (x.contains(GitDirectories.DATASOURCE_DIRECTORY + CommonConstants.DELIMITER_PATH)) { + modifiedDatasources++; + } else if (x.contains(GitDirectories.JS_LIB_DIRECTORY + CommonConstants.DELIMITER_PATH)) { + modifiedJSLibs++; + // remove this code in future when all the older format js libs are migrated to new + // format + + if (x.contains("js.json")) { + /* + As this updated filename has color(:), it means this is the older format js + lib file that we're going to rename with the format without colon. + Hence, we need to show a message to user saying this might be a system level change. + */ + response.setMigrationMessage(FILE_MIGRATION_MESSAGE); + } + } + } + response.setModified(modifiedAssets); + response.setConflicting(status.getConflicting()); + response.setIsClean(status.isClean()); + response.setModifiedPages(modifiedPages); + response.setModifiedQueries(modifiedQueries); + response.setModifiedJSObjects(modifiedJSObjects); + response.setModifiedDatasources(modifiedDatasources); + response.setModifiedJSLibs(modifiedJSLibs); + } + + protected boolean isAModifiedPage(String x) { + return !x.contains(CommonConstants.WIDGETS) + && x.startsWith(GitDirectories.PAGE_DIRECTORY) + && !x.contains(GitDirectories.ACTION_DIRECTORY) + && !x.contains(GitDirectories.ACTION_COLLECTION_DIRECTORY); + } + + private String getPageName(String path) { + String[] pathArray = path.split(CommonConstants.DELIMITER_PATH); + return pathArray[1]; + } + + @Override + public Mono mergeBranch(Path repoSuffix, String sourceBranch, String destinationBranch) { + return Mono.fromCallable(() -> { + Stopwatch processStopwatch = + StopwatchHelpers.startStopwatch(repoSuffix, AnalyticsEvents.GIT_MERGE.getEventName()); + log.debug(Thread.currentThread().getName() + ": Merge branch " + sourceBranch + " on " + + destinationBranch); + try (Git git = Git.open(createRepoPath(repoSuffix).toFile())) { + try { + // checkout the branch on which the merge command is run + git.checkout() + .setName(destinationBranch) + .setCreateBranch(false) + .call(); + + MergeResult mergeResult = git.merge() + .include(git.getRepository().findRef(sourceBranch)) + .setStrategy(MergeStrategy.RECURSIVE) + .call(); + processStopwatch.stopAndLogTimeInMillis(); + return mergeResult.getMergeStatus().name(); + } catch (GitAPIException e) { + // On merge conflicts abort the merge => git merge --abort + git.getRepository().writeMergeCommitMsg(null); + git.getRepository().writeMergeHeads(null); + processStopwatch.stopAndLogTimeInMillis(); + throw new Exception(e); + } + } + }) + .onErrorResume(error -> { + try { + return resetToLastCommit(repoSuffix, destinationBranch).thenReturn(error.getMessage()); + } catch (GitAPIException | IOException e) { + log.error("Error while hard resetting to latest commit {0}", e); + return Mono.error(e); + } + }) + .timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS)) + .subscribeOn(scheduler); + } + + @Override + public Mono fetchRemote( + Path repoSuffix, + String publicKey, + String privateKey, + boolean isRepoPath, + String branchName, + boolean isFetchAll) { + Stopwatch processStopwatch = + StopwatchHelpers.startStopwatch(repoSuffix, AnalyticsEvents.GIT_FETCH.getEventName()); + Path repoPath = Boolean.TRUE.equals(isRepoPath) ? repoSuffix : createRepoPath(repoSuffix); + return Mono.fromCallable(() -> { + TransportConfigCallback config = new SshTransportConfigCallback(privateKey, publicKey); + try (Git git = Git.open(repoPath.toFile())) { + String fetchMessages; + if (Boolean.TRUE.equals(isFetchAll)) { + fetchMessages = git.fetch() + .setRemoveDeletedRefs(true) + .setTransportConfigCallback(config) + .call() + .getMessages(); + } else { + RefSpec ref = + new RefSpec("refs/heads/" + branchName + ":refs/remotes/origin/" + branchName); + fetchMessages = git.fetch() + .setRefSpecs(ref) + .setRemoveDeletedRefs(true) + .setTransportConfigCallback(config) + .call() + .getMessages(); + } + processStopwatch.stopAndLogTimeInMillis(); + return fetchMessages; + } + }) + .onErrorResume(error -> { + log.error(error.getMessage()); + return Mono.error(error); + }) + .timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS)) + .subscribeOn(scheduler); + } + + @Override + public Mono isMergeBranch(Path repoSuffix, String sourceBranch, String destinationBranch) { + Stopwatch processStopwatch = + StopwatchHelpers.startStopwatch(repoSuffix, AnalyticsEvents.GIT_MERGE_CHECK.getEventName()); + return Mono.fromCallable(() -> { + log.debug( + Thread.currentThread().getName() + + ": Check mergeability for repo {} with src: {}, dest: {}", + repoSuffix, + sourceBranch, + destinationBranch); + + try (Git git = Git.open(createRepoPath(repoSuffix).toFile())) { + + // checkout the branch on which the merge command is run + try { + git.checkout() + .setName(destinationBranch) + .setCreateBranch(false) + .call(); + } catch (GitAPIException e) { + if (e instanceof CheckoutConflictException) { + MergeStatusDTO mergeStatus = new MergeStatusDTO(); + mergeStatus.setMergeAble(false); + mergeStatus.setConflictingFiles(((CheckoutConflictException) e).getConflictingPaths()); + processStopwatch.stopAndLogTimeInMillis(); + return mergeStatus; + } + } + + MergeResult mergeResult = git.merge() + .include(git.getRepository().findRef(sourceBranch)) + .setFastForward(MergeCommand.FastForwardMode.NO_FF) + .setCommit(false) + .call(); + + MergeStatusDTO mergeStatus = new MergeStatusDTO(); + if (mergeResult.getMergeStatus().isSuccessful()) { + mergeStatus.setMergeAble(true); + mergeStatus.setMessage(SUCCESS_MERGE_STATUS); + } else { + // If there aer conflicts add the conflicting file names to the response structure + mergeStatus.setMergeAble(false); + List mergeConflictFiles = + new ArrayList<>(mergeResult.getConflicts().keySet()); + mergeStatus.setConflictingFiles(mergeConflictFiles); + StringBuilder errorMessage = new StringBuilder(); + if (mergeResult.getMergeStatus().equals(MergeResult.MergeStatus.CONFLICTING)) { + errorMessage.append("Conflicts"); + } else { + errorMessage.append(mergeResult.getMergeStatus().toString()); + } + errorMessage + .append(" while merging branch: ") + .append(destinationBranch) + .append(" <= ") + .append(sourceBranch); + mergeStatus.setMessage(errorMessage.toString()); + mergeStatus.setReferenceDoc(ErrorReferenceDocUrl.GIT_MERGE_CONFLICT.getDocUrl()); + } + mergeStatus.setStatus(mergeResult.getMergeStatus().name()); + return mergeStatus; + } + }) + .flatMap(status -> { + try { + // Revert uncommitted changes if any + return resetToLastCommit(repoSuffix, destinationBranch).map(ignore -> { + processStopwatch.stopAndLogTimeInMillis(); + return status; + }); + } catch (GitAPIException | IOException e) { + log.error("Error for hard resetting to latest commit {0}", e); + return Mono.error(e); + } + }) + .timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS)) + .subscribeOn(scheduler); + } + + public Mono checkoutRemoteBranch(Path repoSuffix, String branchName) { + // We can safely assume that repo has been already initialised either in commit or clone flow and can directly + // open the repo + return Mono.fromCallable(() -> { + log.debug(Thread.currentThread().getName() + ": Checking out remote branch origin/" + branchName + + " for the repo " + repoSuffix); + // open the repo + Path baseRepoPath = createRepoPath(repoSuffix); + try (Git git = Git.open(baseRepoPath.toFile())) { + // Create and checkout to new branch + git.checkout() + .setCreateBranch(Boolean.TRUE) + .setName(branchName) + .setUpstreamMode(CreateBranchCommand.SetupUpstreamMode.TRACK) + .setStartPoint("origin/" + branchName) + .call(); + + StoredConfig config = git.getRepository().getConfig(); + config.setString("branch", branchName, "remote", "origin"); + config.setString("branch", branchName, "merge", "refs/heads/" + branchName); + config.save(); + return git.getRepository().getBranch(); + } + }) + .timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS)) + .subscribeOn(scheduler); + } + + @Override + public Mono testConnection(String publicKey, String privateKey, String remoteUrl) { + return Mono.fromCallable(() -> { + TransportConfigCallback transportConfigCallback = + new SshTransportConfigCallback(privateKey, publicKey); + Git.lsRemoteRepository() + .setTransportConfigCallback(transportConfigCallback) + .setRemote(remoteUrl) + .setHeads(true) + .setTags(true) + .call(); + return true; + }) + .timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS)) + .subscribeOn(scheduler); + } + + private Mono resetToLastCommit(Git git) throws GitAPIException { + Stopwatch processStopwatch = StopwatchHelpers.startStopwatch( + git.getRepository().getDirectory().toPath().getParent(), AnalyticsEvents.GIT_RESET.getEventName()); + return Mono.fromCallable(() -> { + // Remove tracked files + Ref ref = git.reset().setMode(ResetCommand.ResetType.HARD).call(); + // Remove untracked files + git.clean().setForce(true).setCleanDirectories(true).call(); + processStopwatch.stopAndLogTimeInMillis(); + return ref; + }) + .timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS)) + .subscribeOn(scheduler); + } + + public Mono resetToLastCommit(Path repoSuffix, String branchName) throws GitAPIException, IOException { + try (Git git = Git.open(createRepoPath(repoSuffix).toFile())) { + return this.resetToLastCommit(git) + .flatMap(ref -> checkoutToBranch(repoSuffix, branchName)) + .flatMap(checkedOut -> { + try { + return resetToLastCommit(git).thenReturn(true); + } catch (GitAPIException e) { + log.error(e.getMessage()); + return Mono.error(e); + } + }); + } + } + + public Mono resetHard(Path repoSuffix, String branchName) { + return this.checkoutToBranch(repoSuffix, branchName) + .flatMap(aBoolean -> { + try (Git git = Git.open(createRepoPath(repoSuffix).toFile())) { + Ref ref = git.reset() + .setMode(ResetCommand.ResetType.HARD) + .setRef("HEAD~1") + .call(); + return Mono.just(true); + } catch (GitAPIException | IOException e) { + log.error("Error while resetting the commit, {}", e.getMessage()); + } + return Mono.just(false); + }) + .timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS)) + .subscribeOn(scheduler); + } + + public Mono rebaseBranch(Path repoSuffix, String branchName) { + return this.checkoutToBranch(repoSuffix, branchName) + .flatMap(isCheckedOut -> { + try (Git git = Git.open(createRepoPath(repoSuffix).toFile())) { + RebaseResult result = + git.rebase().setUpstream("origin/" + branchName).call(); + if (result.getStatus().isSuccessful()) { + return Mono.just(true); + } else { + log.error( + "Error while rebasing the branch, {}, {}", + result.getStatus().name(), + result.getConflicts()); + git.rebase() + .setUpstream("origin/" + branchName) + .setOperation(RebaseCommand.Operation.ABORT) + .call(); + return Mono.error(new Exception("Error while rebasing the branch, " + + result.getStatus().name())); + } + } catch (GitAPIException | IOException e) { + log.error("Error while rebasing the branch, {}", e.getMessage()); + return Mono.error(e); + } + }) + .timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS)) + .subscribeOn(scheduler); + } + + @Override + public Mono getBranchTrackingStatus(Path repoPath, String branchName) { + return Mono.fromCallable(() -> { + try (Git git = Git.open(repoPath.toFile())) { + return BranchTrackingStatus.of(git.getRepository(), branchName); + } + }) + .timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS)) + .subscribeOn(scheduler); + } +} diff --git a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/constants/GitConstants.java b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/constants/GitConstants.java index 690b40ccfb..cc2fbb64bb 100644 --- a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/constants/GitConstants.java +++ b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/constants/GitConstants.java @@ -1,24 +1,5 @@ package com.appsmith.external.constants; -public class GitConstants { - // This will be used as a key separator for action and jsobjects name - // pageName{{seperator}}entityName this is needed to filter the entities to save in appropriate page directory - public static final String NAME_SEPARATOR = "##ENTITY_SEPARATOR##"; - public static final String PAGE_LIST = "pageList"; - public static final String CUSTOM_JS_LIB_LIST = "customJSLibList"; - public static final String ACTION_LIST = "actionList"; - public static final String ACTION_COLLECTION_LIST = "actionCollectionList"; +import com.appsmith.external.constants.ce.GitConstantsCE; - public static final String DEFAULT_COMMIT_MESSAGE = "System generated commit, "; - public static final String EMPTY_COMMIT_ERROR_MESSAGE = "On current branch nothing to commit, working tree clean"; - public static final String MERGE_CONFLICT_BRANCH_NAME = "_mergeConflict"; - public static final String CONFLICTED_SUCCESS_MESSAGE = "branch has been created from conflicted state. Please " - + "resolve merge conflicts in remote and pull again"; - - public static final String GIT_CONFIG_ERROR = - "Unable to find the git configuration, please configure your application " - + "with git to use version control service"; - - public static final String GIT_PROFILE_ERROR = "Unable to find git author configuration for logged-in user. You can" - + " set up a git profile from the user profile section."; -} +public class GitConstants extends GitConstantsCE {} diff --git a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/constants/ce/GitConstantsCE.java b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/constants/ce/GitConstantsCE.java new file mode 100644 index 0000000000..467375d976 --- /dev/null +++ b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/constants/ce/GitConstantsCE.java @@ -0,0 +1,24 @@ +package com.appsmith.external.constants.ce; + +public class GitConstantsCE { + // This will be used as a key separator for action and jsobjects name + // pageName{{seperator}}entityName this is needed to filter the entities to save in appropriate page directory + public static final String NAME_SEPARATOR = "##ENTITY_SEPARATOR##"; + public static final String PAGE_LIST = "pageList"; + public static final String CUSTOM_JS_LIB_LIST = "customJSLibList"; + public static final String ACTION_LIST = "actionList"; + public static final String ACTION_COLLECTION_LIST = "actionCollectionList"; + + public static final String DEFAULT_COMMIT_MESSAGE = "System generated commit, "; + public static final String EMPTY_COMMIT_ERROR_MESSAGE = "On current branch nothing to commit, working tree clean"; + public static final String MERGE_CONFLICT_BRANCH_NAME = "_mergeConflict"; + public static final String CONFLICTED_SUCCESS_MESSAGE = "branch has been created from conflicted state. Please " + + "resolve merge conflicts in remote and pull again"; + + public static final String GIT_CONFIG_ERROR = + "Unable to find the git configuration, please configure your application " + + "with git to use version control service"; + + public static final String GIT_PROFILE_ERROR = "Unable to find git author configuration for logged-in user. You can" + + " set up a git profile from the user profile section."; +} diff --git a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/dtos/GitStatusDTO.java b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/dtos/GitStatusDTO.java index 693f4df8ac..84efcd5b19 100644 --- a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/dtos/GitStatusDTO.java +++ b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/dtos/GitStatusDTO.java @@ -1,57 +1,12 @@ package com.appsmith.external.dtos; -import com.appsmith.external.constants.Assets; +import com.appsmith.external.dtos.ce.GitStatusCE_DTO; import lombok.Data; - -import java.util.Set; +import lombok.EqualsAndHashCode; /** * DTO to convey the status local git repo */ +@EqualsAndHashCode(callSuper = true) @Data -public class GitStatusDTO { - - // Name of modified, added and deleted resources in local git repo - Set modified; - - // Name of added resources to local git repo - Set added; - - // Name of deleted resources from local git repo - Set removed; - - // Name of conflicting resources - Set conflicting; - - Boolean isClean; - - // number of modified custom JS libs - int modifiedJSLibs; - - // number of modified pages - int modifiedPages; - - // number of modified actions - int modifiedQueries; - - // number of modified JSObjects - int modifiedJSObjects; - - // number of modified JSObjects - int modifiedDatasources; - - // number of local commits which are not present in remote repo - Integer aheadCount; - - // number of remote commits which are not present in local repo - Integer behindCount; - - // Remote tracking branch name - String remoteBranch; - - // Documentation url for discard and pull functionality - String discardDocUrl = Assets.GIT_DISCARD_DOC_URL; - - // File Format migration - String migrationMessage = ""; -} +public class GitStatusDTO extends GitStatusCE_DTO {} diff --git a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/dtos/ce/GitStatusCE_DTO.java b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/dtos/ce/GitStatusCE_DTO.java new file mode 100644 index 0000000000..cca5d91adf --- /dev/null +++ b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/dtos/ce/GitStatusCE_DTO.java @@ -0,0 +1,57 @@ +package com.appsmith.external.dtos.ce; + +import com.appsmith.external.constants.Assets; +import lombok.Data; + +import java.util.Set; + +/** + * DTO to convey the status local git repo + */ +@Data +public class GitStatusCE_DTO { + + // Name of modified, added and deleted resources in local git repo + Set modified; + + // Name of added resources to local git repo + Set added; + + // Name of deleted resources from local git repo + Set removed; + + // Name of conflicting resources + Set conflicting; + + Boolean isClean; + + // number of modified custom JS libs + int modifiedJSLibs; + + // number of modified pages + int modifiedPages; + + // number of modified actions + int modifiedQueries; + + // number of modified JSObjects + int modifiedJSObjects; + + // number of modified JSObjects + int modifiedDatasources; + + // number of local commits which are not present in remote repo + Integer aheadCount; + + // number of remote commits which are not present in local repo + Integer behindCount; + + // Remote tracking branch name + String remoteBranch; + + // Documentation url for discard and pull functionality + String discardDocUrl = Assets.GIT_DISCARD_DOC_URL; + + // File Format migration + String migrationMessage = ""; +} diff --git a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/ApplicationGitReference.java b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/ApplicationGitReference.java index b7a8fde4f2..bdfb2dca66 100644 --- a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/ApplicationGitReference.java +++ b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/ApplicationGitReference.java @@ -1,36 +1,15 @@ package com.appsmith.external.models; +import com.appsmith.external.models.ce.ApplicationGitReferenceCE; import lombok.Data; +import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; -import java.util.Map; -import java.util.Set; - /** * A DTO class to hold complete information about an application, which will then be serialized to a file so as to * export/save that application into a json files. */ +@EqualsAndHashCode(callSuper = true) @Data @NoArgsConstructor -public class ApplicationGitReference { - - Object application; - Object metadata; - Object theme; - Map actions; - Map actionBody; - Map actionCollections; - Map actionCollectionBody; - Map pages; - Map pageDsl; - Map datasources; - Map jsLibraries; - - /** - * This field will be used to store map of files to be updated in local file system by comparing the recent - * changes in database and the last local git commit. - * This field can be used while saving resources to local file system and only update the resource files which - * are updated in the database. - */ - Map> updatedResources; -} +public class ApplicationGitReference extends ApplicationGitReferenceCE {} diff --git a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/DefaultResources.java b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/DefaultResources.java index 967b46dedc..1627bd8559 100644 --- a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/DefaultResources.java +++ b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/DefaultResources.java @@ -1,37 +1,13 @@ package com.appsmith.external.models; +import com.appsmith.external.models.ce.DefaultResourcesCE; import lombok.Data; +import lombok.EqualsAndHashCode; /** * This class will be used for connecting resources across branches for git connected application * e.g. Page1 in branch1 will have the same defaultResources.pageId as of Page1 of branch2 */ +@EqualsAndHashCode(callSuper = true) @Data -public class DefaultResources { - /** - * When present, actionId will hold the default action id - */ - String actionId; - - /** - * When present, applicationId will hold the default application id - */ - String applicationId; - - /** - * When present, pageId will hold the default page id - */ - String pageId; - - /** - * When present, collectionId will hold the default collection id - */ - String collectionId; - - /** - * When present, branchName will hold the current branch name. - * For example, if we've a page in both main and develop branch, then default resources of those two pages will - * have same applicationId, pageId but branchName will contain the corresponding branch name. - */ - String branchName; -} +public class DefaultResources extends DefaultResourcesCE {} diff --git a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/ce/ActionCE_DTO.java b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/ce/ActionCE_DTO.java index ba8c1ea21c..029fb47e69 100644 --- a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/ce/ActionCE_DTO.java +++ b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/ce/ActionCE_DTO.java @@ -171,8 +171,7 @@ public class ActionCE_DTO implements Identifiable, Executable { DefaultResources defaultResources; // This field will be used to store analytics data related to this specific domain object. It's been introduced in - // order to track - // success metrics of modules. Learn more on GitHub issue#24734 + // order to track success metrics of modules. Learn more on GitHub issue#24734 @JsonView(Views.Public.class) AnalyticsInfo eventData; @@ -216,6 +215,7 @@ public class ActionCE_DTO implements Identifiable, Executable { } public void sanitiseToExportDBObject() { + this.resetTransientFields(); this.setEventData(null); this.setDefaultResources(null); this.setCacheResponse(null); @@ -302,4 +302,19 @@ public class ActionCE_DTO implements Identifiable, Executable { this.datasource.setIsAutoGenerated(true); } } + + protected void resetTransientFields() { + this.setId(null); + this.setApplicationId(null); + this.setWorkspaceId(null); + this.setPluginId(null); + this.setPluginName(null); + this.setPluginType(null); + this.setErrorReports(null); + // this.setMessages(null); + this.setTemplateId(null); + this.setProvider(null); + this.setProviderId(null); + this.setDocumentation(null); + } } diff --git a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/ce/ApplicationGitReferenceCE.java b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/ce/ApplicationGitReferenceCE.java new file mode 100644 index 0000000000..1dc768ecf4 --- /dev/null +++ b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/ce/ApplicationGitReferenceCE.java @@ -0,0 +1,36 @@ +package com.appsmith.external.models.ce; + +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.Map; +import java.util.Set; + +/** + * A DTO class to hold complete information about an application, which will then be serialized to a file so as to + * export/save that application into a json files. + */ +@Data +@NoArgsConstructor +public class ApplicationGitReferenceCE { + + Object application; + Object metadata; + Object theme; + Map actions; + Map actionBody; + Map actionCollections; + Map actionCollectionBody; + Map pages; + Map pageDsl; + Map datasources; + Map jsLibraries; + + /** + * This field will be used to store map of files to be updated in local file system by comparing the recent + * changes in database and the last local git commit. + * This field can be used while saving resources to local file system and only update the resource files which + * are updated in the database. + */ + Map> updatedResources; +} diff --git a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/ce/DefaultResourcesCE.java b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/ce/DefaultResourcesCE.java new file mode 100644 index 0000000000..7df748e400 --- /dev/null +++ b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/ce/DefaultResourcesCE.java @@ -0,0 +1,37 @@ +package com.appsmith.external.models.ce; + +import lombok.Data; + +/** + * This class will be used for connecting resources across branches for git connected application + * e.g. Page1 in branch1 will have the same defaultResources.pageId as of Page1 of branch2 + */ +@Data +public class DefaultResourcesCE { + /** + * When present, actionId will hold the default action id + */ + String actionId; + + /** + * When present, applicationId will hold the default application id + */ + String applicationId; + + /** + * When present, pageId will hold the default page id + */ + String pageId; + + /** + * When present, collectionId will hold the default collection id + */ + String collectionId; + + /** + * When present, branchName will hold the current branch name. + * For example, if we've a page in both main and develop branch, then default resources of those two pages will + * have same applicationId, pageId but branchName will contain the corresponding branch name. + */ + String branchName; +} diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/actioncollections/base/ActionCollectionServiceCEImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/actioncollections/base/ActionCollectionServiceCEImpl.java index f7779c9a11..5f5321b678 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/actioncollections/base/ActionCollectionServiceCEImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/actioncollections/base/ActionCollectionServiceCEImpl.java @@ -9,6 +9,7 @@ import com.appsmith.server.acl.AclPermission; import com.appsmith.server.acl.PolicyGenerator; import com.appsmith.server.applications.base.ApplicationService; import com.appsmith.server.constants.FieldName; +import com.appsmith.server.defaultresources.DefaultResourcesService; import com.appsmith.server.domains.Action; import com.appsmith.server.domains.ActionCollection; import com.appsmith.server.domains.NewAction; @@ -29,7 +30,6 @@ import com.appsmith.server.solutions.ApplicationPermission; import jakarta.validation.Validator; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.ObjectUtils; -import org.apache.commons.lang3.StringUtils; import org.bson.types.ObjectId; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.domain.Sort; @@ -38,6 +38,7 @@ import org.springframework.data.mongodb.core.convert.MongoConverter; import org.springframework.util.CollectionUtils; import org.springframework.util.LinkedMultiValueMap; import org.springframework.util.MultiValueMap; +import org.springframework.util.StringUtils; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import reactor.core.scheduler.Scheduler; @@ -66,6 +67,7 @@ public class ActionCollectionServiceCEImpl extends BaseService defaultResourcesService; @Autowired public ActionCollectionServiceCEImpl( @@ -80,7 +82,8 @@ public class ActionCollectionServiceCEImpl extends BaseService defaultResourcesService) { super(scheduler, validator, mongoConverter, reactiveMongoTemplate, repository, analyticsService); this.newActionService = newActionService; @@ -89,6 +92,7 @@ public class ActionCollectionServiceCEImpl extends BaseService this.update(id, actionCollection)) @@ -492,7 +497,14 @@ public class ActionCollectionServiceCEImpl extends BaseService findByPageIdsForExport(List pageIds, Optional permission) { - return repository.findByPageIds(pageIds, permission); + return repository.findByPageIds(pageIds, permission).doOnNext(actionCollection -> { + actionCollection.getUnpublishedCollection().populateTransientFields(actionCollection); + if (actionCollection.getPublishedCollection() != null + && StringUtils.hasText( + actionCollection.getPublishedCollection().getName())) { + actionCollection.getPublishedCollection().populateTransientFields(actionCollection); + } + }); } @Override diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/actioncollections/base/ActionCollectionServiceImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/actioncollections/base/ActionCollectionServiceImpl.java index 07a215af41..86c507058d 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/actioncollections/base/ActionCollectionServiceImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/actioncollections/base/ActionCollectionServiceImpl.java @@ -2,6 +2,8 @@ package com.appsmith.server.actioncollections.base; import com.appsmith.server.acl.PolicyGenerator; import com.appsmith.server.applications.base.ApplicationService; +import com.appsmith.server.defaultresources.DefaultResourcesService; +import com.appsmith.server.domains.ActionCollection; import com.appsmith.server.helpers.ResponseUtils; import com.appsmith.server.newactions.base.NewActionService; import com.appsmith.server.repositories.ActionCollectionRepository; @@ -31,7 +33,8 @@ public class ActionCollectionServiceImpl extends ActionCollectionServiceCEImpl i ApplicationService applicationService, ResponseUtils responseUtils, ApplicationPermission applicationPermission, - ActionPermission actionPermission) { + ActionPermission actionPermission, + DefaultResourcesService defaultResourcesService) { super( scheduler, validator, @@ -44,6 +47,7 @@ public class ActionCollectionServiceImpl extends ActionCollectionServiceCEImpl i applicationService, responseUtils, applicationPermission, - actionPermission); + actionPermission, + defaultResourcesService); } } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/actioncollections/defaultresources/ActionCollectionDTODefaultResourcesServiceCEImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/actioncollections/defaultresources/ActionCollectionDTODefaultResourcesServiceCEImpl.java new file mode 100644 index 0000000000..fd3e27e904 --- /dev/null +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/actioncollections/defaultresources/ActionCollectionDTODefaultResourcesServiceCEImpl.java @@ -0,0 +1,44 @@ +package com.appsmith.server.actioncollections.defaultresources; + +import com.appsmith.external.models.DefaultResources; +import com.appsmith.server.defaultresources.DefaultResourcesServiceCE; +import com.appsmith.server.dtos.ActionCollectionDTO; +import org.springframework.stereotype.Service; +import org.springframework.util.StringUtils; + +@Service +public class ActionCollectionDTODefaultResourcesServiceCEImpl + implements DefaultResourcesServiceCE { + + @Override + public ActionCollectionDTO initialize( + ActionCollectionDTO domainObject, String branchName, boolean resetExistingValues) { + DefaultResources existingDefaultResources = domainObject.getDefaultResources(); + DefaultResources defaultResources = new DefaultResources(); + + String defaultPageId = domainObject.getPageId(); + + if (existingDefaultResources != null && !resetExistingValues) { + // Check if there are properties to be copied over from existing + if (StringUtils.hasText(existingDefaultResources.getPageId())) { + defaultPageId = existingDefaultResources.getPageId(); + } + } + + defaultResources.setPageId(defaultPageId); + + domainObject.setDefaultResources(defaultResources); + return domainObject; + } + + @Override + public ActionCollectionDTO setFromOtherBranch( + ActionCollectionDTO domainObject, ActionCollectionDTO defaultDomainObject, String branchName) { + DefaultResources defaultResources = new DefaultResources(); + + defaultResources.setPageId(defaultDomainObject.getDefaultResources().getPageId()); + + domainObject.setDefaultResources(defaultResources); + return domainObject; + } +} diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/actioncollections/defaultresources/ActionCollectionDTODefaultResourcesServiceImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/actioncollections/defaultresources/ActionCollectionDTODefaultResourcesServiceImpl.java new file mode 100644 index 0000000000..9772d5b9c9 --- /dev/null +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/actioncollections/defaultresources/ActionCollectionDTODefaultResourcesServiceImpl.java @@ -0,0 +1,9 @@ +package com.appsmith.server.actioncollections.defaultresources; + +import com.appsmith.server.defaultresources.DefaultResourcesService; +import com.appsmith.server.dtos.ActionCollectionDTO; +import org.springframework.stereotype.Service; + +@Service +public class ActionCollectionDTODefaultResourcesServiceImpl extends ActionCollectionDTODefaultResourcesServiceCEImpl + implements DefaultResourcesService {} diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/actioncollections/defaultresources/ActionCollectionDefaultResourcesServiceCEImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/actioncollections/defaultresources/ActionCollectionDefaultResourcesServiceCEImpl.java new file mode 100644 index 0000000000..fc357a048c --- /dev/null +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/actioncollections/defaultresources/ActionCollectionDefaultResourcesServiceCEImpl.java @@ -0,0 +1,56 @@ +package com.appsmith.server.actioncollections.defaultresources; + +import com.appsmith.external.models.DefaultResources; +import com.appsmith.server.defaultresources.DefaultResourcesServiceCE; +import com.appsmith.server.domains.ActionCollection; +import org.springframework.stereotype.Service; +import org.springframework.util.StringUtils; + +@Service +public class ActionCollectionDefaultResourcesServiceCEImpl implements DefaultResourcesServiceCE { + + @Override + public ActionCollection initialize(ActionCollection domainObject, String branchName, boolean resetExistingValues) { + DefaultResources existingDefaultResources = domainObject.getDefaultResources(); + DefaultResources defaultResources = new DefaultResources(); + + String defaultApplicationId = domainObject.getApplicationId(); + String defaultCollectionId = domainObject.getId(); + + if (existingDefaultResources != null && !resetExistingValues) { + // Check if there are properties to be copied over from existing + if (StringUtils.hasText(existingDefaultResources.getApplicationId())) { + defaultApplicationId = existingDefaultResources.getApplicationId(); + } + + if (StringUtils.hasText(existingDefaultResources.getCollectionId())) { + defaultCollectionId = existingDefaultResources.getCollectionId(); + } + } + + defaultResources.setCollectionId(defaultCollectionId); + defaultResources.setApplicationId(defaultApplicationId); + defaultResources.setBranchName(branchName); + + domainObject.setDefaultResources(defaultResources); + return domainObject; + } + + @Override + public ActionCollection setFromOtherBranch( + ActionCollection domainObject, ActionCollection branchedDomainObject, String branchName) { + DefaultResources defaultResources = domainObject.getDefaultResources(); + if (defaultResources == null) { + defaultResources = new DefaultResources(); + } + + DefaultResources otherDefaultResources = branchedDomainObject.getDefaultResources(); + + defaultResources.setCollectionId(otherDefaultResources.getCollectionId()); + defaultResources.setApplicationId(otherDefaultResources.getApplicationId()); + defaultResources.setBranchName(branchName); + + domainObject.setDefaultResources(defaultResources); + return domainObject; + } +} diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/actioncollections/defaultresources/ActionCollectionDefaultResourcesServiceImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/actioncollections/defaultresources/ActionCollectionDefaultResourcesServiceImpl.java new file mode 100644 index 0000000000..521bb023c3 --- /dev/null +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/actioncollections/defaultresources/ActionCollectionDefaultResourcesServiceImpl.java @@ -0,0 +1,9 @@ +package com.appsmith.server.actioncollections.defaultresources; + +import com.appsmith.server.defaultresources.DefaultResourcesService; +import com.appsmith.server.domains.ActionCollection; +import org.springframework.stereotype.Service; + +@Service +public class ActionCollectionDefaultResourcesServiceImpl extends ActionCollectionDefaultResourcesServiceCEImpl + implements DefaultResourcesService {} diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/actioncollections/imports/ActionCollectionImportableServiceCEImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/actioncollections/imports/ActionCollectionImportableServiceCEImpl.java index e6ed1c4191..e0dd9efc89 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/actioncollections/imports/ActionCollectionImportableServiceCEImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/actioncollections/imports/ActionCollectionImportableServiceCEImpl.java @@ -4,6 +4,7 @@ import com.appsmith.external.models.DefaultResources; import com.appsmith.external.models.Policy; import com.appsmith.server.actioncollections.base.ActionCollectionService; import com.appsmith.server.constants.FieldName; +import com.appsmith.server.defaultresources.DefaultResourcesService; import com.appsmith.server.domains.ActionCollection; import com.appsmith.server.domains.Application; import com.appsmith.server.domains.NewPage; @@ -23,6 +24,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.bson.types.ObjectId; +import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import java.util.ArrayList; @@ -39,6 +41,8 @@ import static com.appsmith.external.helpers.AppsmithBeanUtils.copyNestedNonNullP public class ActionCollectionImportableServiceCEImpl implements ImportableServiceCE { private final ActionCollectionService actionCollectionService; private final ActionCollectionRepository repository; + private final DefaultResourcesService defaultResourcesService; + private final DefaultResourcesService dtoDefaultResourcesService; // Requires pageNameMap, pageNameToOldNameMap, pluginMap and actionResultDTO to be present in importable resources. // Updates actionCollectionResultDTO in importable resources. @@ -135,17 +139,16 @@ public class ActionCollectionImportableServiceCEImpl implements ImportableServic final String workspaceId = importedApplication.getWorkspaceId(); // Map of gitSyncId to actionCollection of the existing records in DB - Mono> actionCollectionsInCurrentAppMono = repository - .findByApplicationId(importedApplication.getId()) - .filter(collection -> collection.getGitSyncId() != null) - .collectMap(ActionCollection::getGitSyncId); + Mono> actionCollectionsInCurrentAppMono = + getCollectionsInCurrentAppFlux(importedApplication) + .filter(collection -> collection.getGitSyncId() != null) + .collectMap(ActionCollection::getGitSyncId); Mono> actionCollectionsInBranchesMono; if (importedApplication.getGitApplicationMetadata() != null) { final String defaultApplicationId = importedApplication.getGitApplicationMetadata().getDefaultApplicationId(); - actionCollectionsInBranchesMono = repository - .findByDefaultApplicationId(defaultApplicationId, Optional.empty()) + actionCollectionsInBranchesMono = getCollectionsInOtherBranchesFlux(defaultApplicationId) .filter(actionCollection -> actionCollection.getGitSyncId() != null) .collectMap(ActionCollection::getGitSyncId); } else { @@ -215,6 +218,37 @@ public class ActionCollectionImportableServiceCEImpl implements ImportableServic actionCollection.setWorkspaceId(workspaceId); actionCollection.setApplicationId(importedApplication.getId()); + if (importedApplication.getGitApplicationMetadata() != null) { + final String defaultApplicationId = importedApplication + .getGitApplicationMetadata() + .getDefaultApplicationId(); + if (actionsCollectionsInBranches.containsKey(actionCollection.getGitSyncId())) { + ActionCollection branchedActionCollection = + getExistingCollectionForImportedCollection( + mappedImportableResourcesDTO, + actionsCollectionsInBranches, + actionCollection); + defaultResourcesService.setFromOtherBranch( + actionCollection, + branchedActionCollection, + importingMetaDTO.getBranchName()); + dtoDefaultResourcesService.setFromOtherBranch( + actionCollection.getUnpublishedCollection(), + branchedActionCollection.getUnpublishedCollection(), + importingMetaDTO.getBranchName()); + } else { + defaultResourcesService.initialize( + actionCollection, importingMetaDTO.getBranchName(), false); + actionCollection + .getDefaultResources() + .setApplicationId(defaultApplicationId); + dtoDefaultResourcesService.initialize( + actionCollection.getUnpublishedCollection(), + importingMetaDTO.getBranchName(), + false); + } + } + // Check if the action has gitSyncId and if it's already in DB if (existingAppContainsCollection( actionsCollectionsInCurrentApp, actionCollection)) { @@ -226,26 +260,12 @@ public class ActionCollectionImportableServiceCEImpl implements ImportableServic actionsCollectionsInCurrentApp, actionCollection); - Set existingPolicy = existingActionCollection.getPolicies(); - copyNestedNonNullProperties(actionCollection, existingActionCollection); + updateExistingCollection( + importingMetaDTO, + mappedImportableResourcesDTO, + actionCollection, + existingActionCollection); - populateDomainMappedReferences( - mappedImportableResourcesDTO, existingActionCollection); - - // Update branchName - existingActionCollection - .getDefaultResources() - .setBranchName(importingMetaDTO.getBranchName()); - // Recover the deleted state present in DB from imported actionCollection - existingActionCollection - .getUnpublishedCollection() - .setDeletedAt(actionCollection - .getUnpublishedCollection() - .getDeletedAt()); - existingActionCollection.setDeletedAt(actionCollection.getDeletedAt()); - existingActionCollection.setPolicies(existingPolicy); - - existingActionCollection.updateForBulkWriteOperation(); existingActionCollections.add(existingActionCollection); resultDTO.getSavedActionCollectionIds().add(existingActionCollection.getId()); resultDTO @@ -261,29 +281,6 @@ public class ActionCollectionImportableServiceCEImpl implements ImportableServic parentPage.getId()); } - if (importedApplication.getGitApplicationMetadata() != null) { - final String defaultApplicationId = importedApplication - .getGitApplicationMetadata() - .getDefaultApplicationId(); - if (actionsCollectionsInBranches.containsKey( - actionCollection.getGitSyncId())) { - ActionCollection branchedActionCollection = - getExistingCollectionForImportedCollection( - mappedImportableResourcesDTO, - actionsCollectionsInBranches, - actionCollection); - actionCollectionService.populateDefaultResources( - actionCollection, - branchedActionCollection, - importingMetaDTO.getBranchName()); - } else { - DefaultResources defaultResources = new DefaultResources(); - defaultResources.setApplicationId(defaultApplicationId); - defaultResources.setBranchName(importingMetaDTO.getBranchName()); - actionCollection.setDefaultResources(defaultResources); - } - } - // this will generate the id and other auto generated fields e.g. createdAt actionCollection.updateForBulkWriteOperation(); actionCollectionService.generateAndSetPolicies(parentPage, actionCollection); @@ -323,6 +320,44 @@ public class ActionCollectionImportableServiceCEImpl implements ImportableServic }); } + protected Flux getCollectionsInCurrentAppFlux(Application importedApplication) { + return repository.findByApplicationId(importedApplication.getId()); + } + + protected Flux getCollectionsInOtherBranchesFlux(String defaultApplicationId) { + return repository.findByDefaultApplicationId(defaultApplicationId, Optional.empty()); + } + + private void updateExistingCollection( + ImportingMetaDTO importingMetaDTO, + MappedImportableResourcesDTO mappedImportableResourcesDTO, + ActionCollection actionCollection, + ActionCollection existingActionCollection) { + Set existingPolicy = existingActionCollection.getPolicies(); + + updateImportableCollectionFromExistingCollection(existingActionCollection, actionCollection); + + copyNestedNonNullProperties(actionCollection, existingActionCollection); + + populateDomainMappedReferences(mappedImportableResourcesDTO, existingActionCollection); + + // Update branchName + existingActionCollection.getDefaultResources().setBranchName(importingMetaDTO.getBranchName()); + // Recover the deleted state present in DB from imported actionCollection + existingActionCollection + .getUnpublishedCollection() + .setDeletedAt(actionCollection.getUnpublishedCollection().getDeletedAt()); + existingActionCollection.setDeletedAt(actionCollection.getDeletedAt()); + existingActionCollection.setPolicies(existingPolicy); + + existingActionCollection.updateForBulkWriteOperation(); + } + + protected void updateImportableCollectionFromExistingCollection( + ActionCollection existingActionCollection, ActionCollection actionCollection) { + // Nothing to update from the existing action collection + } + protected ActionCollection getExistingCollectionForImportedCollection( MappedImportableResourcesDTO mappedImportableResourcesDTO, Map actionsCollectionsInCurrentApp, diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/actioncollections/imports/ActionCollectionImportableServiceImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/actioncollections/imports/ActionCollectionImportableServiceImpl.java index 522cfa338f..c65de12e28 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/actioncollections/imports/ActionCollectionImportableServiceImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/actioncollections/imports/ActionCollectionImportableServiceImpl.java @@ -1,7 +1,9 @@ package com.appsmith.server.actioncollections.imports; import com.appsmith.server.actioncollections.base.ActionCollectionService; +import com.appsmith.server.defaultresources.DefaultResourcesService; import com.appsmith.server.domains.ActionCollection; +import com.appsmith.server.dtos.ActionCollectionDTO; import com.appsmith.server.imports.importable.ImportableService; import com.appsmith.server.repositories.ActionCollectionRepository; import org.springframework.stereotype.Service; @@ -10,7 +12,10 @@ import org.springframework.stereotype.Service; public class ActionCollectionImportableServiceImpl extends ActionCollectionImportableServiceCEImpl implements ImportableService { public ActionCollectionImportableServiceImpl( - ActionCollectionService actionCollectionService, ActionCollectionRepository repository) { - super(actionCollectionService, repository); + ActionCollectionService actionCollectionService, + ActionCollectionRepository repository, + DefaultResourcesService defaultResourcesService, + DefaultResourcesService dtoDefaultResourcesService) { + super(actionCollectionService, repository, defaultResourcesService, dtoDefaultResourcesService); } } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/defaultresources/DefaultResourcesService.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/defaultresources/DefaultResourcesService.java new file mode 100644 index 0000000000..5eca6eedd2 --- /dev/null +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/defaultresources/DefaultResourcesService.java @@ -0,0 +1,3 @@ +package com.appsmith.server.defaultresources; + +public interface DefaultResourcesService extends DefaultResourcesServiceCE {} diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/defaultresources/DefaultResourcesServiceCE.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/defaultresources/DefaultResourcesServiceCE.java new file mode 100644 index 0000000000..362d160ca9 --- /dev/null +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/defaultresources/DefaultResourcesServiceCE.java @@ -0,0 +1,8 @@ +package com.appsmith.server.defaultresources; + +public interface DefaultResourcesServiceCE { + + T initialize(T domainObject, String branchName, boolean resetExistingValues); + + T setFromOtherBranch(T domainObject, T defaultDomainObject, String branchName); +} diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/dtos/ce/ActionCollectionCE_DTO.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/dtos/ce/ActionCollectionCE_DTO.java index 3b46f97778..6c2d10294d 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/dtos/ce/ActionCollectionCE_DTO.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/dtos/ce/ActionCollectionCE_DTO.java @@ -139,14 +139,32 @@ public class ActionCollectionCE_DTO { this.setApplicationId(actionCollection.getApplicationId()); this.setWorkspaceId(actionCollection.getWorkspaceId()); this.setUserPermissions(actionCollection.userPermissions); - copyNewFieldValuesIntoOldObject(actionCollection.getDefaultResources(), this.getDefaultResources()); + if (this.getDefaultResources() == null) { + this.setDefaultResources(actionCollection.getDefaultResources()); + } else { + copyNewFieldValuesIntoOldObject(actionCollection.getDefaultResources(), this.getDefaultResources()); + } } public void sanitiseForExport() { + this.resetTransientFields(); this.setDefaultResources(null); this.setDefaultToBranchedActionIdsMap(null); this.setDefaultToBranchedArchivedActionIdsMap(null); this.setActionIds(null); this.setArchivedActionIds(null); } + + public String getUserExecutableName() { + return this.getName(); + } + + protected void resetTransientFields() { + this.setId(null); + this.setWorkspaceId(null); + this.setApplicationId(null); + this.setErrorReports(null); + this.setActions(List.of()); + this.setArchivedActions(List.of()); + } } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/GitFileUtils.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/GitFileUtils.java index 2e38b7b341..da4957fa60 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/GitFileUtils.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/GitFileUtils.java @@ -1,659 +1,28 @@ package com.appsmith.server.helpers; -import com.appsmith.external.constants.AnalyticsEvents; import com.appsmith.external.git.FileInterface; -import com.appsmith.external.helpers.Stopwatch; -import com.appsmith.external.models.ActionDTO; -import com.appsmith.external.models.ApplicationGitReference; -import com.appsmith.external.models.BaseDomain; -import com.appsmith.external.models.DatasourceStorage; -import com.appsmith.external.models.PluginType; import com.appsmith.git.helpers.FileUtilsImpl; -import com.appsmith.server.constants.FieldName; -import com.appsmith.server.domains.ActionCollection; -import com.appsmith.server.domains.Application; -import com.appsmith.server.domains.ApplicationPage; -import com.appsmith.server.domains.CustomJSLib; -import com.appsmith.server.domains.NewAction; -import com.appsmith.server.domains.NewPage; -import com.appsmith.server.domains.Theme; -import com.appsmith.server.dtos.ActionCollectionDTO; -import com.appsmith.server.dtos.ApplicationJson; -import com.appsmith.server.dtos.PageDTO; -import com.appsmith.server.exceptions.AppsmithError; -import com.appsmith.server.exceptions.AppsmithException; +import com.appsmith.server.helpers.ce.GitFileUtilsCE; import com.appsmith.server.services.AnalyticsService; import com.appsmith.server.services.SessionUserService; import com.google.gson.Gson; -import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import net.minidev.json.JSONObject; -import net.minidev.json.parser.JSONParser; -import net.minidev.json.parser.ParseException; -import org.apache.commons.collections.PredicateUtils; -import org.apache.commons.lang3.StringUtils; -import org.eclipse.jgit.api.errors.GitAPIException; -import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Import; import org.springframework.stereotype.Component; -import reactor.core.Exceptions; -import reactor.core.publisher.Mono; - -import java.io.IOException; -import java.lang.reflect.Field; -import java.lang.reflect.Type; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static com.appsmith.external.constants.GitConstants.NAME_SEPARATOR; -import static com.appsmith.external.helpers.AppsmithBeanUtils.copyNestedNonNullProperties; -import static com.appsmith.external.helpers.AppsmithBeanUtils.copyProperties; -import static com.appsmith.server.constants.FieldName.ACTION_COLLECTION_LIST; -import static com.appsmith.server.constants.FieldName.ACTION_LIST; -import static com.appsmith.server.constants.FieldName.CUSTOM_JS_LIB_LIST; -import static com.appsmith.server.constants.FieldName.DATASOURCE_LIST; -import static com.appsmith.server.constants.FieldName.DECRYPTED_FIELDS; -import static com.appsmith.server.constants.FieldName.EDIT_MODE_THEME; -import static com.appsmith.server.constants.FieldName.EXPORTED_APPLICATION; -import static com.appsmith.server.constants.FieldName.PAGE_LIST; @Slf4j -@RequiredArgsConstructor @Component @Import({FileUtilsImpl.class}) -public class GitFileUtils { - - private final FileInterface fileUtils; - private final AnalyticsService analyticsService; - private final SessionUserService sessionUserService; +public class GitFileUtils extends GitFileUtilsCE { private final Gson gson; - // Number of seconds after lock file is stale - @Value("${appsmith.index.lock.file.time}") - public final int INDEX_LOCK_FILE_STALE_TIME = 300; - - // Only include the application helper fields in metadata object - private static final Set blockedMetadataFields = Set.of( - EXPORTED_APPLICATION, - DATASOURCE_LIST, - PAGE_LIST, - ACTION_LIST, - ACTION_COLLECTION_LIST, - DECRYPTED_FIELDS, - EDIT_MODE_THEME, - CUSTOM_JS_LIB_LIST); - - /** - * This method will save the complete application in the local repo directory. - * Path to repo will be : ./container-volumes/git-repo/workspaceId/defaultApplicationId/repoName/{application_data} - * - * @param baseRepoSuffix path suffix used to create a local repo path - * @param applicationJson application reference object from which entire application can be rehydrated - * @param branchName name of the branch for the current application - * @return repo path where the application is stored - */ - public Mono saveApplicationToLocalRepo( - Path baseRepoSuffix, ApplicationJson applicationJson, String branchName) - throws IOException, GitAPIException { - /* - 1. Checkout to branch - 2. Create application reference for appsmith-git module - 3. Save application to git repo - */ - - ApplicationGitReference applicationReference = createApplicationReference(applicationJson); - // Save application to git repo - try { - return fileUtils.saveApplicationToGitRepo(baseRepoSuffix, applicationReference, branchName); - } catch (IOException | GitAPIException e) { - log.error("Error occurred while saving files to local git repo: ", e); - throw Exceptions.propagate(e); - } - } - - public Mono saveApplicationToLocalRepoWithAnalytics( - Path baseRepoSuffix, ApplicationJson applicationJson, String branchName) - throws IOException, GitAPIException { - - /* - 1. Checkout to branch - 2. Create application reference for appsmith-git module - 3. Save application to git repo - */ - Stopwatch stopwatch = new Stopwatch(AnalyticsEvents.GIT_SERIALIZE_APP_RESOURCES_TO_LOCAL_FILE.getEventName()); - // Save application to git repo - try { - Mono repoPathMono = saveApplicationToLocalRepo(baseRepoSuffix, applicationJson, branchName); - return Mono.zip(repoPathMono, sessionUserService.getCurrentUser()).flatMap(tuple -> { - stopwatch.stopTimer(); - Path repoPath = tuple.getT1(); - // Path to repo will be : ./container-volumes/git-repo/workspaceId/defaultApplicationId/repoName/ - final Map data = Map.of( - FieldName.APPLICATION_ID, - repoPath.getParent().getFileName().toString(), - FieldName.ORGANIZATION_ID, - repoPath.getParent().getParent().getFileName().toString(), - FieldName.FLOW_NAME, - stopwatch.getFlow(), - "executionTime", - stopwatch.getExecutionTime()); - return analyticsService - .sendEvent( - AnalyticsEvents.UNIT_EXECUTION_TIME.getEventName(), - tuple.getT2().getUsername(), - data) - .thenReturn(repoPath); - }); - } catch (IOException | GitAPIException e) { - log.error("Error occurred while saving files to local git repo: ", e); - throw Exceptions.propagate(e); - } - } - - public Mono saveApplicationToLocalRepo( - String workspaceId, - String defaultApplicationId, - String repoName, - ApplicationJson applicationJson, - String branchName) - throws GitAPIException, IOException { - Path baseRepoSuffix = Paths.get(workspaceId, defaultApplicationId, repoName); - - return saveApplicationToLocalRepo(baseRepoSuffix, applicationJson, branchName); - } - - /** - * Method to convert application resources to the structure which can be serialised by appsmith-git module for - * serialisation - * - * @param applicationJson application resource including actions, jsobjects, pages - * @return resource which can be saved to file system - */ - public ApplicationGitReference createApplicationReference(ApplicationJson applicationJson) { - ApplicationGitReference applicationReference = new ApplicationGitReference(); - - Application application = applicationJson.getExportedApplication(); - removeUnwantedFieldsFromApplication(application); - // Pass application reference - applicationReference.setApplication(applicationJson.getExportedApplication()); - - // No need to commit publish mode theme as it leads to conflict resolution at both the places if any - applicationJson.setPublishedTheme(null); - - // Pass metadata - Iterable keys = getAllFields(applicationJson) - .map(Field::getName) - .filter(name -> !blockedMetadataFields.contains(name)) - .collect(Collectors.toList()); - - ApplicationJson applicationMetadata = new ApplicationJson(); - Map> updatedResources = applicationJson.getUpdatedResources(); - applicationJson.setUpdatedResources(null); - copyProperties(applicationJson, applicationMetadata, keys); - applicationReference.setMetadata(applicationMetadata); - - // Remove internal fields from the themes - removeUnwantedFieldsFromBaseDomain(applicationJson.getEditModeTheme()); - - applicationReference.setTheme(applicationJson.getEditModeTheme()); - - // Insert only active pages which will then be committed to repo as individual file - Map resourceMap = new HashMap<>(); - Map resourceMapBody = new HashMap<>(); - Map dslBody = new HashMap<>(); - applicationJson.getPageList().stream() - // As we are expecting the commit will happen only after the application is published, so we can safely - // assume if the unpublished version is deleted entity should not be committed to git - .filter(newPage -> newPage.getUnpublishedPage() != null - && newPage.getUnpublishedPage().getDeletedAt() == null) - .forEach(newPage -> { - String pageName = newPage.getUnpublishedPage() != null - ? newPage.getUnpublishedPage().getName() - : newPage.getPublishedPage().getName(); - removeUnwantedFieldsFromPage(newPage); - JSONObject dsl = - newPage.getUnpublishedPage().getLayouts().get(0).getDsl(); - // Get MainContainer widget data, remove the children and club with Canvas.json file - JSONObject mainContainer = new JSONObject(dsl); - mainContainer.remove("children"); - newPage.getUnpublishedPage().getLayouts().get(0).setDsl(mainContainer); - // pageName will be used for naming the json file - dslBody.put(pageName, dsl.toString()); - resourceMap.put(pageName, newPage); - }); - - applicationReference.setPages(new HashMap<>(resourceMap)); - applicationReference.setPageDsl(new HashMap<>(dslBody)); - resourceMap.clear(); - resourceMapBody.clear(); - - // Insert active actions and also assign the keys which later will be used for saving the resource in actual - // filepath - // For actions, we are referring to validNames to maintain unique file names as just name - // field don't guarantee unique constraint for actions within JSObject - // queryValidName_pageName => nomenclature for the keys - applicationJson.getActionList().stream() - // As we are expecting the commit will happen only after the application is published, so we can safely - // assume if the unpublished version is deleted entity should not be committed to git - .filter(newAction -> newAction.getUnpublishedAction() != null - && newAction.getUnpublishedAction().getDeletedAt() == null) - .forEach(newAction -> { - String prefix = newAction.getUnpublishedAction() != null - ? newAction.getUnpublishedAction().getValidName() - + NAME_SEPARATOR - + newAction.getUnpublishedAction().getPageId() - : newAction.getPublishedAction().getValidName() - + NAME_SEPARATOR - + newAction.getPublishedAction().getPageId(); - removeUnwantedFieldFromAction(newAction); - String body = newAction - .getUnpublishedAction() - .getActionConfiguration() - .getBody() - != null - ? newAction - .getUnpublishedAction() - .getActionConfiguration() - .getBody() - : ""; - - // This is a special case where we are handling REMOTE type plugins based actions such as Twilio - // The user configured values are stored in a attribute called formData which is a map unlike the - // body - if (newAction.getPluginType().toString().equals("REMOTE") - && newAction - .getUnpublishedAction() - .getActionConfiguration() - .getFormData() - != null) { - body = new Gson() - .toJson( - newAction - .getUnpublishedAction() - .getActionConfiguration() - .getFormData(), - Map.class); - newAction - .getUnpublishedAction() - .getActionConfiguration() - .setFormData(null); - } - // This is a special case where we are handling JS actions as we don't want to commit the body of JS - // actions - if (newAction.getPluginType().equals(PluginType.JS)) { - newAction - .getUnpublishedAction() - .getActionConfiguration() - .setBody(null); - newAction.getUnpublishedAction().setJsonPathKeys(null); - } else { - // For the regular actions we save the body field to git repo - resourceMapBody.put(prefix, body); - } - resourceMap.put(prefix, newAction); - }); - applicationReference.setActions(new HashMap<>(resourceMap)); - applicationReference.setActionBody(new HashMap<>(resourceMapBody)); - resourceMap.clear(); - resourceMapBody.clear(); - - // Insert JSOObjects and also assign the keys which later will be used for saving the resource in actual - // filepath - // JSObjectName_pageName => nomenclature for the keys - Map resourceMapActionCollectionBody = new HashMap<>(); - applicationJson.getActionCollectionList().stream() - // As we are expecting the commit will happen only after the application is published, so we can safely - // assume if the unpublished version is deleted entity should not be committed to git - .filter(collection -> collection.getUnpublishedCollection() != null - && collection.getUnpublishedCollection().getDeletedAt() == null) - .forEach(actionCollection -> { - String prefix = actionCollection.getUnpublishedCollection() != null - ? actionCollection.getUnpublishedCollection().getName() - + NAME_SEPARATOR - + actionCollection - .getUnpublishedCollection() - .getPageId() - : actionCollection.getPublishedCollection().getName() - + NAME_SEPARATOR - + actionCollection.getPublishedCollection().getPageId(); - removeUnwantedFieldFromActionCollection(actionCollection); - - String body = actionCollection.getUnpublishedCollection().getBody() != null - ? actionCollection.getUnpublishedCollection().getBody() - : ""; - actionCollection.getUnpublishedCollection().setBody(null); - resourceMapActionCollectionBody.put(prefix, body); - resourceMap.put(prefix, actionCollection); - }); - applicationReference.setActionCollections(new HashMap<>(resourceMap)); - applicationReference.setActionCollectionBody(new HashMap<>(resourceMapActionCollectionBody)); - applicationReference.setUpdatedResources(updatedResources); - resourceMap.clear(); - resourceMapActionCollectionBody.clear(); - - // Send datasources - applicationJson.getDatasourceList().forEach(datasource -> { - removeUnwantedFieldsFromDatasource(datasource); - resourceMap.put(datasource.getName(), datasource); - }); - applicationReference.setDatasources(new HashMap<>(resourceMap)); - resourceMap.clear(); - - applicationJson.getCustomJSLibList().forEach(jsLib -> { - removeUnwantedFieldsFromBaseDomain(jsLib); - resourceMap.put(jsLib.getUidString(), jsLib); - }); - applicationReference.setJsLibraries(new HashMap<>(resourceMap)); - resourceMap.clear(); - - return applicationReference; - } - - protected Stream getAllFields(ApplicationJson applicationJson) { - Class currentType = applicationJson.getClass(); - - Set> classes = new HashSet<>(); - - while (currentType != null) { - classes.add(currentType); - currentType = currentType.getSuperclass(); - } - - return classes.stream().flatMap(currentClass -> Arrays.stream(currentClass.getDeclaredFields())); - } - - /** - * Method to reconstruct the application from the local git repo - * - * @param workspaceId To which workspace application needs to be rehydrated - * @param defaultApplicationId Root application for the current branched application - * @param branchName for which branch the application needs to rehydrate - * @return application reference from which entire application can be rehydrated - */ - public Mono reconstructApplicationJsonFromGitRepoWithAnalytics( - String workspaceId, String defaultApplicationId, String repoName, String branchName) { - Stopwatch stopwatch = new Stopwatch(AnalyticsEvents.GIT_DESERIALIZE_APP_RESOURCES_FROM_FILE.getEventName()); - - return Mono.zip( - reconstructApplicationJsonFromGitRepo(workspaceId, defaultApplicationId, repoName, branchName), - sessionUserService.getCurrentUser()) - .flatMap(tuple -> { - stopwatch.stopTimer(); - final Map data = Map.of( - FieldName.APPLICATION_ID, - defaultApplicationId, - FieldName.ORGANIZATION_ID, - workspaceId, - FieldName.FLOW_NAME, - stopwatch.getFlow(), - "executionTime", - stopwatch.getExecutionTime()); - return analyticsService - .sendEvent( - AnalyticsEvents.UNIT_EXECUTION_TIME.getEventName(), - tuple.getT2().getUsername(), - data) - .thenReturn(tuple.getT1()); - }); - } - - public Mono reconstructApplicationJsonFromGitRepo( - String workspaceId, String defaultApplicationId, String repoName, String branchName) { - - Mono appReferenceMono = fileUtils.reconstructApplicationReferenceFromGitRepo( - workspaceId, defaultApplicationId, repoName, branchName); - return appReferenceMono.map(applicationReference -> { - // Extract application metadata from the json - ApplicationJson metadata = - getApplicationResource(applicationReference.getMetadata(), ApplicationJson.class); - ApplicationJson applicationJson = getApplicationJsonFromGitReference(applicationReference); - copyNestedNonNullProperties(metadata, applicationJson); - return applicationJson; - }); - } - - private List getApplicationResource(Map resources, Type type) { - - List deserializedResources = new ArrayList<>(); - if (!CollectionUtils.isNullOrEmpty(resources)) { - for (Map.Entry resource : resources.entrySet()) { - deserializedResources.add(getApplicationResource(resource.getValue(), type)); - } - } - return deserializedResources; - } - - public T getApplicationResource(Object resource, Type type) { - if (resource == null) { - return null; - } - return gson.fromJson(gson.toJson(resource), type); - } - - /** - * Once the user connects the existing application to a remote repo, we will initialize the repo with Readme.md - - * Url to the deployed app(view and edit mode) - * Link to discord channel for support - * Link to appsmith documentation for Git related operations - * Welcome message - * - * @param baseRepoSuffix path suffix used to create a branch repo path as per worktree implementation - * @param viewModeUrl URL to deployed version of the application view only mode - * @param editModeUrl URL to deployed version of the application edit mode - * @return Path where the Application is stored - */ - public Mono initializeReadme(Path baseRepoSuffix, String viewModeUrl, String editModeUrl) throws IOException { - return fileUtils - .initializeReadme(baseRepoSuffix, viewModeUrl, editModeUrl) - .onErrorResume(e -> Mono.error(new AppsmithException(AppsmithError.GIT_FILE_SYSTEM_ERROR, e))); - } - - /** - * When the user clicks on detach remote, we need to remove the repo from the file system - * - * @param baseRepoSuffix path suffix used to create a branch repo path as per worktree implementation - * @return success on remove of file system - */ - public Mono deleteLocalRepo(Path baseRepoSuffix) { - return fileUtils.deleteLocalRepo(baseRepoSuffix); - } - - public Mono checkIfDirectoryIsEmpty(Path baseRepoSuffix) throws IOException { - return fileUtils - .checkIfDirectoryIsEmpty(baseRepoSuffix) - .onErrorResume(e -> Mono.error(new AppsmithException(AppsmithError.GIT_FILE_SYSTEM_ERROR, e))); - } - - private void removeUnwantedFieldsFromBaseDomain(BaseDomain baseDomain) { - baseDomain.setPolicies(null); - baseDomain.setUserPermissions(null); - } - - private void removeUnwantedFieldsFromDatasource(DatasourceStorage datasource) { - datasource.setInvalids(null); - removeUnwantedFieldsFromBaseDomain(datasource); - } - - private void removeUnwantedFieldsFromPage(NewPage page) { - // As we are publishing the app and then committing to git we expect the published and unpublished PageDTO will - // be same, so we only commit unpublished PageDTO. - page.setPublishedPage(null); - removeUnwantedFieldsFromBaseDomain(page); - } - - private void removeUnwantedFieldsFromApplication(Application application) { - // Don't commit application name as while importing we are using the repoName as application name - application.setName(null); - application.setPublishedPages(null); - application.setIsPublic(null); - application.setSlug(null); - application.setPublishedApplicationDetail(null); - removeUnwantedFieldsFromBaseDomain(application); - // we can call the sanitiseToExportDBObject() from BaseDomain as well here - } - - private void removeUnwantedFieldFromAction(NewAction action) { - // As we are publishing the app and then committing to git we expect the published and unpublished ActionDTO - // will be same, so we only commit unpublished ActionDTO. - action.setPublishedAction(null); - removeUnwantedFieldsFromBaseDomain(action); - } - - private void removeUnwantedFieldFromActionCollection(ActionCollection actionCollection) { - // As we are publishing the app and then committing to git we expect the published and unpublished - // ActionCollectionDTO will be same, so we only commit unpublished ActionCollectionDTO. - actionCollection.setPublishedCollection(null); - removeUnwantedFieldsFromBaseDomain(actionCollection); - } - - private ApplicationJson getApplicationJsonFromGitReference(ApplicationGitReference applicationReference) { - ApplicationJson applicationJson = new ApplicationJson(); - // Extract application data from the json - Application application = getApplicationResource(applicationReference.getApplication(), Application.class); - applicationJson.setExportedApplication(application); - applicationJson.setEditModeTheme(getApplicationResource(applicationReference.getTheme(), Theme.class)); - // Clone the edit mode theme to published theme as both should be same for git connected application because we - // do deploy and push as a single operation - applicationJson.setPublishedTheme(applicationJson.getEditModeTheme()); - - if (application != null && !CollectionUtils.isNullOrEmpty(application.getPages())) { - // Remove null values - org.apache.commons.collections.CollectionUtils.filter( - application.getPages(), PredicateUtils.notNullPredicate()); - // Create a deep clone of application pages to update independently - application.setViewMode(false); - final List applicationPages = - new ArrayList<>(application.getPages().size()); - application - .getPages() - .forEach(applicationPage -> - applicationPages.add(gson.fromJson(gson.toJson(applicationPage), ApplicationPage.class))); - application.setPublishedPages(applicationPages); - } - - List customJSLibList = - getApplicationResource(applicationReference.getJsLibraries(), CustomJSLib.class); - - // remove the duplicate js libraries if there is any - List customJSLibListWithoutDuplicates = new ArrayList<>(new HashSet<>(customJSLibList)); - - applicationJson.setCustomJSLibList(customJSLibListWithoutDuplicates); - - // Extract pages - List pages = getApplicationResource(applicationReference.getPages(), NewPage.class); - // Remove null values - org.apache.commons.collections.CollectionUtils.filter(pages, PredicateUtils.notNullPredicate()); - // Set the DSL to page object before saving - Map pageDsl = applicationReference.getPageDsl(); - pages.forEach(page -> { - JSONParser jsonParser = new JSONParser(); - try { - if (pageDsl != null && pageDsl.get(page.getUnpublishedPage().getName()) != null) { - page.getUnpublishedPage().getLayouts().get(0).setDsl((JSONObject) jsonParser.parse( - pageDsl.get(page.getUnpublishedPage().getName()))); - } - } catch (ParseException e) { - log.error( - "Error parsing the page dsl for page: {}", - page.getUnpublishedPage().getName(), - e); - throw new AppsmithException( - AppsmithError.JSON_PROCESSING_ERROR, - page.getUnpublishedPage().getName()); - } - }); - pages.forEach(newPage -> { - // As we are publishing the app and then committing to git we expect the published and unpublished PageDTO - // will be same, so we create a deep copy for the published version for page from the unpublishedPageDTO - newPage.setPublishedPage(gson.fromJson(gson.toJson(newPage.getUnpublishedPage()), PageDTO.class)); - }); - applicationJson.setPageList(pages); - - // Extract actions - if (CollectionUtils.isNullOrEmpty(applicationReference.getActions())) { - applicationJson.setActionList(new ArrayList<>()); - } else { - Map actionBody = applicationReference.getActionBody(); - List actions = getApplicationResource(applicationReference.getActions(), NewAction.class); - // Remove null values if present - org.apache.commons.collections.CollectionUtils.filter(actions, PredicateUtils.notNullPredicate()); - actions.forEach(newAction -> { - // With the file version v4 we have split the actions and metadata separately into two files - // So we need to set the body to the unpublished action - String keyName = newAction.getUnpublishedAction().getName() - + newAction.getUnpublishedAction().getPageId(); - if (actionBody != null - && (actionBody.containsKey(keyName)) - && !StringUtils.isEmpty(actionBody.get(keyName))) { - // For REMOTE plugin like Twilio the user actions are stored in key value pairs and hence they need - // to be - // deserialized separately unlike the body which is stored as string in the db. - if (newAction.getPluginType().toString().equals("REMOTE")) { - Map formData = gson.fromJson(actionBody.get(keyName), Map.class); - newAction - .getUnpublishedAction() - .getActionConfiguration() - .setFormData(formData); - } else { - newAction - .getUnpublishedAction() - .getActionConfiguration() - .setBody(actionBody.get(keyName)); - } - } - // As we are publishing the app and then committing to git we expect the published and unpublished - // actionDTO will be same, so we create a deep copy for the published version for action from - // unpublishedActionDTO - newAction.setPublishedAction( - gson.fromJson(gson.toJson(newAction.getUnpublishedAction()), ActionDTO.class)); - }); - applicationJson.setActionList(actions); - } - - // Extract actionCollection - if (CollectionUtils.isNullOrEmpty(applicationReference.getActionCollections())) { - applicationJson.setActionCollectionList(new ArrayList<>()); - } else { - Map actionCollectionBody = applicationReference.getActionCollectionBody(); - List actionCollections = - getApplicationResource(applicationReference.getActionCollections(), ActionCollection.class); - // Remove null values if present - org.apache.commons.collections.CollectionUtils.filter(actionCollections, PredicateUtils.notNullPredicate()); - actionCollections.forEach(actionCollection -> { - // Set the js object body to the unpublished collection - // Since file version v3 we are splitting the js object code and metadata separately - String keyName = actionCollection.getUnpublishedCollection().getName() - + actionCollection.getUnpublishedCollection().getPageId(); - if (actionCollectionBody != null && actionCollectionBody.containsKey(keyName)) { - actionCollection.getUnpublishedCollection().setBody(actionCollectionBody.get(keyName)); - } - // As we are publishing the app and then committing to git we expect the published and unpublished - // actionCollectionDTO will be same, so we create a deep copy for the published version for - // actionCollection from unpublishedActionCollectionDTO - actionCollection.setPublishedCollection(gson.fromJson( - gson.toJson(actionCollection.getUnpublishedCollection()), ActionCollectionDTO.class)); - }); - applicationJson.setActionCollectionList(actionCollections); - } - - // Extract datasources - applicationJson.setDatasourceList( - getApplicationResource(applicationReference.getDatasources(), DatasourceStorage.class)); - - return applicationJson; - } - - public Mono deleteIndexLockFile(Path path) { - return fileUtils.deleteIndexLockFile(path, INDEX_LOCK_FILE_STALE_TIME); + public GitFileUtils( + FileInterface fileUtils, + AnalyticsService analyticsService, + SessionUserService sessionUserService, + Gson gson) { + super(fileUtils, analyticsService, sessionUserService, gson); + this.gson = gson; } } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/ResponseUtils.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/ResponseUtils.java index f551121ed7..aeea1fcd6a 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/ResponseUtils.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/ResponseUtils.java @@ -1,345 +1,9 @@ package com.appsmith.server.helpers; -import com.appsmith.external.models.ActionDTO; -import com.appsmith.external.models.DefaultResources; -import com.appsmith.server.domains.ActionCollection; -import com.appsmith.server.domains.Application; -import com.appsmith.server.domains.Layout; -import com.appsmith.server.domains.NewAction; -import com.appsmith.server.domains.NewPage; -import com.appsmith.server.dtos.ActionCollectionDTO; -import com.appsmith.server.dtos.ActionCollectionViewDTO; -import com.appsmith.server.dtos.ActionViewDTO; -import com.appsmith.server.dtos.ApplicationPagesDTO; -import com.appsmith.server.dtos.LayoutDTO; -import com.appsmith.server.dtos.PageDTO; -import com.appsmith.server.dtos.PageNameIdDTO; -import com.appsmith.server.exceptions.AppsmithError; -import com.appsmith.server.exceptions.AppsmithException; -import com.appsmith.server.migrations.JsonSchemaVersions; +import com.appsmith.server.helpers.ce.ResponseUtilsCE; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang.StringUtils; import org.springframework.stereotype.Component; -import org.springframework.util.CollectionUtils; - -import java.util.List; @Slf4j @Component -public class ResponseUtils { - - public PageDTO updatePageDTOWithDefaultResources(PageDTO page) { - DefaultResources defaultResourceIds = page.getDefaultResources(); - if (defaultResourceIds == null - || StringUtils.isEmpty(defaultResourceIds.getApplicationId()) - || StringUtils.isEmpty(defaultResourceIds.getPageId())) { - - if (defaultResourceIds == null) { - return page; - } - if (StringUtils.isEmpty(defaultResourceIds.getApplicationId())) { - defaultResourceIds.setApplicationId(page.getApplicationId()); - } - if (StringUtils.isEmpty(defaultResourceIds.getPageId())) { - defaultResourceIds.setPageId(page.getId()); - } - } - page.setApplicationId(defaultResourceIds.getApplicationId()); - page.setId(defaultResourceIds.getPageId()); - - page.getLayouts().stream() - .filter(layout -> !CollectionUtils.isEmpty(layout.getLayoutOnLoadActions())) - .forEach(layout -> this.updateLayoutWithDefaultResources(layout)); - return page; - } - - public NewPage updateNewPageWithDefaultResources(NewPage newPage) { - DefaultResources defaultResourceIds = newPage.getDefaultResources(); - if (defaultResourceIds == null - || StringUtils.isEmpty(defaultResourceIds.getApplicationId()) - || StringUtils.isEmpty(defaultResourceIds.getPageId())) { - log.error( - "Unable to find default ids for page: {}", - newPage.getId(), - new AppsmithException(AppsmithError.DEFAULT_RESOURCES_UNAVAILABLE, "page", newPage.getId())); - - if (defaultResourceIds == null) { - return newPage; - } - if (StringUtils.isEmpty(defaultResourceIds.getApplicationId())) { - defaultResourceIds.setApplicationId(newPage.getApplicationId()); - } - if (StringUtils.isEmpty(defaultResourceIds.getPageId())) { - defaultResourceIds.setPageId(newPage.getId()); - } - } - newPage.setId(defaultResourceIds.getPageId()); - newPage.setApplicationId(defaultResourceIds.getApplicationId()); - if (newPage.getUnpublishedPage() != null) { - newPage.setUnpublishedPage(this.updatePageDTOWithDefaultResources(newPage.getUnpublishedPage())); - } - if (newPage.getPublishedPage() != null) { - newPage.setPublishedPage(this.updatePageDTOWithDefaultResources(newPage.getPublishedPage())); - } - return newPage; - } - - public ApplicationPagesDTO updateApplicationPagesDTOWithDefaultResources(ApplicationPagesDTO applicationPages) { - List pageNameIdList = applicationPages.getPages(); - for (PageNameIdDTO page : pageNameIdList) { - if (StringUtils.isEmpty(page.getDefaultPageId())) { - log.error( - "Unable to find default pageId for applicationPage: {}", - page.getId(), - new AppsmithException( - AppsmithError.DEFAULT_RESOURCES_UNAVAILABLE, "applicationPage", page.getId())); - continue; - } - page.setId(page.getDefaultPageId()); - } - // need to update the application also if it's present - if (applicationPages.getApplication() != null) { - applicationPages.setApplication(updateApplicationWithDefaultResources(applicationPages.getApplication())); - } - return applicationPages; - } - - public ActionDTO updateActionDTOWithDefaultResources(ActionDTO action) { - DefaultResources defaultResourceIds = action.getDefaultResources(); - if (defaultResourceIds == null - || StringUtils.isEmpty(defaultResourceIds.getApplicationId()) - || StringUtils.isEmpty(defaultResourceIds.getPageId()) - || StringUtils.isEmpty(defaultResourceIds.getActionId())) { - - if (defaultResourceIds == null) { - return action; - } - if (StringUtils.isEmpty(defaultResourceIds.getApplicationId())) { - defaultResourceIds.setApplicationId(action.getApplicationId()); - } - if (StringUtils.isEmpty(defaultResourceIds.getPageId())) { - defaultResourceIds.setPageId(action.getPageId()); - } - if (StringUtils.isEmpty(defaultResourceIds.getActionId())) { - defaultResourceIds.setActionId(action.getId()); - } - } - action.setApplicationId(defaultResourceIds.getApplicationId()); - action.setPageId(defaultResourceIds.getPageId()); - action.setId(defaultResourceIds.getActionId()); - if (!StringUtils.isEmpty(defaultResourceIds.getCollectionId())) { - action.setCollectionId(defaultResourceIds.getCollectionId()); - } - return action; - } - - public LayoutDTO updateLayoutDTOWithDefaultResources(LayoutDTO layout) { - if (!CollectionUtils.isEmpty(layout.getActionUpdates())) { - layout.getActionUpdates() - .forEach(updateLayoutAction -> updateLayoutAction.setId(updateLayoutAction.getDefaultActionId())); - } - if (!CollectionUtils.isEmpty(layout.getLayoutOnLoadActions())) { - layout.getLayoutOnLoadActions() - .forEach(layoutOnLoadAction -> layoutOnLoadAction.forEach(onLoadAction -> { - if (!StringUtils.isEmpty(onLoadAction.getDefaultActionId())) { - onLoadAction.setId(onLoadAction.getDefaultActionId()); - } - if (!StringUtils.isEmpty(onLoadAction.getDefaultCollectionId())) { - onLoadAction.setCollectionId(onLoadAction.getDefaultCollectionId()); - } - })); - } - return layout; - } - - public Layout updateLayoutWithDefaultResources(Layout layout) { - if (!CollectionUtils.isEmpty(layout.getLayoutOnLoadActions())) { - layout.getLayoutOnLoadActions() - .forEach(layoutOnLoadAction -> layoutOnLoadAction.forEach(onLoadAction -> { - if (!StringUtils.isEmpty(onLoadAction.getDefaultActionId())) { - onLoadAction.setId(onLoadAction.getDefaultActionId()); - } - if (!StringUtils.isEmpty(onLoadAction.getDefaultCollectionId())) { - onLoadAction.setCollectionId(onLoadAction.getDefaultCollectionId()); - } - })); - } - return layout; - } - - public ActionViewDTO updateActionViewDTOWithDefaultResources(ActionViewDTO viewDTO) { - DefaultResources defaultResourceIds = viewDTO.getDefaultResources(); - if (defaultResourceIds == null - || StringUtils.isEmpty(defaultResourceIds.getPageId()) - || StringUtils.isEmpty(defaultResourceIds.getActionId())) { - - if (defaultResourceIds == null) { - return viewDTO; - } - if (StringUtils.isEmpty(defaultResourceIds.getPageId())) { - defaultResourceIds.setPageId(viewDTO.getPageId()); - } - if (StringUtils.isEmpty(defaultResourceIds.getActionId())) { - defaultResourceIds.setActionId(viewDTO.getId()); - } - } - viewDTO.setId(defaultResourceIds.getActionId()); - viewDTO.setPageId(defaultResourceIds.getPageId()); - return viewDTO; - } - - public NewAction updateNewActionWithDefaultResources(NewAction newAction) { - DefaultResources defaultResourceIds = newAction.getDefaultResources(); - if (defaultResourceIds == null - || StringUtils.isEmpty(defaultResourceIds.getApplicationId()) - || StringUtils.isEmpty(defaultResourceIds.getActionId())) { - log.error( - "Unable to find default ids for newAction: {}", - newAction.getId(), - new AppsmithException(AppsmithError.DEFAULT_RESOURCES_UNAVAILABLE, "newAction", newAction.getId())); - - if (defaultResourceIds == null) { - return newAction; - } - if (StringUtils.isEmpty(defaultResourceIds.getApplicationId())) { - defaultResourceIds.setApplicationId(newAction.getApplicationId()); - } - if (StringUtils.isEmpty(defaultResourceIds.getActionId())) { - defaultResourceIds.setActionId(newAction.getId()); - } - } - - newAction.setId(defaultResourceIds.getActionId()); - newAction.setApplicationId(defaultResourceIds.getApplicationId()); - if (newAction.getUnpublishedAction() != null) { - newAction.setUnpublishedAction(this.updateActionDTOWithDefaultResources(newAction.getUnpublishedAction())); - } - if (newAction.getPublishedAction() != null) { - newAction.setPublishedAction(this.updateActionDTOWithDefaultResources(newAction.getPublishedAction())); - } - return newAction; - } - - public ActionCollection updateActionCollectionWithDefaultResources(ActionCollection actionCollection) { - DefaultResources defaultResourceIds = actionCollection.getDefaultResources(); - if (defaultResourceIds == null - || StringUtils.isEmpty(defaultResourceIds.getApplicationId()) - || StringUtils.isEmpty(defaultResourceIds.getCollectionId())) { - log.error( - "Unable to find default ids for actionCollection: {}", - actionCollection.getId(), - new AppsmithException( - AppsmithError.DEFAULT_RESOURCES_UNAVAILABLE, "actionCollection", actionCollection.getId())); - - if (defaultResourceIds == null) { - return actionCollection; - } - if (StringUtils.isEmpty(defaultResourceIds.getApplicationId())) { - defaultResourceIds.setApplicationId(actionCollection.getApplicationId()); - } - if (StringUtils.isEmpty(defaultResourceIds.getCollectionId())) { - defaultResourceIds.setCollectionId(actionCollection.getId()); - } - } - actionCollection.setId(defaultResourceIds.getCollectionId()); - actionCollection.setApplicationId(defaultResourceIds.getApplicationId()); - if (actionCollection.getUnpublishedCollection() != null) { - actionCollection.setUnpublishedCollection( - this.updateCollectionDTOWithDefaultResources(actionCollection.getUnpublishedCollection())); - } - if (actionCollection.getPublishedCollection() != null) { - actionCollection.setPublishedCollection( - this.updateCollectionDTOWithDefaultResources(actionCollection.getPublishedCollection())); - } - return actionCollection; - } - - public ActionCollectionDTO updateCollectionDTOWithDefaultResources(ActionCollectionDTO collection) { - DefaultResources defaultResourceIds = collection.getDefaultResources(); - if (defaultResourceIds == null - || StringUtils.isEmpty(defaultResourceIds.getApplicationId()) - || StringUtils.isEmpty(defaultResourceIds.getPageId()) - || StringUtils.isEmpty(defaultResourceIds.getCollectionId())) { - - if (defaultResourceIds == null) { - return collection; - } - if (StringUtils.isEmpty(defaultResourceIds.getApplicationId())) { - defaultResourceIds.setApplicationId(collection.getApplicationId()); - } - if (StringUtils.isEmpty(defaultResourceIds.getPageId())) { - defaultResourceIds.setPageId(collection.getPageId()); - } - if (StringUtils.isEmpty(defaultResourceIds.getCollectionId())) { - defaultResourceIds.setCollectionId(collection.getId()); - } - } - collection.setApplicationId(defaultResourceIds.getApplicationId()); - collection.setPageId(defaultResourceIds.getPageId()); - collection.setId(defaultResourceIds.getCollectionId()); - - // Update actions within the collection - collection.getActions().forEach(this::updateActionDTOWithDefaultResources); - collection.getArchivedActions().forEach(this::updateActionDTOWithDefaultResources); - - return collection; - } - - public ActionCollectionViewDTO updateActionCollectionViewDTOWithDefaultResources(ActionCollectionViewDTO viewDTO) { - DefaultResources defaultResourceIds = viewDTO.getDefaultResources(); - if (defaultResourceIds == null - || StringUtils.isEmpty(defaultResourceIds.getPageId()) - || StringUtils.isEmpty(defaultResourceIds.getApplicationId()) - || StringUtils.isEmpty(defaultResourceIds.getCollectionId())) { - - if (defaultResourceIds == null) { - return viewDTO; - } - if (StringUtils.isEmpty(defaultResourceIds.getApplicationId())) { - defaultResourceIds.setApplicationId(viewDTO.getApplicationId()); - } - if (StringUtils.isEmpty(defaultResourceIds.getPageId())) { - defaultResourceIds.setPageId(viewDTO.getPageId()); - } - if (StringUtils.isEmpty(defaultResourceIds.getCollectionId())) { - defaultResourceIds.setCollectionId(viewDTO.getId()); - } - } - viewDTO.setId(defaultResourceIds.getCollectionId()); - viewDTO.setApplicationId(defaultResourceIds.getApplicationId()); - viewDTO.setPageId(defaultResourceIds.getPageId()); - viewDTO.getActions().forEach(this::updateActionDTOWithDefaultResources); - return viewDTO; - } - - public Application updateApplicationWithDefaultResources(Application application) { - if (application.getGitApplicationMetadata() != null - && !StringUtils.isEmpty(application.getGitApplicationMetadata().getDefaultApplicationId())) { - application.setId(application.getGitApplicationMetadata().getDefaultApplicationId()); - } - if (!CollectionUtils.isEmpty(application.getPages())) { - application.getPages().forEach(page -> { - if (!StringUtils.isEmpty(page.getDefaultPageId())) { - page.setId(page.getDefaultPageId()); - } - }); - } - if (!CollectionUtils.isEmpty(application.getPublishedPages())) { - application.getPublishedPages().forEach(page -> { - if (!StringUtils.isEmpty(page.getDefaultPageId())) { - page.setId(page.getDefaultPageId()); - } - }); - } - - if (application.getClientSchemaVersion() == null - || application.getServerSchemaVersion() == null - || (JsonSchemaVersions.clientVersion.equals(application.getClientSchemaVersion()) - && JsonSchemaVersions.serverVersion.equals(application.getServerSchemaVersion()))) { - application.setIsAutoUpdate(false); - } else { - application.setIsAutoUpdate(true); - } - return application; - } -} +public class ResponseUtils extends ResponseUtilsCE {} diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/ce/GitFileUtilsCE.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/ce/GitFileUtilsCE.java new file mode 100644 index 0000000000..252938ec73 --- /dev/null +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/ce/GitFileUtilsCE.java @@ -0,0 +1,737 @@ +package com.appsmith.server.helpers.ce; + +import com.appsmith.external.constants.AnalyticsEvents; +import com.appsmith.external.git.FileInterface; +import com.appsmith.external.helpers.Stopwatch; +import com.appsmith.external.models.ActionDTO; +import com.appsmith.external.models.ApplicationGitReference; +import com.appsmith.external.models.BaseDomain; +import com.appsmith.external.models.DatasourceStorage; +import com.appsmith.external.models.PluginType; +import com.appsmith.git.helpers.FileUtilsImpl; +import com.appsmith.server.constants.FieldName; +import com.appsmith.server.domains.ActionCollection; +import com.appsmith.server.domains.Application; +import com.appsmith.server.domains.ApplicationPage; +import com.appsmith.server.domains.CustomJSLib; +import com.appsmith.server.domains.NewAction; +import com.appsmith.server.domains.NewPage; +import com.appsmith.server.domains.Theme; +import com.appsmith.server.dtos.ActionCollectionDTO; +import com.appsmith.server.dtos.ApplicationJson; +import com.appsmith.server.dtos.PageDTO; +import com.appsmith.server.exceptions.AppsmithError; +import com.appsmith.server.exceptions.AppsmithException; +import com.appsmith.server.helpers.CollectionUtils; +import com.appsmith.server.services.AnalyticsService; +import com.appsmith.server.services.SessionUserService; +import com.google.gson.Gson; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import net.minidev.json.JSONObject; +import net.minidev.json.parser.JSONParser; +import net.minidev.json.parser.ParseException; +import org.apache.commons.collections.PredicateUtils; +import org.apache.commons.lang3.StringUtils; +import org.eclipse.jgit.api.errors.GitAPIException; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Import; +import org.springframework.stereotype.Component; +import reactor.core.Exceptions; +import reactor.core.publisher.Mono; + +import java.io.IOException; +import java.lang.reflect.Field; +import java.lang.reflect.Type; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static com.appsmith.external.constants.GitConstants.NAME_SEPARATOR; +import static com.appsmith.external.helpers.AppsmithBeanUtils.copyNestedNonNullProperties; +import static com.appsmith.external.helpers.AppsmithBeanUtils.copyProperties; +import static com.appsmith.server.constants.FieldName.ACTION_COLLECTION_LIST; +import static com.appsmith.server.constants.FieldName.ACTION_LIST; +import static com.appsmith.server.constants.FieldName.CUSTOM_JS_LIB_LIST; +import static com.appsmith.server.constants.FieldName.DATASOURCE_LIST; +import static com.appsmith.server.constants.FieldName.DECRYPTED_FIELDS; +import static com.appsmith.server.constants.FieldName.EDIT_MODE_THEME; +import static com.appsmith.server.constants.FieldName.EXPORTED_APPLICATION; +import static com.appsmith.server.constants.FieldName.PAGE_LIST; + +@Slf4j +@RequiredArgsConstructor +@Component +@Import({FileUtilsImpl.class}) +public class GitFileUtilsCE { + + private final FileInterface fileUtils; + private final AnalyticsService analyticsService; + private final SessionUserService sessionUserService; + + private final Gson gson; + + // Number of seconds after lock file is stale + @Value("${appsmith.index.lock.file.time}") + public final int INDEX_LOCK_FILE_STALE_TIME = 300; + + // Only include the application helper fields in metadata object + protected Set getBlockedMetadataFields() { + return Set.of( + EXPORTED_APPLICATION, + DATASOURCE_LIST, + PAGE_LIST, + ACTION_LIST, + ACTION_COLLECTION_LIST, + DECRYPTED_FIELDS, + EDIT_MODE_THEME, + CUSTOM_JS_LIB_LIST); + } + + /** + * This method will save the complete application in the local repo directory. + * Path to repo will be : ./container-volumes/git-repo/workspaceId/defaultApplicationId/repoName/{application_data} + * + * @param baseRepoSuffix path suffix used to create a local repo path + * @param applicationJson application reference object from which entire application can be rehydrated + * @param branchName name of the branch for the current application + * @return repo path where the application is stored + */ + public Mono saveApplicationToLocalRepo( + Path baseRepoSuffix, ApplicationJson applicationJson, String branchName) + throws IOException, GitAPIException { + /* + 1. Checkout to branch + 2. Create application reference for appsmith-git module + 3. Save application to git repo + */ + + ApplicationGitReference applicationReference = createApplicationReference(applicationJson); + // Save application to git repo + try { + return fileUtils.saveApplicationToGitRepo(baseRepoSuffix, applicationReference, branchName); + } catch (IOException | GitAPIException e) { + log.error("Error occurred while saving files to local git repo: ", e); + throw Exceptions.propagate(e); + } + } + + public Mono saveApplicationToLocalRepoWithAnalytics( + Path baseRepoSuffix, ApplicationJson applicationJson, String branchName) + throws IOException, GitAPIException { + + /* + 1. Checkout to branch + 2. Create application reference for appsmith-git module + 3. Save application to git repo + */ + Stopwatch stopwatch = new Stopwatch(AnalyticsEvents.GIT_SERIALIZE_APP_RESOURCES_TO_LOCAL_FILE.getEventName()); + // Save application to git repo + try { + Mono repoPathMono = saveApplicationToLocalRepo(baseRepoSuffix, applicationJson, branchName); + return Mono.zip(repoPathMono, sessionUserService.getCurrentUser()).flatMap(tuple -> { + stopwatch.stopTimer(); + Path repoPath = tuple.getT1(); + // Path to repo will be : ./container-volumes/git-repo/workspaceId/defaultApplicationId/repoName/ + final Map data = Map.of( + FieldName.APPLICATION_ID, + repoPath.getParent().getFileName().toString(), + FieldName.ORGANIZATION_ID, + repoPath.getParent().getParent().getFileName().toString(), + FieldName.FLOW_NAME, + stopwatch.getFlow(), + "executionTime", + stopwatch.getExecutionTime()); + return analyticsService + .sendEvent( + AnalyticsEvents.UNIT_EXECUTION_TIME.getEventName(), + tuple.getT2().getUsername(), + data) + .thenReturn(repoPath); + }); + } catch (IOException | GitAPIException e) { + log.error("Error occurred while saving files to local git repo: ", e); + throw Exceptions.propagate(e); + } + } + + public Mono saveApplicationToLocalRepo( + String workspaceId, + String defaultApplicationId, + String repoName, + ApplicationJson applicationJson, + String branchName) + throws GitAPIException, IOException { + Path baseRepoSuffix = Paths.get(workspaceId, defaultApplicationId, repoName); + + return saveApplicationToLocalRepo(baseRepoSuffix, applicationJson, branchName); + } + + /** + * Method to convert application resources to the structure which can be serialised by appsmith-git module for + * serialisation + * + * @param applicationJson application resource including actions, jsobjects, pages + * @return resource which can be saved to file system + */ + public ApplicationGitReference createApplicationReference(ApplicationJson applicationJson) { + ApplicationGitReference applicationReference = new ApplicationGitReference(); + applicationReference.setUpdatedResources(applicationJson.getUpdatedResources()); + + setApplicationInApplicationReference(applicationJson, applicationReference); + + setThemesInApplicationReference(applicationJson, applicationReference); + + setApplicationMetadataInApplicationReference(applicationJson, applicationReference); + + setNewPagesInApplicationReference(applicationJson, applicationReference); + + setNewActionsInApplicationReference(applicationJson, applicationReference); + + setActionCollectionsInApplicationReference(applicationJson, applicationReference); + + setDatasourcesInApplicationReference(applicationJson, applicationReference); + + setCustomJSLibsInApplicationReference(applicationJson, applicationReference); + + return applicationReference; + } + + private void setApplicationInApplicationReference( + ApplicationJson applicationJson, ApplicationGitReference applicationReference) { + Application application = applicationJson.getExportedApplication(); + removeUnwantedFieldsFromApplication(application); + // Pass application reference + applicationReference.setApplication(applicationJson.getExportedApplication()); + } + + private void setApplicationMetadataInApplicationReference( + ApplicationJson applicationJson, ApplicationGitReference applicationReference) { + // Pass metadata + Iterable keys = getAllFields(applicationJson) + .map(Field::getName) + .filter(name -> !getBlockedMetadataFields().contains(name)) + .collect(Collectors.toList()); + + ApplicationJson applicationMetadata = new ApplicationJson(); + applicationJson.setUpdatedResources(null); + copyProperties(applicationJson, applicationMetadata, keys); + applicationReference.setMetadata(applicationMetadata); + } + + private void setThemesInApplicationReference( + ApplicationJson applicationJson, ApplicationGitReference applicationReference) { + // No need to commit publish mode theme as it leads to conflict resolution at both the places if any + applicationJson.setPublishedTheme(null); + + // Remove internal fields from the themes + removeUnwantedFieldsFromBaseDomain(applicationJson.getEditModeTheme()); + + applicationReference.setTheme(applicationJson.getEditModeTheme()); + } + + private void setCustomJSLibsInApplicationReference( + ApplicationJson applicationJson, ApplicationGitReference applicationReference) { + Map resourceMap = new HashMap<>(); + applicationJson.getCustomJSLibList().forEach(jsLib -> { + removeUnwantedFieldsFromBaseDomain(jsLib); + resourceMap.put(jsLib.getUidString(), jsLib); + }); + applicationReference.setJsLibraries(resourceMap); + } + + private void setDatasourcesInApplicationReference( + ApplicationJson applicationJson, ApplicationGitReference applicationReference) { + Map resourceMap = new HashMap<>(); + // Send datasources + applicationJson.getDatasourceList().forEach(datasource -> { + removeUnwantedFieldsFromDatasource(datasource); + resourceMap.put(datasource.getName(), datasource); + }); + applicationReference.setDatasources(resourceMap); + } + + private void setActionCollectionsInApplicationReference( + ApplicationJson applicationJson, ApplicationGitReference applicationReference) { + Map resourceMap = new HashMap<>(); + Map resourceMapActionCollectionBody = new HashMap<>(); + // Insert JSOObjects and also assign the keys which later will be used for saving the resource in actual + // filepath + // JSObjectName_pageName => nomenclature for the keys + applicationJson.getActionCollectionList().stream() + // As we are expecting the commit will happen only after the application is published, so we can safely + // assume if the unpublished version is deleted entity should not be committed to git + .filter(collection -> collection.getUnpublishedCollection() != null + && collection.getUnpublishedCollection().getDeletedAt() == null) + .forEach(actionCollection -> { + String prefix = actionCollection.getUnpublishedCollection().getUserExecutableName() + + NAME_SEPARATOR + + actionCollection.getUnpublishedCollection().getPageId(); + removeUnwantedFieldFromActionCollection(actionCollection); + + String body = actionCollection.getUnpublishedCollection().getBody() != null + ? actionCollection.getUnpublishedCollection().getBody() + : ""; + actionCollection.getUnpublishedCollection().setBody(null); + resourceMapActionCollectionBody.put(prefix, body); + resourceMap.put(prefix, actionCollection); + }); + applicationReference.setActionCollections(resourceMap); + applicationReference.setActionCollectionBody(resourceMapActionCollectionBody); + } + + private void setNewPagesInApplicationReference( + ApplicationJson applicationJson, ApplicationGitReference applicationReference) { + // Insert only active pages which will then be committed to repo as individual file + Map resourceMap = new HashMap<>(); + Map dslBody = new HashMap<>(); + applicationJson.getPageList().stream() + // As we are expecting the commit will happen only after the application is published, so we can safely + // assume if the unpublished version is deleted entity should not be committed to git + .filter(newPage -> newPage.getUnpublishedPage() != null + && newPage.getUnpublishedPage().getDeletedAt() == null) + .forEach(newPage -> { + String pageName = newPage.getUnpublishedPage() != null + ? newPage.getUnpublishedPage().getName() + : newPage.getPublishedPage().getName(); + removeUnwantedFieldsFromPage(newPage); + JSONObject dsl = + newPage.getUnpublishedPage().getLayouts().get(0).getDsl(); + // Get MainContainer widget data, remove the children and club with Canvas.json file + JSONObject mainContainer = new JSONObject(dsl); + mainContainer.remove("children"); + newPage.getUnpublishedPage().getLayouts().get(0).setDsl(mainContainer); + // pageName will be used for naming the json file + dslBody.put(pageName, dsl.toString()); + resourceMap.put(pageName, newPage); + }); + + applicationReference.setPages(resourceMap); + applicationReference.setPageDsl(dslBody); + } + + private void setNewActionsInApplicationReference( + ApplicationJson applicationJson, ApplicationGitReference applicationReference) { + Map resourceMap = new HashMap<>(); + Map resourceMapBody = new HashMap<>(); + // Insert active actions and also assign the keys which later will be used for saving the resource in actual + // filepath + // For actions, we are referring to validNames to maintain unique file names as just name + // field don't guarantee unique constraint for actions within JSObject + // queryValidName_pageName => nomenclature for the keys + applicationJson.getActionList().stream() + // As we are expecting the commit will happen only after the application is published, so we can safely + // assume if the unpublished version is deleted entity should not be committed to git + .filter(newAction -> newAction.getUnpublishedAction() != null + && newAction.getUnpublishedAction().getDeletedAt() == null) + .forEach(newAction -> { + String prefix; + if (newAction.getUnpublishedAction() != null) { + prefix = newAction.getUnpublishedAction().getUserExecutableName() + + NAME_SEPARATOR + + newAction.getUnpublishedAction().getPageId(); + } else { + prefix = newAction.getPublishedAction().getUserExecutableName() + + NAME_SEPARATOR + + newAction.getPublishedAction().getPageId(); + } + removeUnwantedFieldFromAction(newAction); + String body = newAction.getUnpublishedAction().getActionConfiguration() != null + && newAction + .getUnpublishedAction() + .getActionConfiguration() + .getBody() + != null + ? newAction + .getUnpublishedAction() + .getActionConfiguration() + .getBody() + : ""; + + // This is a special case where we are handling REMOTE type plugins based actions such as Twilio + // The user configured values are stored in a attribute called formData which is a map unlike the + // body + if (PluginType.REMOTE.equals(newAction.getPluginType()) + && newAction.getUnpublishedAction().getActionConfiguration() != null + && newAction + .getUnpublishedAction() + .getActionConfiguration() + .getFormData() + != null) { + body = new Gson() + .toJson( + newAction + .getUnpublishedAction() + .getActionConfiguration() + .getFormData(), + Map.class); + newAction + .getUnpublishedAction() + .getActionConfiguration() + .setFormData(null); + } + // This is a special case where we are handling JS actions as we don't want to commit the body of JS + // actions + if (PluginType.JS.equals(newAction.getPluginType())) { + if (newAction.getUnpublishedAction().getActionConfiguration() != null) { + newAction + .getUnpublishedAction() + .getActionConfiguration() + .setBody(null); + newAction.getUnpublishedAction().setJsonPathKeys(null); + } + } else { + // For the regular actions we save the body field to git repo + resourceMapBody.put(prefix, body); + } + resourceMap.put(prefix, newAction); + }); + applicationReference.setActions(resourceMap); + applicationReference.setActionBody(resourceMapBody); + } + + protected Stream getAllFields(ApplicationJson applicationJson) { + Class currentType = applicationJson.getClass(); + + Set> classes = new HashSet<>(); + + while (currentType != null) { + classes.add(currentType); + currentType = currentType.getSuperclass(); + } + + return classes.stream().flatMap(currentClass -> Arrays.stream(currentClass.getDeclaredFields())); + } + + /** + * Method to reconstruct the application from the local git repo + * + * @param workspaceId To which workspace application needs to be rehydrated + * @param defaultApplicationId Root application for the current branched application + * @param branchName for which branch the application needs to rehydrate + * @return application reference from which entire application can be rehydrated + */ + public Mono reconstructApplicationJsonFromGitRepoWithAnalytics( + String workspaceId, String defaultApplicationId, String repoName, String branchName) { + Stopwatch stopwatch = new Stopwatch(AnalyticsEvents.GIT_DESERIALIZE_APP_RESOURCES_FROM_FILE.getEventName()); + + return Mono.zip( + reconstructApplicationJsonFromGitRepo(workspaceId, defaultApplicationId, repoName, branchName), + sessionUserService.getCurrentUser()) + .flatMap(tuple -> { + stopwatch.stopTimer(); + final Map data = Map.of( + FieldName.APPLICATION_ID, + defaultApplicationId, + FieldName.ORGANIZATION_ID, + workspaceId, + FieldName.FLOW_NAME, + stopwatch.getFlow(), + "executionTime", + stopwatch.getExecutionTime()); + return analyticsService + .sendEvent( + AnalyticsEvents.UNIT_EXECUTION_TIME.getEventName(), + tuple.getT2().getUsername(), + data) + .thenReturn(tuple.getT1()); + }); + } + + public Mono reconstructApplicationJsonFromGitRepo( + String workspaceId, String defaultApplicationId, String repoName, String branchName) { + + Mono appReferenceMono = fileUtils.reconstructApplicationReferenceFromGitRepo( + workspaceId, defaultApplicationId, repoName, branchName); + return appReferenceMono.map(applicationReference -> { + // Extract application metadata from the json + ApplicationJson metadata = + getApplicationResource(applicationReference.getMetadata(), ApplicationJson.class); + ApplicationJson applicationJson = getApplicationJsonFromGitReference(applicationReference); + copyNestedNonNullProperties(metadata, applicationJson); + return applicationJson; + }); + } + + protected List getApplicationResource(Map resources, Type type) { + + List deserializedResources = new ArrayList<>(); + if (!CollectionUtils.isNullOrEmpty(resources)) { + for (Map.Entry resource : resources.entrySet()) { + deserializedResources.add(getApplicationResource(resource.getValue(), type)); + } + } + return deserializedResources; + } + + public T getApplicationResource(Object resource, Type type) { + if (resource == null) { + return null; + } + return gson.fromJson(gson.toJson(resource), type); + } + + /** + * Once the user connects the existing application to a remote repo, we will initialize the repo with Readme.md - + * Url to the deployed app(view and edit mode) + * Link to discord channel for support + * Link to appsmith documentation for Git related operations + * Welcome message + * + * @param baseRepoSuffix path suffix used to create a branch repo path as per worktree implementation + * @param viewModeUrl URL to deployed version of the application view only mode + * @param editModeUrl URL to deployed version of the application edit mode + * @return Path where the Application is stored + */ + public Mono initializeReadme(Path baseRepoSuffix, String viewModeUrl, String editModeUrl) throws IOException { + return fileUtils + .initializeReadme(baseRepoSuffix, viewModeUrl, editModeUrl) + .onErrorResume(e -> Mono.error(new AppsmithException(AppsmithError.GIT_FILE_SYSTEM_ERROR, e))); + } + + /** + * When the user clicks on detach remote, we need to remove the repo from the file system + * + * @param baseRepoSuffix path suffix used to create a branch repo path as per worktree implementation + * @return success on remove of file system + */ + public Mono deleteLocalRepo(Path baseRepoSuffix) { + return fileUtils.deleteLocalRepo(baseRepoSuffix); + } + + public Mono checkIfDirectoryIsEmpty(Path baseRepoSuffix) throws IOException { + return fileUtils + .checkIfDirectoryIsEmpty(baseRepoSuffix) + .onErrorResume(e -> Mono.error(new AppsmithException(AppsmithError.GIT_FILE_SYSTEM_ERROR, e))); + } + + protected void removeUnwantedFieldsFromBaseDomain(BaseDomain baseDomain) { + baseDomain.setPolicies(null); + baseDomain.setUserPermissions(null); + } + + private void removeUnwantedFieldsFromDatasource(DatasourceStorage datasource) { + datasource.setInvalids(null); + removeUnwantedFieldsFromBaseDomain(datasource); + } + + private void removeUnwantedFieldsFromPage(NewPage page) { + // As we are publishing the app and then committing to git we expect the published and unpublished PageDTO will + // be same, so we only commit unpublished PageDTO. + page.setPublishedPage(null); + removeUnwantedFieldsFromBaseDomain(page); + } + + private void removeUnwantedFieldsFromApplication(Application application) { + // Don't commit application name as while importing we are using the repoName as application name + application.setName(null); + application.setPublishedPages(null); + application.setIsPublic(null); + application.setSlug(null); + application.setPublishedApplicationDetail(null); + removeUnwantedFieldsFromBaseDomain(application); + // we can call the sanitiseToExportDBObject() from BaseDomain as well here + } + + private void removeUnwantedFieldFromAction(NewAction action) { + // As we are publishing the app and then committing to git we expect the published and unpublished ActionDTO + // will be same, so we only commit unpublished ActionDTO. + action.setPublishedAction(null); + action.getUnpublishedAction().sanitiseToExportDBObject(); + removeUnwantedFieldsFromBaseDomain(action); + } + + private void removeUnwantedFieldFromActionCollection(ActionCollection actionCollection) { + // As we are publishing the app and then committing to git we expect the published and unpublished + // ActionCollectionDTO will be same, so we only commit unpublished ActionCollectionDTO. + actionCollection.setPublishedCollection(null); + actionCollection.getUnpublishedCollection().sanitiseForExport(); + removeUnwantedFieldsFromBaseDomain(actionCollection); + } + + protected ApplicationJson getApplicationJsonFromGitReference(ApplicationGitReference applicationReference) { + ApplicationJson applicationJson = new ApplicationJson(); + + setApplicationInApplicationJson(applicationReference, applicationJson); + + setThemesInApplicationJson(applicationReference, applicationJson); + + setCustomJsLibsInApplicationJson(applicationReference, applicationJson); + + setNewPagesInApplicationJson(applicationReference, applicationJson); + + setNewActionsInApplicationJson(applicationReference, applicationJson); + + setActionCollectionsInApplicationJson(applicationReference, applicationJson); + + setDatasourcesInApplicationJson(applicationReference, applicationJson); + + return applicationJson; + } + + private void setApplicationInApplicationJson( + ApplicationGitReference applicationReference, ApplicationJson applicationJson) { + // Extract application data from the json + Application application = getApplicationResource(applicationReference.getApplication(), Application.class); + applicationJson.setExportedApplication(application); + + if (application != null && !CollectionUtils.isNullOrEmpty(application.getPages())) { + // Remove null values + org.apache.commons.collections.CollectionUtils.filter( + application.getPages(), PredicateUtils.notNullPredicate()); + // Create a deep clone of application pages to update independently + application.setViewMode(false); + final List applicationPages = + new ArrayList<>(application.getPages().size()); + application + .getPages() + .forEach(applicationPage -> + applicationPages.add(gson.fromJson(gson.toJson(applicationPage), ApplicationPage.class))); + application.setPublishedPages(applicationPages); + } + } + + private void setThemesInApplicationJson( + ApplicationGitReference applicationReference, ApplicationJson applicationJson) { + applicationJson.setEditModeTheme(getApplicationResource(applicationReference.getTheme(), Theme.class)); + // Clone the edit mode theme to published theme as both should be same for git connected application because we + // do deploy and push as a single operation + applicationJson.setPublishedTheme(applicationJson.getEditModeTheme()); + } + + private void setDatasourcesInApplicationJson( + ApplicationGitReference applicationReference, ApplicationJson applicationJson) { + // Extract datasources + applicationJson.setDatasourceList( + getApplicationResource(applicationReference.getDatasources(), DatasourceStorage.class)); + } + + private void setActionCollectionsInApplicationJson( + ApplicationGitReference applicationReference, ApplicationJson applicationJson) { + // Extract actionCollection + if (CollectionUtils.isNullOrEmpty(applicationReference.getActionCollections())) { + applicationJson.setActionCollectionList(new ArrayList<>()); + } else { + Map actionCollectionBody = applicationReference.getActionCollectionBody(); + List actionCollections = + getApplicationResource(applicationReference.getActionCollections(), ActionCollection.class); + // Remove null values if present + org.apache.commons.collections.CollectionUtils.filter(actionCollections, PredicateUtils.notNullPredicate()); + actionCollections.forEach(actionCollection -> { + // Set the js object body to the unpublished collection + // Since file version v3 we are splitting the js object code and metadata separately + String keyName = actionCollection.getUnpublishedCollection().getName() + + actionCollection.getUnpublishedCollection().getPageId(); + if (actionCollectionBody != null && actionCollectionBody.containsKey(keyName)) { + actionCollection.getUnpublishedCollection().setBody(actionCollectionBody.get(keyName)); + } + // As we are publishing the app and then committing to git we expect the published and unpublished + // actionCollectionDTO will be same, so we create a deep copy for the published version for + // actionCollection from unpublishedActionCollectionDTO + actionCollection.setPublishedCollection(gson.fromJson( + gson.toJson(actionCollection.getUnpublishedCollection()), ActionCollectionDTO.class)); + }); + applicationJson.setActionCollectionList(actionCollections); + } + } + + private void setNewActionsInApplicationJson( + ApplicationGitReference applicationReference, ApplicationJson applicationJson) { + // Extract actions + if (CollectionUtils.isNullOrEmpty(applicationReference.getActions())) { + applicationJson.setActionList(new ArrayList<>()); + } else { + Map actionBody = applicationReference.getActionBody(); + List actions = getApplicationResource(applicationReference.getActions(), NewAction.class); + // Remove null values if present + org.apache.commons.collections.CollectionUtils.filter(actions, PredicateUtils.notNullPredicate()); + actions.forEach(newAction -> { + // With the file version v4 we have split the actions and metadata separately into two files + // So we need to set the body to the unpublished action + String keyName = newAction.getUnpublishedAction().getName() + + newAction.getUnpublishedAction().getPageId(); + if (actionBody != null + && (actionBody.containsKey(keyName)) + && !StringUtils.isEmpty(actionBody.get(keyName))) { + // For REMOTE plugin like Twilio the user actions are stored in key value pairs and hence they need + // to be + // deserialized separately unlike the body which is stored as string in the db. + if (newAction.getPluginType().toString().equals("REMOTE")) { + Map formData = gson.fromJson(actionBody.get(keyName), Map.class); + newAction + .getUnpublishedAction() + .getActionConfiguration() + .setFormData(formData); + } else { + newAction + .getUnpublishedAction() + .getActionConfiguration() + .setBody(actionBody.get(keyName)); + } + } + // As we are publishing the app and then committing to git we expect the published and unpublished + // actionDTO will be same, so we create a deep copy for the published version for action from + // unpublishedActionDTO + newAction.setPublishedAction( + gson.fromJson(gson.toJson(newAction.getUnpublishedAction()), ActionDTO.class)); + }); + applicationJson.setActionList(actions); + } + } + + private void setCustomJsLibsInApplicationJson( + ApplicationGitReference applicationReference, ApplicationJson applicationJson) { + List customJSLibList = + getApplicationResource(applicationReference.getJsLibraries(), CustomJSLib.class); + + // remove the duplicate js libraries if there is any + List customJSLibListWithoutDuplicates = new ArrayList<>(new HashSet<>(customJSLibList)); + + applicationJson.setCustomJSLibList(customJSLibListWithoutDuplicates); + } + + private void setNewPagesInApplicationJson( + ApplicationGitReference applicationReference, ApplicationJson applicationJson) { + // Extract pages + List pages = getApplicationResource(applicationReference.getPages(), NewPage.class); + // Remove null values + org.apache.commons.collections.CollectionUtils.filter(pages, PredicateUtils.notNullPredicate()); + // Set the DSL to page object before saving + Map pageDsl = applicationReference.getPageDsl(); + pages.forEach(page -> { + JSONParser jsonParser = new JSONParser(); + try { + if (pageDsl != null && pageDsl.get(page.getUnpublishedPage().getName()) != null) { + page.getUnpublishedPage().getLayouts().get(0).setDsl((JSONObject) jsonParser.parse( + pageDsl.get(page.getUnpublishedPage().getName()))); + } + } catch (ParseException e) { + log.error( + "Error parsing the page dsl for page: {}", + page.getUnpublishedPage().getName(), + e); + throw new AppsmithException( + AppsmithError.JSON_PROCESSING_ERROR, + page.getUnpublishedPage().getName()); + } + }); + pages.forEach(newPage -> { + // As we are publishing the app and then committing to git we expect the published and unpublished PageDTO + // will be same, so we create a deep copy for the published version for page from the unpublishedPageDTO + newPage.setPublishedPage(gson.fromJson(gson.toJson(newPage.getUnpublishedPage()), PageDTO.class)); + }); + applicationJson.setPageList(pages); + } + + public Mono deleteIndexLockFile(Path path) { + return fileUtils.deleteIndexLockFile(path, INDEX_LOCK_FILE_STALE_TIME); + } +} diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/ce/ResponseUtilsCE.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/ce/ResponseUtilsCE.java new file mode 100644 index 0000000000..b4cefdf815 --- /dev/null +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/ce/ResponseUtilsCE.java @@ -0,0 +1,321 @@ +package com.appsmith.server.helpers.ce; + +import com.appsmith.external.models.ActionDTO; +import com.appsmith.external.models.DefaultResources; +import com.appsmith.server.domains.ActionCollection; +import com.appsmith.server.domains.Application; +import com.appsmith.server.domains.Layout; +import com.appsmith.server.domains.NewAction; +import com.appsmith.server.domains.NewPage; +import com.appsmith.server.dtos.ActionCollectionDTO; +import com.appsmith.server.dtos.ActionCollectionViewDTO; +import com.appsmith.server.dtos.ActionViewDTO; +import com.appsmith.server.dtos.ApplicationPagesDTO; +import com.appsmith.server.dtos.LayoutDTO; +import com.appsmith.server.dtos.PageDTO; +import com.appsmith.server.dtos.PageNameIdDTO; +import com.appsmith.server.exceptions.AppsmithError; +import com.appsmith.server.exceptions.AppsmithException; +import com.appsmith.server.migrations.JsonSchemaVersions; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang.StringUtils; +import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; + +import java.util.List; + +@Slf4j +@Component +public class ResponseUtilsCE { + + public PageDTO updatePageDTOWithDefaultResources(PageDTO page) { + DefaultResources defaultResourceIds = page.getDefaultResources(); + if (defaultResourceIds == null + || StringUtils.isEmpty(defaultResourceIds.getApplicationId()) + || StringUtils.isEmpty(defaultResourceIds.getPageId())) { + + if (defaultResourceIds == null) { + return page; + } + if (StringUtils.isEmpty(defaultResourceIds.getApplicationId())) { + defaultResourceIds.setApplicationId(page.getApplicationId()); + } + if (StringUtils.isEmpty(defaultResourceIds.getPageId())) { + defaultResourceIds.setPageId(page.getId()); + } + } + page.setApplicationId(defaultResourceIds.getApplicationId()); + page.setId(defaultResourceIds.getPageId()); + + page.getLayouts().stream() + .filter(layout -> !CollectionUtils.isEmpty(layout.getLayoutOnLoadActions())) + .forEach(layout -> this.updateLayoutWithDefaultResources(layout)); + return page; + } + + public NewPage updateNewPageWithDefaultResources(NewPage newPage) { + DefaultResources defaultResourceIds = newPage.getDefaultResources(); + if (defaultResourceIds == null + || StringUtils.isEmpty(defaultResourceIds.getApplicationId()) + || StringUtils.isEmpty(defaultResourceIds.getPageId())) { + log.error( + "Unable to find default ids for page: {}", + newPage.getId(), + new AppsmithException(AppsmithError.DEFAULT_RESOURCES_UNAVAILABLE, "page", newPage.getId())); + + if (defaultResourceIds == null) { + return newPage; + } + if (StringUtils.isEmpty(defaultResourceIds.getApplicationId())) { + defaultResourceIds.setApplicationId(newPage.getApplicationId()); + } + if (StringUtils.isEmpty(defaultResourceIds.getPageId())) { + defaultResourceIds.setPageId(newPage.getId()); + } + } + newPage.setId(defaultResourceIds.getPageId()); + newPage.setApplicationId(defaultResourceIds.getApplicationId()); + if (newPage.getUnpublishedPage() != null) { + newPage.setUnpublishedPage(this.updatePageDTOWithDefaultResources(newPage.getUnpublishedPage())); + } + if (newPage.getPublishedPage() != null) { + newPage.setPublishedPage(this.updatePageDTOWithDefaultResources(newPage.getPublishedPage())); + } + return newPage; + } + + public ApplicationPagesDTO updateApplicationPagesDTOWithDefaultResources(ApplicationPagesDTO applicationPages) { + List pageNameIdList = applicationPages.getPages(); + for (PageNameIdDTO page : pageNameIdList) { + if (StringUtils.isEmpty(page.getDefaultPageId())) { + log.error( + "Unable to find default pageId for applicationPage: {}", + page.getId(), + new AppsmithException( + AppsmithError.DEFAULT_RESOURCES_UNAVAILABLE, "applicationPage", page.getId())); + continue; + } + page.setId(page.getDefaultPageId()); + } + // need to update the application also if it's present + if (applicationPages.getApplication() != null) { + applicationPages.setApplication(updateApplicationWithDefaultResources(applicationPages.getApplication())); + } + return applicationPages; + } + + public ActionDTO updateActionDTOWithDefaultResources(ActionDTO action) { + DefaultResources defaultResourceIds = action.getDefaultResources(); + if (defaultResourceIds == null) { + return action; + } + if (StringUtils.isEmpty(defaultResourceIds.getApplicationId())) { + defaultResourceIds.setApplicationId(action.getApplicationId()); + } + if (StringUtils.isEmpty(defaultResourceIds.getPageId())) { + defaultResourceIds.setPageId(action.getPageId()); + } + if (StringUtils.isEmpty(defaultResourceIds.getActionId())) { + defaultResourceIds.setActionId(action.getId()); + } + action.setApplicationId(defaultResourceIds.getApplicationId()); + action.setPageId(defaultResourceIds.getPageId()); + action.setId(defaultResourceIds.getActionId()); + if (!StringUtils.isEmpty(defaultResourceIds.getCollectionId())) { + action.setCollectionId(defaultResourceIds.getCollectionId()); + } + return action; + } + + public LayoutDTO updateLayoutDTOWithDefaultResources(LayoutDTO layout) { + if (!CollectionUtils.isEmpty(layout.getActionUpdates())) { + layout.getActionUpdates() + .forEach(updateLayoutAction -> updateLayoutAction.setId(updateLayoutAction.getDefaultActionId())); + } + if (!CollectionUtils.isEmpty(layout.getLayoutOnLoadActions())) { + layout.getLayoutOnLoadActions() + .forEach(layoutOnLoadAction -> layoutOnLoadAction.forEach(onLoadAction -> { + if (!StringUtils.isEmpty(onLoadAction.getDefaultActionId())) { + onLoadAction.setId(onLoadAction.getDefaultActionId()); + } + if (!StringUtils.isEmpty(onLoadAction.getDefaultCollectionId())) { + onLoadAction.setCollectionId(onLoadAction.getDefaultCollectionId()); + } + })); + } + return layout; + } + + public Layout updateLayoutWithDefaultResources(Layout layout) { + if (!CollectionUtils.isEmpty(layout.getLayoutOnLoadActions())) { + layout.getLayoutOnLoadActions() + .forEach(layoutOnLoadAction -> layoutOnLoadAction.forEach(onLoadAction -> { + if (!StringUtils.isEmpty(onLoadAction.getDefaultActionId())) { + onLoadAction.setId(onLoadAction.getDefaultActionId()); + } + if (!StringUtils.isEmpty(onLoadAction.getDefaultCollectionId())) { + onLoadAction.setCollectionId(onLoadAction.getDefaultCollectionId()); + } + })); + } + return layout; + } + + public ActionViewDTO updateActionViewDTOWithDefaultResources(ActionViewDTO viewDTO) { + DefaultResources defaultResourceIds = viewDTO.getDefaultResources(); + + if (defaultResourceIds == null) { + return viewDTO; + } + if (StringUtils.isEmpty(defaultResourceIds.getPageId())) { + defaultResourceIds.setPageId(viewDTO.getPageId()); + } + if (StringUtils.isEmpty(defaultResourceIds.getActionId())) { + defaultResourceIds.setActionId(viewDTO.getId()); + } + + viewDTO.setId(defaultResourceIds.getActionId()); + viewDTO.setPageId(defaultResourceIds.getPageId()); + return viewDTO; + } + + public NewAction updateNewActionWithDefaultResources(NewAction newAction) { + DefaultResources defaultResourceIds = newAction.getDefaultResources(); + if (defaultResourceIds == null + || StringUtils.isEmpty(defaultResourceIds.getApplicationId()) + || StringUtils.isEmpty(defaultResourceIds.getActionId())) { + log.error( + "Unable to find default ids for newAction: {}", + newAction.getId(), + new AppsmithException(AppsmithError.DEFAULT_RESOURCES_UNAVAILABLE, "newAction", newAction.getId())); + + if (defaultResourceIds == null) { + return newAction; + } + if (StringUtils.isEmpty(defaultResourceIds.getApplicationId())) { + defaultResourceIds.setApplicationId(newAction.getApplicationId()); + } + if (StringUtils.isEmpty(defaultResourceIds.getActionId())) { + defaultResourceIds.setActionId(newAction.getId()); + } + } + + newAction.setId(defaultResourceIds.getActionId()); + newAction.setApplicationId(defaultResourceIds.getApplicationId()); + if (newAction.getUnpublishedAction() != null) { + newAction.setUnpublishedAction(this.updateActionDTOWithDefaultResources(newAction.getUnpublishedAction())); + } + if (newAction.getPublishedAction() != null) { + newAction.setPublishedAction(this.updateActionDTOWithDefaultResources(newAction.getPublishedAction())); + } + return newAction; + } + + public ActionCollection updateActionCollectionWithDefaultResources(ActionCollection actionCollection) { + DefaultResources defaultResourceIds = actionCollection.getDefaultResources(); + + if (defaultResourceIds == null) { + return actionCollection; + } + if (StringUtils.isEmpty(defaultResourceIds.getApplicationId())) { + defaultResourceIds.setApplicationId(actionCollection.getApplicationId()); + } + if (StringUtils.isEmpty(defaultResourceIds.getCollectionId())) { + defaultResourceIds.setCollectionId(actionCollection.getId()); + } + + actionCollection.setId(defaultResourceIds.getCollectionId()); + actionCollection.setApplicationId(defaultResourceIds.getApplicationId()); + if (actionCollection.getUnpublishedCollection() != null) { + actionCollection.setUnpublishedCollection( + this.updateCollectionDTOWithDefaultResources(actionCollection.getUnpublishedCollection())); + } + if (actionCollection.getPublishedCollection() != null) { + actionCollection.setPublishedCollection( + this.updateCollectionDTOWithDefaultResources(actionCollection.getPublishedCollection())); + } + return actionCollection; + } + + public ActionCollectionDTO updateCollectionDTOWithDefaultResources(ActionCollectionDTO collection) { + DefaultResources defaultResourceIds = collection.getDefaultResources(); + + if (defaultResourceIds == null) { + return collection; + } + if (StringUtils.isEmpty(defaultResourceIds.getApplicationId())) { + defaultResourceIds.setApplicationId(collection.getApplicationId()); + } + if (StringUtils.isEmpty(defaultResourceIds.getPageId())) { + defaultResourceIds.setPageId(collection.getPageId()); + } + if (StringUtils.isEmpty(defaultResourceIds.getCollectionId())) { + defaultResourceIds.setCollectionId(collection.getId()); + } + + collection.setApplicationId(defaultResourceIds.getApplicationId()); + collection.setPageId(defaultResourceIds.getPageId()); + collection.setId(defaultResourceIds.getCollectionId()); + + // Update actions within the collection + collection.getActions().forEach(this::updateActionDTOWithDefaultResources); + collection.getArchivedActions().forEach(this::updateActionDTOWithDefaultResources); + + return collection; + } + + public ActionCollectionViewDTO updateActionCollectionViewDTOWithDefaultResources(ActionCollectionViewDTO viewDTO) { + DefaultResources defaultResourceIds = viewDTO.getDefaultResources(); + + if (defaultResourceIds == null) { + return viewDTO; + } + if (StringUtils.isEmpty(defaultResourceIds.getApplicationId())) { + defaultResourceIds.setApplicationId(viewDTO.getApplicationId()); + } + if (StringUtils.isEmpty(defaultResourceIds.getPageId())) { + defaultResourceIds.setPageId(viewDTO.getPageId()); + } + if (StringUtils.isEmpty(defaultResourceIds.getCollectionId())) { + defaultResourceIds.setCollectionId(viewDTO.getId()); + } + + viewDTO.setId(defaultResourceIds.getCollectionId()); + viewDTO.setApplicationId(defaultResourceIds.getApplicationId()); + viewDTO.setPageId(defaultResourceIds.getPageId()); + + viewDTO.getActions().forEach(this::updateActionDTOWithDefaultResources); + return viewDTO; + } + + public Application updateApplicationWithDefaultResources(Application application) { + if (application.getGitApplicationMetadata() != null + && !StringUtils.isEmpty(application.getGitApplicationMetadata().getDefaultApplicationId())) { + application.setId(application.getGitApplicationMetadata().getDefaultApplicationId()); + } + if (!CollectionUtils.isEmpty(application.getPages())) { + application.getPages().forEach(page -> { + if (!StringUtils.isEmpty(page.getDefaultPageId())) { + page.setId(page.getDefaultPageId()); + } + }); + } + if (!CollectionUtils.isEmpty(application.getPublishedPages())) { + application.getPublishedPages().forEach(page -> { + if (!StringUtils.isEmpty(page.getDefaultPageId())) { + page.setId(page.getDefaultPageId()); + } + }); + } + + if (application.getClientSchemaVersion() == null + || application.getServerSchemaVersion() == null + || (JsonSchemaVersions.clientVersion.equals(application.getClientSchemaVersion()) + && JsonSchemaVersions.serverVersion.equals(application.getServerSchemaVersion()))) { + application.setIsAutoUpdate(false); + } else { + application.setIsAutoUpdate(true); + } + return application; + } +} diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/newactions/base/NewActionServiceCE.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/newactions/base/NewActionServiceCE.java index a6e7d5bc7d..68753d8509 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/newactions/base/NewActionServiceCE.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/newactions/base/NewActionServiceCE.java @@ -133,8 +133,6 @@ public interface NewActionServiceCE extends CrudService { Map getAnalyticsProperties(NewAction savedAction); - void populateDefaultResources(NewAction newAction, NewAction branchedAction, String branchName); - Mono updateActionsWithImportedCollectionIds( ImportActionCollectionResultDTO importActionCollectionResultDTO, ImportActionResultDTO importActionResultDTO); diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/newactions/base/NewActionServiceCEImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/newactions/base/NewActionServiceCEImpl.java index 4f11271f6a..49bc7da4ec 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/newactions/base/NewActionServiceCEImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/newactions/base/NewActionServiceCEImpl.java @@ -23,6 +23,7 @@ import com.appsmith.server.acl.PolicyGenerator; import com.appsmith.server.applications.base.ApplicationService; import com.appsmith.server.constants.FieldName; import com.appsmith.server.datasources.base.DatasourceService; +import com.appsmith.server.defaultresources.DefaultResourcesService; import com.appsmith.server.domains.Action; import com.appsmith.server.domains.ActionCollection; import com.appsmith.server.domains.Application; @@ -136,6 +137,8 @@ public class NewActionServiceCEImpl extends BaseService defaultPluginMap = new HashMap<>(); private final AtomicReference jsTypePluginReference = new AtomicReference<>(); + private final DefaultResourcesService defaultResourcesService; + private final DefaultResourcesService dtoDefaultResourcesService; public NewActionServiceCEImpl( Scheduler scheduler, @@ -161,7 +164,9 @@ public class NewActionServiceCEImpl extends BaseService defaultResourcesService, + DefaultResourcesService dtoDefaultResourcesService) { super(scheduler, validator, mongoConverter, reactiveMongoTemplate, repository, analyticsService); this.repository = repository; @@ -183,6 +188,8 @@ public class NewActionServiceCEImpl extends BaseService invalids = new HashSet<>(); @@ -639,18 +641,15 @@ public class NewActionServiceCEImpl extends BaseService * This method performs an update of an unpublished action in the database without triggering an analytics event. * - * @param id The unique identifier of the unpublished action to be updated. - * @param action The updated action object. + * @param id The unique identifier of the unpublished action to be updated. + * @param action The updated action object. * @param permission An optional permission parameter for access control. * @return A Mono emitting a Tuple containing the updated ActionDTO and NewAction after modification. - * * @throws AppsmithException if the provided ID is invalid or if the action is not found. - * - * @implNote - * This method is used by {#updateUnpublishedAction(String, ActionDTO)}, but it does not send an analytics event. If analytics event tracking is not required for the update, this method can be used to improve performance and reduce overhead. + * @implNote This method is used by {#updateUnpublishedAction(String, ActionDTO)}, but it does not send an analytics event. If analytics event tracking is not required for the update, this method can be used to improve performance and reduce overhead. */ @Override public Mono> updateUnpublishedActionWithoutAnalytics( @@ -1646,33 +1645,6 @@ public class NewActionServiceCEImpl extends BaseService updateActionsWithImportedCollectionIds( ImportActionCollectionResultDTO importActionCollectionResultDTO, @@ -1722,7 +1694,7 @@ public class NewActionServiceCEImpl extends BaseService findByPageIdsForExport( List unpublishedPages, Optional optionalPermission) { - return repository.findByPageIds(unpublishedPages, optionalPermission); + return repository.findByPageIds(unpublishedPages, optionalPermission).doOnNext(newAction -> { + this.setCommonFieldsFromNewActionIntoAction(newAction, newAction.getUnpublishedAction()); + this.setCommonFieldsFromNewActionIntoAction(newAction, newAction.getPublishedAction()); + }); } @Override @@ -1857,8 +1832,6 @@ public class NewActionServiceCEImpl extends BaseService defaultResourcesService, + DefaultResourcesService dtoDefaultResourcesService) { super( scheduler, validator, @@ -81,6 +85,8 @@ public class NewActionServiceImpl extends NewActionServiceCEImpl implements NewA pagePermission, actionPermission, entityValidationService, - observationRegistry); + observationRegistry, + defaultResourcesService, + dtoDefaultResourcesService); } } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/newactions/defaultresources/ActionDTODefaultResourcesServiceCEImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/newactions/defaultresources/ActionDTODefaultResourcesServiceCEImpl.java new file mode 100644 index 0000000000..2086373efb --- /dev/null +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/newactions/defaultresources/ActionDTODefaultResourcesServiceCEImpl.java @@ -0,0 +1,41 @@ +package com.appsmith.server.newactions.defaultresources; + +import com.appsmith.external.models.ActionDTO; +import com.appsmith.external.models.DefaultResources; +import com.appsmith.server.defaultresources.DefaultResourcesServiceCE; +import org.springframework.stereotype.Service; +import org.springframework.util.StringUtils; + +@Service +public class ActionDTODefaultResourcesServiceCEImpl implements DefaultResourcesServiceCE { + + @Override + public ActionDTO initialize(ActionDTO domainObject, String branchName, boolean resetExistingValues) { + DefaultResources existingDefaultResources = domainObject.getDefaultResources(); + DefaultResources defaultResources = new DefaultResources(); + + String defaultPageId = domainObject.getPageId(); + + if (existingDefaultResources != null && !resetExistingValues) { + // Check if there are properties to be copied over from existing + if (StringUtils.hasText(existingDefaultResources.getPageId())) { + defaultPageId = existingDefaultResources.getPageId(); + } + } + + defaultResources.setPageId(defaultPageId); + + domainObject.setDefaultResources(defaultResources); + return domainObject; + } + + @Override + public ActionDTO setFromOtherBranch(ActionDTO domainObject, ActionDTO defaultDomainObject, String branchName) { + DefaultResources defaultResources = new DefaultResources(); + + defaultResources.setPageId(defaultDomainObject.getDefaultResources().getPageId()); + + domainObject.setDefaultResources(defaultResources); + return domainObject; + } +} diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/newactions/defaultresources/ActionDTODefaultResourcesServiceImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/newactions/defaultresources/ActionDTODefaultResourcesServiceImpl.java new file mode 100644 index 0000000000..d0680bf3ec --- /dev/null +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/newactions/defaultresources/ActionDTODefaultResourcesServiceImpl.java @@ -0,0 +1,9 @@ +package com.appsmith.server.newactions.defaultresources; + +import com.appsmith.external.models.ActionDTO; +import com.appsmith.server.defaultresources.DefaultResourcesService; +import org.springframework.stereotype.Service; + +@Service +public class ActionDTODefaultResourcesServiceImpl extends ActionDTODefaultResourcesServiceCEImpl + implements DefaultResourcesService {} diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/newactions/defaultresources/NewActionDefaultResourcesServiceCEImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/newactions/defaultresources/NewActionDefaultResourcesServiceCEImpl.java new file mode 100644 index 0000000000..e81dc33acf --- /dev/null +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/newactions/defaultresources/NewActionDefaultResourcesServiceCEImpl.java @@ -0,0 +1,52 @@ +package com.appsmith.server.newactions.defaultresources; + +import com.appsmith.external.models.DefaultResources; +import com.appsmith.server.defaultresources.DefaultResourcesServiceCE; +import com.appsmith.server.domains.NewAction; +import org.springframework.stereotype.Service; +import org.springframework.util.StringUtils; + +@Service +public class NewActionDefaultResourcesServiceCEImpl implements DefaultResourcesServiceCE { + + @Override + public NewAction initialize(NewAction domainObject, String branchName, boolean resetExistingValues) { + DefaultResources existingDefaultResources = domainObject.getDefaultResources(); + DefaultResources defaultResources = new DefaultResources(); + + String defaultApplicationId = domainObject.getApplicationId(); + String defaultActionId = domainObject.getId(); + + if (existingDefaultResources != null && !resetExistingValues) { + // Check if there are properties to be copied over from existing + if (StringUtils.hasText(existingDefaultResources.getApplicationId())) { + defaultApplicationId = existingDefaultResources.getApplicationId(); + } + + if (StringUtils.hasText(existingDefaultResources.getActionId())) { + defaultActionId = existingDefaultResources.getActionId(); + } + } + + defaultResources.setActionId(defaultActionId); + defaultResources.setApplicationId(defaultApplicationId); + defaultResources.setBranchName(branchName); + + domainObject.setDefaultResources(defaultResources); + return domainObject; + } + + @Override + public NewAction setFromOtherBranch(NewAction domainObject, NewAction defaultDomainObject, String branchName) { + DefaultResources defaultResources = new DefaultResources(); + + DefaultResources otherDefaultResources = defaultDomainObject.getDefaultResources(); + + defaultResources.setActionId(otherDefaultResources.getActionId()); + defaultResources.setApplicationId(otherDefaultResources.getApplicationId()); + defaultResources.setBranchName(branchName); + + domainObject.setDefaultResources(defaultResources); + return domainObject; + } +} diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/newactions/defaultresources/NewActionDefaultResourcesServiceImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/newactions/defaultresources/NewActionDefaultResourcesServiceImpl.java new file mode 100644 index 0000000000..055db5cca3 --- /dev/null +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/newactions/defaultresources/NewActionDefaultResourcesServiceImpl.java @@ -0,0 +1,9 @@ +package com.appsmith.server.newactions.defaultresources; + +import com.appsmith.server.defaultresources.DefaultResourcesService; +import com.appsmith.server.domains.NewAction; +import org.springframework.stereotype.Service; + +@Service +public class NewActionDefaultResourcesServiceImpl extends NewActionDefaultResourcesServiceCEImpl + implements DefaultResourcesService {} diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/newactions/imports/NewActionImportableServiceCEImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/newactions/imports/NewActionImportableServiceCEImpl.java index 1516235fe4..aec0187de2 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/newactions/imports/NewActionImportableServiceCEImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/newactions/imports/NewActionImportableServiceCEImpl.java @@ -5,6 +5,7 @@ import com.appsmith.external.models.DefaultResources; import com.appsmith.external.models.Policy; import com.appsmith.server.actioncollections.base.ActionCollectionService; import com.appsmith.server.constants.FieldName; +import com.appsmith.server.defaultresources.DefaultResourcesService; import com.appsmith.server.domains.ActionCollection; import com.appsmith.server.domains.Application; import com.appsmith.server.domains.NewAction; @@ -49,6 +50,8 @@ public class NewActionImportableServiceCEImpl implements ImportableServiceCE defaultResourcesService; + private final DefaultResourcesService dtoDefaultResourcesService; // Requires pageNameMap, pageNameToOldNameMap, pluginMap and datasourceNameToIdMap, to be present in importable // resources. @@ -229,8 +232,7 @@ public class NewActionImportableServiceCEImpl implements ImportableServiceCE newAction.getGitSyncId() != null) .collectMap(NewAction::getGitSyncId); } else { @@ -307,6 +309,38 @@ public class NewActionImportableServiceCEImpl implements ImportableServiceCE actionsInOtherBranches, + NewAction newAction) { + // this will generate the id and other auto generated fields e.g. createdAt + newAction.updateForBulkWriteOperation(); + + populateDomainMappedReferences(mappedImportableResourcesDTO, newAction); + + // set gitSyncId, if it doesn't exist + if (newAction.getGitSyncId() == null) { + newAction.setGitSyncId( + newAction.getApplicationId() + "_" + Instant.now().toString()); + } + } + protected NewAction getExistingActionForImportedAction( MappedImportableResourcesDTO mappedImportableResourcesDTO, Map actionsInCurrentApp, @@ -440,6 +459,10 @@ public class NewActionImportableServiceCEImpl implements ImportableServiceCE newAction.getGitSyncId() != null); } + protected Flux getActionInOtherBranchesMono(String defaultApplicationId) { + return repository.findByDefaultApplicationId(defaultApplicationId, Optional.empty()); + } + private NewPage updatePageInAction( ActionDTO action, Map pageNameMap, Map actionIdMap) { NewPage parentPage = pageNameMap.get(action.getPageId()); @@ -488,6 +511,13 @@ public class NewActionImportableServiceCEImpl implements ImportableServiceCE actionIds = importActionResultDTO .getUnpublishedCollectionIdToActionIdsMap() .get(unpublishedAction.getCollectionId()); - actionIds.put(newAction.getDefaultResources().getActionId(), newAction.getId()); + + actionIds.put(defaultResourcesActionId, newAction.getId()); } } if (newAction.getPublishedAction() != null) { @@ -525,7 +556,7 @@ public class NewActionImportableServiceCEImpl implements ImportableServiceCE actionIds = importActionResultDTO .getPublishedCollectionIdToActionIdsMap() .get(publishedAction.getCollectionId()); - actionIds.put(newAction.getDefaultResources().getActionId(), newAction.getId()); + actionIds.put(defaultResourcesActionId, newAction.getId()); } } } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/newactions/imports/NewActionImportableServiceImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/newactions/imports/NewActionImportableServiceImpl.java index a7e8f9c394..31663ab8a7 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/newactions/imports/NewActionImportableServiceImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/newactions/imports/NewActionImportableServiceImpl.java @@ -1,6 +1,8 @@ package com.appsmith.server.newactions.imports; +import com.appsmith.external.models.ActionDTO; import com.appsmith.server.actioncollections.base.ActionCollectionService; +import com.appsmith.server.defaultresources.DefaultResourcesService; import com.appsmith.server.domains.NewAction; import com.appsmith.server.imports.importable.ImportableService; import com.appsmith.server.newactions.base.NewActionService; @@ -13,7 +15,14 @@ public class NewActionImportableServiceImpl extends NewActionImportableServiceCE public NewActionImportableServiceImpl( NewActionService newActionService, NewActionRepository repository, - ActionCollectionService actionCollectionService) { - super(newActionService, repository, actionCollectionService); + ActionCollectionService actionCollectionService, + DefaultResourcesService defaultResourcesService, + DefaultResourcesService dtoDefaultResourcesService) { + super( + newActionService, + repository, + actionCollectionService, + defaultResourcesService, + dtoDefaultResourcesService); } } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/newpages/defaultresources/NewPageDefaultResourcesServiceCEImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/newpages/defaultresources/NewPageDefaultResourcesServiceCEImpl.java new file mode 100644 index 0000000000..fda825c906 --- /dev/null +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/newpages/defaultresources/NewPageDefaultResourcesServiceCEImpl.java @@ -0,0 +1,50 @@ +package com.appsmith.server.newpages.defaultresources; + +import com.appsmith.external.models.DefaultResources; +import com.appsmith.server.defaultresources.DefaultResourcesServiceCE; +import com.appsmith.server.domains.NewPage; +import org.springframework.stereotype.Service; +import org.springframework.util.StringUtils; + +@Service +public class NewPageDefaultResourcesServiceCEImpl implements DefaultResourcesServiceCE { + + @Override + public NewPage initialize(NewPage domainObject, String branchName, boolean resetExistingValues) { + DefaultResources existingDefaultResources = domainObject.getDefaultResources(); + DefaultResources defaultResources = new DefaultResources(); + + String defaultApplicationId = domainObject.getApplicationId(); + String defaultPageId = domainObject.getId(); + + if (existingDefaultResources != null && !resetExistingValues) { + // Check if there are properties to be copied over from existing + if (StringUtils.hasText(existingDefaultResources.getApplicationId())) { + defaultApplicationId = existingDefaultResources.getApplicationId(); + } + + if (StringUtils.hasText(existingDefaultResources.getPageId())) { + defaultPageId = existingDefaultResources.getPageId(); + } + } + + defaultResources.setPageId(defaultPageId); + defaultResources.setApplicationId(defaultApplicationId); + defaultResources.setBranchName(branchName); + + domainObject.setDefaultResources(defaultResources); + return domainObject; + } + + @Override + public NewPage setFromOtherBranch(NewPage domainObject, NewPage defaultDomainObject, String branchName) { + DefaultResources defaultResources = new DefaultResources(); + + defaultResources.setPageId(defaultDomainObject.getId()); + defaultResources.setApplicationId(defaultDomainObject.getApplicationId()); + defaultResources.setBranchName(branchName); + + domainObject.setDefaultResources(defaultResources); + return domainObject; + } +} diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/newpages/defaultresources/NewPageDefaultResourcesServiceImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/newpages/defaultresources/NewPageDefaultResourcesServiceImpl.java new file mode 100644 index 0000000000..8d3709d3e5 --- /dev/null +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/newpages/defaultresources/NewPageDefaultResourcesServiceImpl.java @@ -0,0 +1,9 @@ +package com.appsmith.server.newpages.defaultresources; + +import com.appsmith.server.defaultresources.DefaultResourcesService; +import com.appsmith.server.domains.NewPage; +import org.springframework.stereotype.Service; + +@Service +public class NewPageDefaultResourcesServiceImpl extends NewPageDefaultResourcesServiceCEImpl + implements DefaultResourcesService {} diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/AppsmithRepository.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/AppsmithRepository.java index 1523a69054..79a281d75a 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/AppsmithRepository.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/AppsmithRepository.java @@ -1,6 +1,7 @@ package com.appsmith.server.repositories; import com.appsmith.server.acl.AclPermission; +import com.mongodb.bulk.BulkWriteResult; import com.mongodb.client.result.InsertManyResult; import org.springframework.data.domain.Sort; import org.springframework.data.mongodb.core.query.Criteria; @@ -48,4 +49,6 @@ public interface AppsmithRepository { * @return List of actions that were passed in the method */ Mono> bulkInsert(List domainList); + + Mono> bulkUpdate(List domainList); } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/ce/BaseAppsmithRepositoryCEImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/ce/BaseAppsmithRepositoryCEImpl.java index 1a67db85c1..874442c770 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/ce/BaseAppsmithRepositoryCEImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/ce/BaseAppsmithRepositoryCEImpl.java @@ -11,11 +11,15 @@ import com.appsmith.server.exceptions.AppsmithException; import com.appsmith.server.repositories.CacheableRepositoryHelper; import com.mongodb.BasicDBObject; import com.mongodb.DBObject; +import com.mongodb.bulk.BulkWriteResult; +import com.mongodb.client.model.UpdateOneModel; +import com.mongodb.client.model.WriteModel; import com.mongodb.client.result.InsertManyResult; import com.mongodb.client.result.UpdateResult; import com.querydsl.core.types.Path; import jakarta.validation.constraints.NotNull; import org.bson.Document; +import org.bson.types.ObjectId; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.GenericTypeResolver; import org.springframework.data.domain.Sort; @@ -743,4 +747,28 @@ public abstract class BaseAppsmithRepositoryCEImpl { .flatMapMany(documentMongoCollection -> documentMongoCollection.insertMany(dbObjects)) .collectList(); } + + public Mono> bulkUpdate(List domainObjects) { + if (CollectionUtils.isEmpty(domainObjects)) { + return Mono.just(Collections.emptyList()); + } + + // convert the list of new actions to a list of DBObjects + List> dbObjects = domainObjects.stream() + .map(actionCollection -> { + assert actionCollection.getId() != null; + Document document = new Document(); + mongoOperations.getConverter().write(actionCollection, document); + document.remove("_id"); + return (WriteModel) new UpdateOneModel( + new Document("_id", new ObjectId(actionCollection.getId())), + new Document("$set", document)); + }) + .collect(Collectors.toList()); + + return mongoOperations + .getCollection(mongoOperations.getCollectionName(genericDomain)) + .flatMapMany(documentMongoCollection -> documentMongoCollection.bulkWrite(dbObjects)) + .collectList(); + } } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/ce/CustomActionCollectionRepositoryCE.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/ce/CustomActionCollectionRepositoryCE.java index 2950953973..3ef8878a0d 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/ce/CustomActionCollectionRepositoryCE.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/ce/CustomActionCollectionRepositoryCE.java @@ -4,7 +4,6 @@ import com.appsmith.external.models.CreatorContextType; import com.appsmith.server.acl.AclPermission; import com.appsmith.server.domains.ActionCollection; import com.appsmith.server.repositories.AppsmithRepository; -import com.mongodb.bulk.BulkWriteResult; import org.springframework.data.domain.Sort; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @@ -49,8 +48,6 @@ public interface CustomActionCollectionRepositoryCE extends AppsmithRepository findByPageIds(List pageIds, Optional permission); - Mono> bulkUpdate(List actionCollections); - Flux findAllByApplicationIds(List applicationIds, List includeFields); Flux findAllUnpublishedActionCollectionsByContextIdAndContextType( diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/ce/CustomActionCollectionRepositoryCEImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/ce/CustomActionCollectionRepositoryCEImpl.java index 9a781cd003..24e0e88486 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/ce/CustomActionCollectionRepositoryCEImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/ce/CustomActionCollectionRepositoryCEImpl.java @@ -8,25 +8,17 @@ import com.appsmith.server.domains.ActionCollection; import com.appsmith.server.domains.QActionCollection; import com.appsmith.server.repositories.BaseAppsmithRepositoryImpl; import com.appsmith.server.repositories.CacheableRepositoryHelper; -import com.mongodb.bulk.BulkWriteResult; -import com.mongodb.client.model.UpdateOneModel; -import com.mongodb.client.model.WriteModel; import org.apache.commons.lang3.StringUtils; -import org.bson.Document; -import org.bson.types.ObjectId; import org.springframework.data.domain.Sort; import org.springframework.data.mongodb.core.ReactiveMongoOperations; import org.springframework.data.mongodb.core.convert.MongoConverter; import org.springframework.data.mongodb.core.query.Criteria; -import org.springframework.util.CollectionUtils; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Optional; -import java.util.stream.Collectors; import static org.springframework.data.mongodb.core.query.Criteria.where; @@ -236,31 +228,6 @@ public class CustomActionCollectionRepositoryCEImpl extends BaseAppsmithReposito return queryAll(List.of(pageIdCriteria), permission); } - @Override - public Mono> bulkUpdate(List actionCollections) { - if (CollectionUtils.isEmpty(actionCollections)) { - return Mono.just(Collections.emptyList()); - } - - // convert the list of new actions to a list of DBObjects - List> dbObjects = actionCollections.stream() - .map(actionCollection -> { - assert actionCollection.getId() != null; - Document document = new Document(); - mongoOperations.getConverter().write(actionCollection, document); - document.remove("_id"); - return (WriteModel) new UpdateOneModel( - new Document("_id", new ObjectId(actionCollection.getId())), - new Document("$set", document)); - }) - .collect(Collectors.toList()); - - return mongoOperations - .getCollection(mongoOperations.getCollectionName(ActionCollection.class)) - .flatMapMany(documentMongoCollection -> documentMongoCollection.bulkWrite(dbObjects)) - .collectList(); - } - @Override public Flux findAllByApplicationIds(List applicationIds, List includeFields) { Criteria applicationCriteria = Criteria.where(FieldName.APPLICATION_ID).in(applicationIds); diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/ce/CustomNewActionRepositoryCE.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/ce/CustomNewActionRepositoryCE.java index 223521d461..9f5b406c73 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/ce/CustomNewActionRepositoryCE.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/ce/CustomNewActionRepositoryCE.java @@ -73,8 +73,6 @@ public interface CustomNewActionRepositoryCE extends AppsmithRepository findAllNonJsActionsByNameAndPageIdsAndViewMode( String name, List pageIds, Boolean viewMode, AclPermission aclPermission, Sort sort); - Mono> bulkUpdate(List newActions); - Mono> publishActions(String applicationId, AclPermission permission); Mono archiveDeletedUnpublishedActions(String applicationId, AclPermission permission); diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/ce/CustomNewActionRepositoryCEImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/ce/CustomNewActionRepositoryCEImpl.java index 8dd8c75d85..76fb36e557 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/ce/CustomNewActionRepositoryCEImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/ce/CustomNewActionRepositoryCEImpl.java @@ -12,11 +12,8 @@ import com.appsmith.server.dtos.PluginTypeAndCountDTO; import com.appsmith.server.repositories.BaseAppsmithRepositoryImpl; import com.appsmith.server.repositories.CacheableRepositoryHelper; import com.mongodb.bulk.BulkWriteResult; -import com.mongodb.client.model.UpdateOneModel; -import com.mongodb.client.model.WriteModel; import com.mongodb.client.result.UpdateResult; import lombok.extern.slf4j.Slf4j; -import org.bson.Document; import org.bson.types.ObjectId; import org.springframework.data.domain.Sort; import org.springframework.data.mongodb.core.MongoTemplate; @@ -31,18 +28,15 @@ import org.springframework.data.mongodb.core.convert.MongoConverter; import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Query; import org.springframework.data.mongodb.core.query.Update; -import org.springframework.util.CollectionUtils; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import reactor.core.scheduler.Schedulers; import java.time.Instant; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.Set; -import java.util.stream.Collectors; import static org.springframework.data.mongodb.core.aggregation.Aggregation.group; import static org.springframework.data.mongodb.core.aggregation.Aggregation.match; @@ -537,30 +531,6 @@ public class CustomNewActionRepositoryCEImpl extends BaseAppsmithRepositoryImpl< return criteriaList; } - @Override - public Mono> bulkUpdate(List newActions) { - if (CollectionUtils.isEmpty(newActions)) { - return Mono.just(Collections.emptyList()); - } - - // convert the list of new actions to a list of DBObjects - List> dbObjects = newActions.stream() - .map(newAction -> { - assert newAction.getId() != null; - Document document = new Document(); - mongoOperations.getConverter().write(newAction, document); - document.remove("_id"); - return (WriteModel) new UpdateOneModel( - new Document("_id", new ObjectId(newAction.getId())), new Document("$set", document)); - }) - .collect(Collectors.toList()); - - return mongoOperations - .getCollection(mongoOperations.getCollectionName(NewAction.class)) - .flatMapMany(documentMongoCollection -> documentMongoCollection.bulkWrite(dbObjects)) - .collectList(); - } - @Override public Flux findByDefaultApplicationId(String defaultApplicationId, Optional permission) { final String defaultResources = fieldName(QBranchAwareDomain.branchAwareDomain.defaultResources); diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/ce/CustomNewPageRepositoryCE.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/ce/CustomNewPageRepositoryCE.java index e01cfbb121..2a8a0cf626 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/ce/CustomNewPageRepositoryCE.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/ce/CustomNewPageRepositoryCE.java @@ -45,7 +45,5 @@ public interface CustomNewPageRepositoryCE extends AppsmithRepository { Mono> publishPages(Collection pageIds, AclPermission permission); - Mono> bulkUpdate(List newPages); - Flux findAllByApplicationIdsWithoutPermission(List applicationIds, List includeFields); } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/ce/CustomNewPageRepositoryCEImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/ce/CustomNewPageRepositoryCEImpl.java index adb7236948..f0cb49c011 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/ce/CustomNewPageRepositoryCEImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/ce/CustomNewPageRepositoryCEImpl.java @@ -10,11 +10,7 @@ import com.appsmith.server.dtos.PageDTO; import com.appsmith.server.repositories.BaseAppsmithRepositoryImpl; import com.appsmith.server.repositories.CacheableRepositoryHelper; import com.mongodb.bulk.BulkWriteResult; -import com.mongodb.client.model.UpdateOneModel; -import com.mongodb.client.model.WriteModel; import lombok.extern.slf4j.Slf4j; -import org.bson.Document; -import org.bson.types.ObjectId; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.ReactiveMongoOperations; import org.springframework.data.mongodb.core.aggregation.Aggregation; @@ -23,18 +19,15 @@ import org.springframework.data.mongodb.core.aggregation.Fields; import org.springframework.data.mongodb.core.convert.MongoConverter; import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Query; -import org.springframework.util.CollectionUtils; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import reactor.core.scheduler.Schedulers; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.Set; -import java.util.stream.Collectors; import static org.springframework.data.mongodb.core.query.Criteria.where; @@ -298,30 +291,6 @@ public class CustomNewPageRepositoryCEImpl extends BaseAppsmithRepositoryImpl> bulkUpdate(List newPages) { - if (CollectionUtils.isEmpty(newPages)) { - return Mono.just(Collections.emptyList()); - } - - // convert the list of new pages to a list of DBObjects - List> dbObjects = newPages.stream() - .map(newPage -> { - assert newPage.getId() != null; - Document document = new Document(); - mongoOperations.getConverter().write(newPage, document); - document.remove("_id"); - return (WriteModel) new UpdateOneModel( - new Document("_id", new ObjectId(newPage.getId())), new Document("$set", document)); - }) - .collect(Collectors.toList()); - - return mongoOperations - .getCollection(mongoOperations.getCollectionName(NewPage.class)) - .flatMapMany(documentMongoCollection -> documentMongoCollection.bulkWrite(dbObjects)) - .collectList(); - } - @Override public Flux findAllByApplicationIdsWithoutPermission( List applicationIds, List includeFields) { diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/LayoutCollectionServiceCEImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/LayoutCollectionServiceCEImpl.java index 14250f7965..7b41ca62f2 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/LayoutCollectionServiceCEImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/LayoutCollectionServiceCEImpl.java @@ -160,7 +160,6 @@ public class LayoutCollectionServiceCEImpl implements LayoutCollectionServiceCE defaultResources.setBranchName(branchName); collectionDTO.setDefaultResources(defaultResources); actionCollection.setDefaultResources(defaultResources); - actionCollection.setUnpublishedCollection(collectionDTO); actionCollectionService.generateAndSetPolicies(newPage, actionCollection); actionCollection.setUnpublishedCollection(collectionDTO); return Mono.zip( diff --git a/app/server/appsmith-server/src/test/java/com/appsmith/server/services/ActionCollectionServiceImplTest.java b/app/server/appsmith-server/src/test/java/com/appsmith/server/services/ActionCollectionServiceImplTest.java index 733cefe1a7..895b2db5d0 100644 --- a/app/server/appsmith-server/src/test/java/com/appsmith/server/services/ActionCollectionServiceImplTest.java +++ b/app/server/appsmith-server/src/test/java/com/appsmith/server/services/ActionCollectionServiceImplTest.java @@ -10,6 +10,7 @@ import com.appsmith.server.actioncollections.base.ActionCollectionService; import com.appsmith.server.actioncollections.base.ActionCollectionServiceImpl; import com.appsmith.server.applications.base.ApplicationService; import com.appsmith.server.constants.FieldName; +import com.appsmith.server.defaultresources.DefaultResourcesService; import com.appsmith.server.domains.ActionCollection; import com.appsmith.server.domains.Layout; import com.appsmith.server.domains.NewAction; @@ -124,6 +125,9 @@ public class ActionCollectionServiceImplTest { @MockBean private PolicyGenerator policyGenerator; + @MockBean + private DefaultResourcesService actionCollectionDefaultResourcesService; + @BeforeEach public void setUp() { applicationPermission = new ApplicationPermissionImpl(); @@ -141,7 +145,8 @@ public class ActionCollectionServiceImplTest { applicationService, responseUtils, applicationPermission, - actionPermission); + actionPermission, + actionCollectionDefaultResourcesService); layoutCollectionService = new LayoutCollectionServiceImpl( newPageService, diff --git a/app/server/appsmith-server/src/test/java/com/appsmith/server/services/ce/NewActionServiceUnitTest.java b/app/server/appsmith-server/src/test/java/com/appsmith/server/services/ce/NewActionServiceUnitTest.java index 67730a243f..68d4f5396d 100644 --- a/app/server/appsmith-server/src/test/java/com/appsmith/server/services/ce/NewActionServiceUnitTest.java +++ b/app/server/appsmith-server/src/test/java/com/appsmith/server/services/ce/NewActionServiceUnitTest.java @@ -6,6 +6,7 @@ import com.appsmith.external.models.PluginType; import com.appsmith.server.acl.PolicyGenerator; import com.appsmith.server.applications.base.ApplicationService; import com.appsmith.server.datasources.base.DatasourceService; +import com.appsmith.server.defaultresources.DefaultResourcesService; import com.appsmith.server.domains.NewAction; import com.appsmith.server.domains.Plugin; import com.appsmith.server.helpers.PluginExecutorHelper; @@ -123,6 +124,12 @@ public class NewActionServiceUnitTest { @MockBean ObservationRegistry observationRegistry; + @MockBean + DefaultResourcesService defaultResourcesService; + + @MockBean + DefaultResourcesService dtoDefaultResourcesService; + @BeforeEach public void setup() { newActionService = new NewActionServiceCEImpl( @@ -149,7 +156,9 @@ public class NewActionServiceUnitTest { pagePermission, actionPermission, entityValidationService, - observationRegistry); + observationRegistry, + defaultResourcesService, + dtoDefaultResourcesService); ObservationRegistry.ObservationConfig mockObservationConfig = Mockito.mock(ObservationRegistry.ObservationConfig.class);