Protei/PromucFlow_constructor
2
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

chore: Git resource map conversions (#37920)

Browse Source 
## Description
> [!TIP]  
> _Add a TL;DR when the description is longer than 500 words or
extremely technical (helps the content, marketing, and DevRel team)._
>
> _Please also include relevant motivation and context. List any
dependencies that are required for this change. Add links to Notion,
Figma or any other documents that might be relevant to the PR._


Fixes #`Issue Number`  
_or_  
Fixes `Issue URL`
> [!WARNING]  
> _If no issue exists, please create an issue first, and check with the
maintainers if the issue is valid._

## Automation

/ok-to-test tags="@tag.Git"

### 🔍 Cypress test results
<!-- This is an auto-generated comment: Cypress test results  -->
> [!IMPORTANT]
> 🟣 🟣 🟣 Your tests are running.
> Tests running at:
<https://github.com/appsmithorg/appsmith/actions/runs/12145100933>
> Commit: c78857b0266e6cfd7be488205544de956e88647e
> Workflow: `PR Automation test suite`
> Tags: `@tag.Git`
> Spec: ``
> <hr>Tue, 03 Dec 2024 17:36:22 UTC
<!-- end of auto-generated comment: Cypress test results  -->


## Communication
Should the DevRel and Marketing teams inform users about this change?
- [ ] Yes
- [ ] No


<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

## Release Notes

- **New Features**
- Enhanced Git resource handling with the ability to save artifacts to
Git repositories.
- Introduced new methods for managing modified resources and artifact
exchange JSON.
- Improved tracking of updated entities during export operations,
including custom JavaScript libraries and new pages.
- New functionality for handling application artifacts and contexts in
Git.

- **Bug Fixes**
- Refined error handling and control flow in auto-commit and migration
processes.

- **Documentation**
- Updated comments and method signatures for clarity and improved
understanding.

- **Tests**
- Added new test cases to validate the conversion processes and resource
comparisons.
- Enhanced existing tests to utilize new data structures and improve
clarity.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
Nidhi 2024-12-04 00:01:51 +05:30 committed by GitHub
parent 008a94673d
commit a2c5caa819
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
24 changed files with 906 additions and 164 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

219
app/server/appsmith-git/src/main/java/com/appsmith/git/files/FileUtilsCEImpl.java
View File

@ -6,6 +6,9 @@ import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginException
import com.appsmith.external.git.FileInterface;
import com.appsmith.external.git.GitExecutor;
import com.appsmith.external.git.constants.GitSpan;
import com.appsmith.external.git.models.GitResourceIdentity;
import com.appsmith.external.git.models.GitResourceMap;
import com.appsmith.external.git.models.GitResourceType;
import com.appsmith.external.git.operations.FileOperations;
import com.appsmith.external.helpers.ObservationHelper;
import com.appsmith.external.helpers.Stopwatch;
@ -14,10 +17,12 @@ import com.appsmith.external.models.ArtifactGitReference;
import com.appsmith.git.configurations.GitServiceConfig;
import com.appsmith.git.constants.CommonConstants;
import com.appsmith.git.helpers.DSLTransformerHelper;
import com.fasterxml.jackson.databind.ObjectMapper;
import io.micrometer.tracing.Span;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.io.IOUtils;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.json.JSONObject;
@ -28,6 +33,8 @@ import org.springframework.util.StringUtils;
import reactor.core.publisher.Mono;
import reactor.core.scheduler.Scheduler;
import reactor.core.scheduler.Schedulers;
import reactor.util.function.Tuple2;
import reactor.util.function.Tuples;
import java.io.BufferedWriter;
import java.io.File;
@ -48,7 +55,10 @@ import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import static com.appsmith.external.git.constants.GitConstants.ACTION_COLLECTION_LIST;
import static com.appsmith.external.git.constants.GitConstants.ACTION_LIST;
@ -74,6 +84,7 @@ public class FileUtilsCEImpl implements FileInterface {
private final GitExecutor gitExecutor;
protected final FileOperations fileOperations;
private final ObservationHelper observationHelper;
protected final ObjectMapper objectMapper;
private static final String EDIT_MODE_URL_TEMPLATE = "{{editModeUrl}}";
@ -90,11 +101,23 @@ public class FileUtilsCEImpl implements FileInterface {
GitServiceConfig gitServiceConfig,
GitExecutor gitExecutor,
FileOperations fileOperations,
ObservationHelper observationHelper) {
ObservationHelper observationHelper,
ObjectMapper objectMapper) {
this.gitServiceConfig = gitServiceConfig;
this.gitExecutor = gitExecutor;
this.fileOperations = fileOperations;
this.observationHelper = observationHelper;
this.objectMapper = objectMapper;
}
protected Map<GitResourceType, GitResourceType> getModifiedResourcesTypes() {
return Map.of(
GitResourceType.JSLIB_CONFIG, GitResourceType.JSLIB_CONFIG,
GitResourceType.CONTEXT_CONFIG, GitResourceType.CONTEXT_CONFIG,
GitResourceType.QUERY_CONFIG, GitResourceType.QUERY_CONFIG,
GitResourceType.QUERY_DATA, GitResourceType.QUERY_CONFIG,
GitResourceType.JSOBJECT_CONFIG, GitResourceType.JSOBJECT_CONFIG,
GitResourceType.JSOBJECT_DATA, GitResourceType.JSOBJECT_CONFIG);
}
/**
@ -215,6 +238,101 @@ public class FileUtilsCEImpl implements FileInterface {
.subscribeOn(scheduler);
}
@Override
public Mono<Path> saveArtifactToGitRepo(Path baseRepoSuffix, GitResourceMap gitResourceMap, String branchName)
throws GitAPIException, IOException {
// Repo path will be:
// baseRepo : root/orgId/defaultAppId/repoName/{applicationData}
// Checkout to mentioned branch if not already checked-out
return gitExecutor
.resetToLastCommit(baseRepoSuffix, branchName)
.flatMap(isSwitched -> {
Path baseRepo = Paths.get(gitServiceConfig.getGitRootPath()).resolve(baseRepoSuffix);
try {
updateEntitiesInRepo(gitResourceMap, baseRepo);
} catch (IOException e) {
return Mono.error(e);
}
return Mono.just(baseRepo);
})
.subscribeOn(scheduler);
}
protected Set<String> getExistingFilesInRepo(Path baseRepo) throws IOException {
try (Stream<Path> stream = Files.walk(baseRepo).parallel()) {
return stream.filter(path -> {
try {
return Files.isRegularFile(path) || FileUtils.isEmptyDirectory(path.toFile());
} catch (IOException e) {
log.error("Unable to find file details. Please check the file at file path: {}", path);
log.error("Assuming that it does not exist for now ...");
return false;
}
})
.map(baseRepo::relativize)
.map(Path::toString)
.collect(Collectors.toSet());
}
}
protected Set<String> updateEntitiesInRepo(GitResourceMap gitResourceMap, Path baseRepo) throws IOException {
ModifiedResources modifiedResources = gitResourceMap.getModifiedResources();
Map<GitResourceIdentity, Object> resourceMap = gitResourceMap.getGitResourceMap();
Set<String> filesInRepo = getExistingFilesInRepo(baseRepo);
Set<String> updatedFilesToBeSerialized = resourceMap.keySet().parallelStream()
.map(gitResourceIdentity -> gitResourceIdentity.getFilePath())
.collect(Collectors.toSet());
// Remove all files that need to be serialized from the existing files list, as well as the README file
// What we are left with are all the files to be deleted
filesInRepo.removeAll(updatedFilesToBeSerialized);
filesInRepo.remove("README.md");
// Delete all the files because they are no longer needed
// This covers both older structures of storing files and,
// legitimate changes in the artifact that might cause deletions
filesInRepo.stream().parallel().forEach(filePath -> {
try {
Files.deleteIfExists(baseRepo.resolve(filePath));
} catch (IOException e) {
// We ignore files that could not be deleted and expect to come back to this at a later point
// Just log the path for now
log.error("Unable to delete file at path: {}", filePath);
}
});
// Now go through the resource map and based on resource type, check if the resource is modified before
// serialization
// Or simply choose the mechanism for serialization
Map<GitResourceType, GitResourceType> modifiedResourcesTypes = getModifiedResourcesTypes();
return resourceMap.entrySet().parallelStream()
.map(entry -> {
GitResourceIdentity key = entry.getKey();
boolean resourceUpdated = true;
if (modifiedResourcesTypes.containsKey(key.getResourceType()) && modifiedResources != null) {
GitResourceType comparisonType = modifiedResourcesTypes.get(key.getResourceType());
resourceUpdated =
modifiedResources.isResourceUpdatedNew(comparisonType, key.getResourceIdentifier());
}
if (resourceUpdated) {
String filePath = key.getFilePath();
saveResourceCommon(entry.getValue(), baseRepo.resolve(filePath));
return filePath;
}
return null;
})
.filter(Objects::nonNull)
.collect(Collectors.toSet());
}
protected Set<String> updateEntitiesInRepo(ApplicationGitReference applicationGitReference, Path baseRepo) {
Set<String> validDatasourceFileNames = new HashSet<>();
@ -434,6 +552,23 @@ public class FileUtilsCEImpl implements FileInterface {
return false;
}
protected void saveResourceCommon(Object sourceEntity, Path path) {
try {
Files.createDirectories(path.getParent());
if (sourceEntity instanceof String s) {
writeStringToFile(s, path);
return;
}
if (sourceEntity instanceof JSONObject) {
sourceEntity = objectMapper.readTree(sourceEntity.toString());
}
fileOperations.writeToFile(sourceEntity, path);
} catch (IOException e) {
log.error("Error while writing resource to file {} with {}", path, e.getMessage());
log.debug(e.getMessage());
}
}
/**
* This method is used to write actionCollection specific resource to file system. We write the data in two steps
* 1. Actual js code
@ -514,9 +649,9 @@ public class FileUtilsCEImpl implements FileInterface {
/**
* This will reconstruct the application from the repo
*
* @param organisationId To which organisation application needs to be rehydrated
* @param organisationId To which organisation application needs to be rehydrated
* @param baseApplicationId To which organisation application needs to be rehydrated
* @param branchName for which the application needs to be rehydrate
* @param branchName for which the application needs to be rehydrate
* @return application reference from which entire application can be rehydrated
*/
public Mono<ApplicationGitReference> reconstructApplicationReferenceFromGitRepo(
@ -672,6 +807,84 @@ public class FileUtilsCEImpl implements FileInterface {
directoryPath.resolve(directoryPath.toFile().getName() + CommonConstants.JSON_EXTENSION));
}
protected GitResourceMap fetchGitResourceMap(Path baseRepoPath) throws IOException {
// Extract application metadata from the json
Object metadata = fileOperations.readFile(
baseRepoPath.resolve(CommonConstants.METADATA + CommonConstants.JSON_EXTENSION));
Integer fileFormatVersion = fileOperations.getFileFormatVersion(metadata);
// Check if fileFormat of the saved files in repo is compatible
if (!isFileFormatCompatible(fileFormatVersion)) {
throw new AppsmithPluginException(AppsmithPluginError.INCOMPATIBLE_FILE_FORMAT);
}
GitResourceMap gitResourceMap = new GitResourceMap();
Map<GitResourceIdentity, Object> resourceMap = gitResourceMap.getGitResourceMap();
Set<String> filesInRepo = getExistingFilesInRepo(baseRepoPath);
filesInRepo.parallelStream()
.filter(path -> !Files.isDirectory(baseRepoPath.resolve(path)))
.forEach(filePath -> {
Tuple2<GitResourceIdentity, Object> identity = getGitResourceIdentity(baseRepoPath, filePath);
resourceMap.put(identity.getT1(), identity.getT2());
});
return gitResourceMap;
}
protected Tuple2<GitResourceIdentity, Object> getGitResourceIdentity(Path baseRepoPath, String filePath) {
Path path = baseRepoPath.resolve(filePath);
GitResourceIdentity identity;
Object contents = fileOperations.readFile(path);
if (!filePath.contains("/")) {
identity = new GitResourceIdentity(GitResourceType.ROOT_CONFIG, filePath, filePath);
} else if (filePath.matches(DATASOURCE_DIRECTORY + "/.*")) {
String gitSyncId =
objectMapper.valueToTree(contents).get("gitSyncId").asText();
identity = new GitResourceIdentity(GitResourceType.DATASOURCE_CONFIG, gitSyncId, filePath);
} else if (filePath.matches(JS_LIB_DIRECTORY + "/.*")) {
String fileName = FilenameUtils.getBaseName(filePath);
identity = new GitResourceIdentity(GitResourceType.JSLIB_CONFIG, fileName, filePath);
} else if (filePath.matches(PAGE_DIRECTORY + "/[^/]*/[^/]*.json]")) {
String gitSyncId =
objectMapper.valueToTree(contents).get("gitSyncId").asText();
identity = new GitResourceIdentity(GitResourceType.CONTEXT_CONFIG, gitSyncId, filePath);
} else if (filePath.matches(PAGE_DIRECTORY + "/[^/]*/" + ACTION_DIRECTORY + "/.*/metadata.json")) {
String gitSyncId =
objectMapper.valueToTree(contents).get("gitSyncId").asText();
identity = new GitResourceIdentity(GitResourceType.QUERY_CONFIG, gitSyncId, filePath);
} else if (filePath.matches(PAGE_DIRECTORY + "/[^/]*/" + ACTION_DIRECTORY + "/.*\\.txt")) {
Object configContents = fileOperations.readFile(path.getParent().resolve("metadata.json"));
String gitSyncId =
objectMapper.valueToTree(configContents).get("gitSyncId").asText();
identity = new GitResourceIdentity(GitResourceType.QUERY_DATA, gitSyncId, filePath);
} else if (filePath.matches(PAGE_DIRECTORY + "/[^/]*/" + ACTION_COLLECTION_DIRECTORY + "/.*/metadata.json")) {
String gitSyncId =
objectMapper.valueToTree(contents).get("gitSyncId").asText();
identity = new GitResourceIdentity(GitResourceType.JSOBJECT_CONFIG, gitSyncId, filePath);
} else if (filePath.matches(PAGE_DIRECTORY + "/[^/]*/" + ACTION_COLLECTION_DIRECTORY + "/.*\\.js")) {
Object configContents = fileOperations.readFile(path.getParent().resolve("metadata.json"));
String gitSyncId =
objectMapper.valueToTree(configContents).get("gitSyncId").asText();
identity = new GitResourceIdentity(GitResourceType.JSOBJECT_DATA, gitSyncId, filePath);
} else if (filePath.matches(PAGE_DIRECTORY + "/[^/]*/widgets/.*\\.json")) {
Pattern pageDirPattern = Pattern.compile("(" + PAGE_DIRECTORY + "/([^/]*))/widgets/.*\\.json");
Matcher matcher = pageDirPattern.matcher(filePath);
matcher.find();
String pageDirectory = matcher.group(1);
String pageName = matcher.group(2) + ".json";
Object configContents =
fileOperations.readFile(baseRepoPath.resolve(pageDirectory).resolve(pageName));
String gitSyncId =
objectMapper.valueToTree(configContents).get("gitSyncId").asText();
String widgetId = objectMapper.valueToTree(contents).get("widgetId").asText();
identity = new GitResourceIdentity(GitResourceType.JSOBJECT_DATA, gitSyncId + "-" + widgetId, filePath);
} else return null;
return Tuples.of(identity, contents);
}
private ApplicationGitReference fetchApplicationReference(Path baseRepoPath) {
ApplicationGitReference applicationGitReference = new ApplicationGitReference();
// Extract application metadata from the json

6
app/server/appsmith-git/src/main/java/com/appsmith/git/files/FileUtilsImpl.java
View File

@ -5,6 +5,7 @@ import com.appsmith.external.git.GitExecutor;
import com.appsmith.external.git.operations.FileOperations;
import com.appsmith.external.helpers.ObservationHelper;
import com.appsmith.git.configurations.GitServiceConfig;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.annotation.Import;
@ -22,7 +23,8 @@ public class FileUtilsImpl extends FileUtilsCEImpl implements FileInterface {
GitServiceConfig gitServiceConfig,
GitExecutor gitExecutor,
FileOperations fileOperations,
ObservationHelper observationHelper) {
super(gitServiceConfig, gitExecutor, fileOperations, observationHelper);
ObservationHelper observationHelper,
ObjectMapper objectMapper) {
super(gitServiceConfig, gitExecutor, fileOperations, observationHelper, objectMapper);
}
}

34
app/server/appsmith-git/src/main/java/com/appsmith/git/helpers/DSLTransformerHelper.java
View File

@ -16,6 +16,10 @@ import java.util.TreeMap;
import java.util.stream.Collectors;
import static com.appsmith.git.constants.CommonConstants.CANVAS_WIDGET;
import static com.appsmith.git.constants.CommonConstants.DELIMITER_PATH;
import static com.appsmith.git.constants.CommonConstants.DELIMITER_POINT;
import static com.appsmith.git.constants.CommonConstants.EMPTY_STRING;
import static com.appsmith.git.constants.CommonConstants.MAIN_CONTAINER;
@Component
@RequiredArgsConstructor
@ -24,7 +28,7 @@ public class DSLTransformerHelper {
public static Map<String, JSONObject> flatten(JSONObject jsonObject) {
Map<String, JSONObject> flattenedMap = new HashMap<>();
flattenObject(jsonObject, CommonConstants.EMPTY_STRING, flattenedMap);
flattenObject(jsonObject, EMPTY_STRING, flattenedMap);
return new TreeMap<>(flattenedMap);
}
@ -44,8 +48,7 @@ public class DSLTransformerHelper {
for (int i = 0; i < children.length(); i++) {
JSONObject childObject = children.getJSONObject(i);
String childPrefix =
isCanvasWidget(childObject) ? prefix + widgetName + CommonConstants.DELIMITER_POINT : prefix;
String childPrefix = isCanvasWidget(childObject) ? prefix + widgetName + DELIMITER_POINT : prefix;
String widgetType = getWidgetType(jsonObject);
flattenObject(childObject, childPrefix, flattenedMap);
}
@ -103,13 +106,17 @@ public class DSLTransformerHelper {
Map<String, List<String>> parentDirectories = new HashMap<>();
paths = paths.stream()
.map(currentPath -> currentPath.replace(CommonConstants.JSON_EXTENSION, CommonConstants.EMPTY_STRING))
.map(currentPath -> currentPath.replace(CommonConstants.JSON_EXTENSION, EMPTY_STRING))
.collect(Collectors.toList());
for (String path : paths) {
String[] directories = path.split(CommonConstants.DELIMITER_PATH);
String[] directories = path.split(DELIMITER_PATH);
int lastDirectoryIndex = directories.length - 1;
if (lastDirectoryIndex > 0 && directories[lastDirectoryIndex].equals(directories[lastDirectoryIndex - 1])) {
if (lastDirectoryIndex <= 0) {
// This is not a valid path anymore, ignore
continue;
}
if (directories[lastDirectoryIndex].equals(directories[lastDirectoryIndex - 1])) {
if (lastDirectoryIndex - 2 >= 0) {
String parentDirectory = directories[lastDirectoryIndex - 2];
List<String> pathsList = parentDirectories.getOrDefault(parentDirectory, new ArrayList<>());
@ -143,10 +150,10 @@ public class DSLTransformerHelper {
Map<String, JSONObject> jsonMap, Map<String, List<String>> pathMapping, JSONObject mainContainer) {
// start from the root
// Empty page with no widgets
if (!pathMapping.containsKey(CommonConstants.MAIN_CONTAINER)) {
if (!pathMapping.containsKey(MAIN_CONTAINER)) {
return mainContainer;
}
for (String path : pathMapping.get(CommonConstants.MAIN_CONTAINER)) {
for (String path : pathMapping.get(MAIN_CONTAINER)) {
JSONObject child = getChildren(path, jsonMap, pathMapping);
JSONArray children = mainContainer.optJSONArray(CommonConstants.CHILDREN);
if (children == null) {
@ -179,7 +186,7 @@ public class DSLTransformerHelper {
}
public static String getWidgetName(String path) {
String[] directories = path.split(CommonConstants.DELIMITER_PATH);
String[] directories = path.split(DELIMITER_PATH);
return directories[directories.length - 1];
}
@ -229,15 +236,16 @@ public class DSLTransformerHelper {
public static String getPathToWidgetFile(String key, JSONObject jsonObject, String widgetName) {
// get path with splitting the name via key
String childPath = key.replace(CommonConstants.MAIN_CONTAINER, CommonConstants.EMPTY_STRING)
.replace(CommonConstants.DELIMITER_POINT, CommonConstants.DELIMITER_PATH);
String childPath = key.replace(MAIN_CONTAINER, EMPTY_STRING).replace(DELIMITER_POINT, DELIMITER_PATH);
// Replace the canvas Widget as a child and add it to the same level as parent
childPath = childPath.replaceAll(CANVAS_WIDGET, CommonConstants.EMPTY_STRING);
childPath = childPath.replaceAll(CANVAS_WIDGET, EMPTY_STRING);
if (!DSLTransformerHelper.hasChildren(jsonObject) && !DSLTransformerHelper.isTabsWidget(jsonObject)) {
// Save the widget as a directory or Save the widget as a file
// Only consider widgetName at the end of the childPath to reset
// For example, "foobar/bar" should convert into "foobar/"
childPath = childPath.replaceAll(widgetName + "$", CommonConstants.EMPTY_STRING);
childPath = childPath.replaceAll(widgetName + "$", EMPTY_STRING);
} else {
childPath += DELIMITER_PATH;
}
return childPath;

4
app/server/appsmith-git/src/test/java/com/appsmith/git/helpers/FileUtilsImplTest.java
View File

@ -7,6 +7,7 @@ import com.appsmith.git.configurations.GitServiceConfig;
import com.appsmith.git.files.FileUtilsImpl;
import com.appsmith.git.files.operations.FileOperationsImpl;
import com.appsmith.git.service.GitExecutorImpl;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.commons.io.FileUtils;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.junit.jupiter.api.AfterEach;
@ -42,7 +43,8 @@ public class FileUtilsImplTest {
GitServiceConfig gitServiceConfig = new GitServiceConfig();
gitServiceConfig.setGitRootPath(localTestDirectoryPath.toString());
FileOperations fileOperations = new FileOperationsImpl(null, ObservationHelper.NOOP);
fileUtils = new FileUtilsImpl(gitServiceConfig, gitExecutor, fileOperations, ObservationHelper.NOOP);
fileUtils = new FileUtilsImpl(
gitServiceConfig, gitExecutor, fileOperations, ObservationHelper.NOOP, new ObjectMapper());
}
@AfterEach

55
app/server/appsmith-interfaces/src/main/java/com/appsmith/external/dtos/ModifiedResources.java vendored
View File

@ -1,13 +1,7 @@
package com.appsmith.external.dtos;
import com.appsmith.external.dtos.ce.ModifiedResourcesCE;
import lombok.Data;
import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
/**
* This DTO class is used to store which resources have been updated after the last commit.
@ -17,49 +11,4 @@ import java.util.concurrent.ConcurrentHashMap;
* the pages to file system for difference git processes e.g. check git status, commit etc
*/
@Data
public class ModifiedResources {
// boolean flag to set whether all the resources should be considered as updated or not, it'll be false by default
private boolean isAllModified;
// a map to store the type of the resources and related entries
Map<String, Set<String>> modifiedResourceMap = new ConcurrentHashMap<>();
/**
* Checks whether the provided resource name should be considered as modified or not.
* It'll return true if the isAllModified flag is set or the resource is present in the modifiedResourceMap
* @param resourceType String, type of the resource e.g. PAGE_LIST
* @param resourceName String, name of the resource e.g. "Home Page"
* @return true if modified, false otherwise
*/
public boolean isResourceUpdated(String resourceType, String resourceName) {
return StringUtils.isNotEmpty(resourceType)
&& (isAllModified
|| (!CollectionUtils.isEmpty(modifiedResourceMap.get(resourceType))
&& modifiedResourceMap.get(resourceType).contains(resourceName)));
}
/**
* Adds a new resource to the map. Will create a new set if no set found for the provided resource type.
* @param resourceType String, type of the resource e.g. PAGE_LST
* @param resourceName String, name of the resource e.g. Home Page
*/
public void putResource(String resourceType, String resourceName) {
if (!this.modifiedResourceMap.containsKey(resourceType)) {
this.modifiedResourceMap.put(resourceType, new HashSet<>());
}
this.modifiedResourceMap.get(resourceType).add(resourceName);
}
/**
* Adds a set of resources to the map. Will create a new set if no set found for the provided resource type.
* It'll append the resources to the set.
* @param resourceType String, type of the resource e.g. PAGE_LST
* @param resourceNames Set of String, names of the resource e.g. Home Page, About page
*/
public void putResource(String resourceType, Set<String> resourceNames) {
if (!this.modifiedResourceMap.containsKey(resourceType)) {
this.modifiedResourceMap.put(resourceType, new HashSet<>());
}
this.modifiedResourceMap.get(resourceType).addAll(resourceNames);
}
}
public class ModifiedResources extends ModifiedResourcesCE {}

86
app/server/appsmith-interfaces/src/main/java/com/appsmith/external/dtos/ce/ModifiedResourcesCE.java vendored Normal file
View File

@ -0,0 +1,86 @@
package com.appsmith.external.dtos.ce;
import com.appsmith.external.git.models.GitResourceType;
import lombok.Data;
import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
/**
* This DTO class is used to store which resources have been updated after the last commit.
* Primarily the export process sets this information and git import process uses this information to identify
* which resources need to be written in file system. For example, if a page has not been updated after the last commit,
* the name of the page should not be part of the modifiedResourceMap so that git will skip this page when it writes
* the pages to file system for difference git processes e.g. check git status, commit etc
*/
@Data
public class ModifiedResourcesCE {
// boolean flag to set whether all the resources should be considered as updated or not, it'll be false by default
private boolean isAllModified;
// a map to store the type of the resources and related entries
Map<String, Set<String>> modifiedResourceMap = new ConcurrentHashMap<>();
Map<GitResourceType, Set<String>> modifiedResourceIdentifiers = new ConcurrentHashMap<>();
public Map<GitResourceType, Set<String>> getModifiedResourceIdentifiers() {
if (this.modifiedResourceIdentifiers.isEmpty()) {
this.modifiedResourceIdentifiers.putAll(Map.of(
GitResourceType.CONTEXT_CONFIG, ConcurrentHashMap.newKeySet(),
GitResourceType.JSLIB_CONFIG, ConcurrentHashMap.newKeySet(),
GitResourceType.QUERY_CONFIG, ConcurrentHashMap.newKeySet(),
GitResourceType.JSOBJECT_CONFIG, ConcurrentHashMap.newKeySet()));
}
return modifiedResourceIdentifiers;
}
/**
* Checks whether the provided resource name should be considered as modified or not.
* It'll return true if the isAllModified flag is set or the resource is present in the modifiedResourceMap
* @param resourceType String, type of the resource e.g. PAGE_LIST
* @param resourceName String, name of the resource e.g. "Home Page"
* @return true if modified, false otherwise
*/
public boolean isResourceUpdated(String resourceType, String resourceName) {
return StringUtils.isNotEmpty(resourceType)
&& (isAllModified
|| (!CollectionUtils.isEmpty(modifiedResourceMap.get(resourceType))
&& modifiedResourceMap.get(resourceType).contains(resourceName)));
}
public boolean isResourceUpdatedNew(GitResourceType resourceType, String resourceIdentifier) {
return StringUtils.isNotEmpty(resourceIdentifier)
&& (isAllModified
|| (!CollectionUtils.isEmpty(modifiedResourceIdentifiers.get(resourceType))
&& modifiedResourceIdentifiers.get(resourceType).contains(resourceIdentifier)));
}
/**
* Adds a new resource to the map. Will create a new set if no set found for the provided resource type.
* @param resourceType String, type of the resource e.g. PAGE_LST
* @param resourceName String, name of the resource e.g. Home Page
*/
public void putResource(String resourceType, String resourceName) {
if (!this.modifiedResourceMap.containsKey(resourceType)) {
this.modifiedResourceMap.put(resourceType, new HashSet<>());
}
this.modifiedResourceMap.get(resourceType).add(resourceName);
}
/**
* Adds a set of resources to the map. Will create a new set if no set found for the provided resource type.
* It'll append the resources to the set.
* @param resourceType String, type of the resource e.g. PAGE_LST
* @param resourceNames Set of String, names of the resource e.g. Home Page, About page
*/
public void putResource(String resourceType, Set<String> resourceNames) {
if (!this.modifiedResourceMap.containsKey(resourceType)) {
this.modifiedResourceMap.put(resourceType, new HashSet<>());
}
this.modifiedResourceMap.get(resourceType).addAll(resourceNames);
}
}

4
app/server/appsmith-interfaces/src/main/java/com/appsmith/external/git/FileInterface.java vendored
View File

@ -1,5 +1,6 @@
package com.appsmith.external.git;
import com.appsmith.external.git.models.GitResourceMap;
import com.appsmith.external.models.ApplicationGitReference;
import com.appsmith.external.models.ArtifactGitReference;
import org.eclipse.jgit.api.errors.GitAPIException;
@ -34,6 +35,9 @@ public interface FileInterface {
Path baseRepoSuffix, ArtifactGitReference artifactGitReference, String branchName)
throws IOException, GitAPIException;
Mono<Path> saveArtifactToGitRepo(Path baseRepoSuffix, GitResourceMap gitResourceMap, String branchName)
throws GitAPIException, IOException;
/**
* This method will reconstruct the application from the repo
*

4
app/server/appsmith-interfaces/src/main/java/com/appsmith/external/git/models/GitResourceIdentity.java vendored
View File

@ -8,8 +8,6 @@ import lombok.RequiredArgsConstructor;
@Data
@RequiredArgsConstructor
public class GitResourceIdentity {
// TODO @Nidhi should we persist the info from parsing this filePath ?
String filePath;
// TODO @Nidhi should we persist this sha against the Appsmith domain to integrate with the isModified logic?
String sha;
@ -25,4 +23,6 @@ public class GitResourceIdentity {
// root dir files -> fileName
@NonNull @EqualsAndHashCode.Include
String resourceIdentifier;
@NonNull String filePath;
}

18
app/server/appsmith-server/src/main/java/com/appsmith/server/actioncollections/exportable/ActionCollectionExportableServiceCEImpl.java
View File

@ -1,5 +1,6 @@
package com.appsmith.server.actioncollections.exportable;
import com.appsmith.external.git.models.GitResourceType;
import com.appsmith.server.acl.AclPermission;
import com.appsmith.server.constants.FieldName;
import com.appsmith.server.domains.ActionCollection;
@ -11,7 +12,6 @@ import com.appsmith.server.dtos.ExportingMetaDTO;
import com.appsmith.server.dtos.MappedExportableResourcesDTO;
import com.appsmith.server.exports.exportable.ExportableServiceCE;
import com.appsmith.server.exports.exportable.artifactbased.ArtifactBasedExportableService;
import com.appsmith.server.helpers.ImportExportUtils;
import com.appsmith.server.solutions.ActionPermission;
import lombok.RequiredArgsConstructor;
import reactor.core.publisher.Flux;
@ -67,6 +67,7 @@ public class ActionCollectionExportableServiceCEImpl implements ExportableServic
// Because the actions will have a reference to the collection
Set<String> updatedActionCollectionSet = new HashSet<>();
Set<String> updatedIdentifiers = new HashSet<>();
actionCollections.forEach(actionCollection -> {
ActionCollectionDTO publishedActionCollectionDTO = actionCollection.getPublishedCollection();
ActionCollectionDTO unpublishedActionCollectionDTO =
@ -78,9 +79,12 @@ public class ActionCollectionExportableServiceCEImpl implements ExportableServic
// we've replaced page id with page name in previous step
String contextNameAtIdReference =
artifactBasedExportableService.getContextNameAtIdReference(actionCollectionDTO);
String contextListPath = artifactBasedExportableService.getContextListPath();
boolean isContextUpdated = ImportExportUtils.isContextNameInUpdatedList(
artifactExchangeJson, contextNameAtIdReference, contextListPath);
String contextGitSyncId = mappedExportableResourcesDTO
.getContextNameToGitSyncIdMap()
.get(contextNameAtIdReference);
boolean isContextUpdated = artifactExchangeJson
.getModifiedResources()
.isResourceUpdatedNew(GitResourceType.CONTEXT_CONFIG, contextGitSyncId);
String actionCollectionName =
actionCollectionDTO.getUserExecutableName() + NAME_SEPARATOR + contextNameAtIdReference;
Instant actionCollectionUpdatedAt = actionCollection.getUpdatedAt();
@ -92,6 +96,7 @@ public class ActionCollectionExportableServiceCEImpl implements ExportableServic
|| exportingMetaDTO.getArtifactLastCommittedAt().isBefore(actionCollectionUpdatedAt);
if (isActionCollectionUpdated) {
updatedActionCollectionSet.add(actionCollectionName);
updatedIdentifiers.add(actionCollection.getGitSyncId());
}
actionCollection.sanitiseToExportDBObject();
});
@ -100,6 +105,11 @@ public class ActionCollectionExportableServiceCEImpl implements ExportableServic
artifactExchangeJson
.getModifiedResources()
.putResource(FieldName.ACTION_COLLECTION_LIST, updatedActionCollectionSet);
artifactExchangeJson
.getModifiedResources()
.getModifiedResourceIdentifiers()
.get(GitResourceType.JSOBJECT_CONFIG)
.addAll(updatedIdentifiers);
return actionCollections;
})

142
app/server/appsmith-server/src/main/java/com/appsmith/server/applications/git/ApplicationGitFileUtilsCEImpl.java
View File

@ -19,6 +19,7 @@ import com.appsmith.server.domains.ActionCollection;
import com.appsmith.server.domains.Application;
import com.appsmith.server.domains.ApplicationPage;
import com.appsmith.server.domains.CustomJSLib;
import com.appsmith.server.domains.Layout;
import com.appsmith.server.domains.NewAction;
import com.appsmith.server.domains.NewPage;
import com.appsmith.server.domains.Theme;
@ -32,9 +33,12 @@ import com.appsmith.server.helpers.CollectionUtils;
import com.appsmith.server.helpers.ce.ArtifactGitFileUtilsCE;
import com.appsmith.server.migrations.JsonSchemaMigration;
import com.appsmith.server.newactions.base.NewActionService;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.MapperFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.gson.Gson;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import net.minidev.json.JSONObject;
import net.minidev.json.parser.JSONParser;
@ -60,6 +64,11 @@ import java.util.stream.Collectors;
import static com.appsmith.external.git.constants.GitConstants.NAME_SEPARATOR;
import static com.appsmith.external.helpers.AppsmithBeanUtils.copyNestedNonNullProperties;
import static com.appsmith.external.helpers.AppsmithBeanUtils.copyProperties;
import static com.appsmith.git.constants.CommonConstants.DELIMITER_PATH;
import static com.appsmith.git.constants.CommonConstants.JSON_EXTENSION;
import static com.appsmith.git.constants.CommonConstants.MAIN_CONTAINER;
import static com.appsmith.git.constants.CommonConstants.WIDGETS;
import static com.appsmith.git.constants.ce.GitDirectoriesCE.PAGE_DIRECTORY;
import static com.appsmith.server.constants.FieldName.ACTION_COLLECTION_LIST;
import static com.appsmith.server.constants.FieldName.ACTION_LIST;
import static com.appsmith.server.constants.FieldName.CHILDREN;
@ -70,20 +79,36 @@ import static com.appsmith.server.constants.FieldName.EDIT_MODE_THEME;
import static com.appsmith.server.constants.FieldName.EXPORTED_APPLICATION;
import static com.appsmith.server.constants.FieldName.PAGE_LIST;
import static com.appsmith.server.constants.FieldName.WIDGET_ID;
import static com.appsmith.server.constants.ce.FieldNameCE.WIDGET_NAME;
import static com.appsmith.server.helpers.ce.CommonGitFileUtilsCE.removeUnwantedFieldsFromBaseDomain;
@Slf4j
@Component
@RequiredArgsConstructor
@Import({FileUtilsImpl.class})
public class ApplicationGitFileUtilsCEImpl implements ArtifactGitFileUtilsCE<ApplicationGitReference> {
private final Gson gson;
private final ObjectMapper objectMapper;
private final NewActionService newActionService;
private final FileInterface fileUtils;
private final JsonSchemaMigration jsonSchemaMigration;
private final ActionCollectionService actionCollectionService;
public ApplicationGitFileUtilsCEImpl(
Gson gson,
ObjectMapper objectMapper,
NewActionService newActionService,
FileInterface fileUtils,
JsonSchemaMigration jsonSchemaMigration,
ActionCollectionService actionCollectionService) {
this.gson = gson;
this.objectMapper = objectMapper.copy().disable(MapperFeature.USE_ANNOTATIONS);
this.newActionService = newActionService;
this.fileUtils = fileUtils;
this.jsonSchemaMigration = jsonSchemaMigration;
this.actionCollectionService = actionCollectionService;
}
// Only include the application helper fields in metadata object
protected Set<String> getBlockedMetadataFields() {
return Set.of(
@ -109,6 +134,11 @@ public class ApplicationGitFileUtilsCEImpl implements ArtifactGitFileUtilsCE<App
return new ApplicationGitReference();
}
@Override
public ArtifactExchangeJson createArtifactExchangeJsonObject() {
return new ApplicationJson();
}
@Override
public void addArtifactReferenceFromExportedJson(
ArtifactExchangeJson artifactExchangeJson, ArtifactGitReference artifactGitReference) {
@ -141,8 +171,9 @@ public class ApplicationGitFileUtilsCEImpl implements ArtifactGitFileUtilsCE<App
// application
Application application = applicationJson.getExportedApplication();
removeUnwantedFieldsFromApplication(application);
GitResourceIdentity applicationIdentity = new GitResourceIdentity(
GitResourceType.ROOT_CONFIG, CommonConstants.APPLICATION + CommonConstants.JSON_EXTENSION);
final String applicationFilePath = CommonConstants.APPLICATION + JSON_EXTENSION;
GitResourceIdentity applicationIdentity =
new GitResourceIdentity(GitResourceType.ROOT_CONFIG, applicationFilePath, applicationFilePath);
resourceMap.put(applicationIdentity, application);
// metadata
@ -154,9 +185,11 @@ public class ApplicationGitFileUtilsCEImpl implements ArtifactGitFileUtilsCE<App
ApplicationJson applicationMetadata = new ApplicationJson();
applicationJson.setModifiedResources(null);
copyProperties(applicationJson, applicationMetadata, keys);
GitResourceIdentity metadataIdentity = new GitResourceIdentity(
GitResourceType.ROOT_CONFIG, CommonConstants.METADATA + CommonConstants.JSON_EXTENSION);
resourceMap.put(metadataIdentity, applicationMetadata);
final String metadataFilePath = CommonConstants.METADATA + JSON_EXTENSION;
ObjectNode metadata = objectMapper.valueToTree(applicationMetadata);
GitResourceIdentity metadataIdentity =
new GitResourceIdentity(GitResourceType.ROOT_CONFIG, metadataFilePath, metadataFilePath);
resourceMap.put(metadataIdentity, metadata);
// pages and widgets
applicationJson.getPageList().stream()
@ -166,23 +199,30 @@ public class ApplicationGitFileUtilsCEImpl implements ArtifactGitFileUtilsCE<App
&& newPage.getUnpublishedPage().getDeletedAt() == null)
.forEach(newPage -> {
removeUnwantedFieldsFromPage(newPage);
JSONObject dsl =
newPage.getUnpublishedPage().getLayouts().get(0).getDsl();
PageDTO pageDTO = newPage.getUnpublishedPage();
JSONObject dsl = pageDTO.getLayouts().get(0).getDsl();
// Get MainContainer widget data, remove the children and club with Canvas.json file
JSONObject mainContainer = new JSONObject(dsl);
mainContainer.remove(CHILDREN);
newPage.getUnpublishedPage().getLayouts().get(0).setDsl(mainContainer);
pageDTO.getLayouts().get(0).setDsl(mainContainer);
// pageName will be used for naming the json file
GitResourceIdentity pageIdentity =
new GitResourceIdentity(GitResourceType.CONTEXT_CONFIG, newPage.getGitSyncId());
final String pagePathPrefix = PAGE_DIRECTORY + DELIMITER_PATH + pageDTO.getName() + DELIMITER_PATH;
final String pageFilePath = pagePathPrefix + pageDTO.getName() + JSON_EXTENSION;
GitResourceIdentity pageIdentity = new GitResourceIdentity(
GitResourceType.CONTEXT_CONFIG, newPage.getGitSyncId(), pageFilePath);
resourceMap.put(pageIdentity, newPage);
Map<String, org.json.JSONObject> result =
DSLTransformerHelper.flatten(new org.json.JSONObject(dsl.toString()));
result.forEach((key, jsonObject) -> {
String widgetId = newPage.getGitSyncId() + "-" + jsonObject.getString(WIDGET_ID);
String widgetsPath = pagePathPrefix + WIDGETS + DELIMITER_PATH;
String widgetName = jsonObject.getString(WIDGET_NAME);
String subPath = DSLTransformerHelper.getPathToWidgetFile(key, jsonObject, widgetName);
String widgetPath = widgetsPath + subPath + widgetName + JSON_EXTENSION;
GitResourceIdentity widgetIdentity =
new GitResourceIdentity(GitResourceType.WIDGET_CONFIG, widgetId);
new GitResourceIdentity(GitResourceType.WIDGET_CONFIG, widgetId, widgetPath);
resourceMap.put(widgetIdentity, jsonObject);
});
});
@ -629,4 +669,80 @@ public class ApplicationGitFileUtilsCEImpl implements ArtifactGitFileUtilsCE<App
varargs.addAll(List.of(args));
return Paths.get(workspaceId, varargs.toArray(new String[0]));
}
@Override
public void setArtifactDependentPropertiesInJson(
GitResourceMap gitResourceMap, ArtifactExchangeJson artifactExchangeJson) {
Map<GitResourceIdentity, Object> resourceMap = gitResourceMap.getGitResourceMap();
// exported application
final String applicationFilePath = CommonConstants.APPLICATION + JSON_EXTENSION;
GitResourceIdentity applicationJsonIdentity =
new GitResourceIdentity(GitResourceType.ROOT_CONFIG, applicationFilePath, applicationFilePath);
Object applicationObject = resourceMap.get(applicationJsonIdentity);
Application application = objectMapper.convertValue(applicationObject, Application.class);
artifactExchangeJson.setArtifact(application);
// metadata
final String metadataFilePath = CommonConstants.METADATA + JSON_EXTENSION;
GitResourceIdentity metadataIdentity =
new GitResourceIdentity(GitResourceType.ROOT_CONFIG, metadataFilePath, metadataFilePath);
Object metadataObject = resourceMap.get(metadataIdentity);
ApplicationJson metadata = objectMapper.convertValue(metadataObject, ApplicationJson.class);
copyNestedNonNullProperties(metadata, artifactExchangeJson);
// pages
List<NewPage> pageList = resourceMap.entrySet().stream()
.filter(entry -> {
GitResourceIdentity key = entry.getKey();
return GitResourceType.CONTEXT_CONFIG.equals(key.getResourceType());
})
.map(Map.Entry::getValue)
.map(pageObject -> objectMapper.convertValue(pageObject, NewPage.class))
.collect(Collectors.toList());
artifactExchangeJson.setContextList(pageList);
// widgets
pageList.parallelStream().forEach(newPage -> {
Map<String, org.json.JSONObject> widgetsData = resourceMap.entrySet().stream()
.filter(entry -> {
GitResourceIdentity key = entry.getKey();
return GitResourceType.WIDGET_CONFIG.equals(key.getResourceType())
&& key.getResourceIdentifier().startsWith(newPage.getGitSyncId() + "-");
})
.collect(Collectors.toMap(
entry -> entry.getKey()
.getFilePath()
.replaceFirst(
PAGE_DIRECTORY
+ newPage.getUnpublishedPage()
.getName()
+ DELIMITER_PATH
+ WIDGETS
+ DELIMITER_PATH,
MAIN_CONTAINER + DELIMITER_PATH),
entry -> (org.json.JSONObject) entry.getValue()));
Layout layout = newPage.getUnpublishedPage().getLayouts().get(0);
org.json.JSONObject mainContainer;
try {
mainContainer = new org.json.JSONObject(objectMapper.writeValueAsString(layout.getDsl()));
Map<String, List<String>> parentDirectories = DSLTransformerHelper.calculateParentDirectories(
widgetsData.keySet().stream().toList());
org.json.JSONObject nestedDSL =
DSLTransformerHelper.getNestedDSL(widgetsData, parentDirectories, mainContainer);
JSONParser jsonParser = new JSONParser();
JSONObject parsedDSL = jsonParser.parse(nestedDSL.toString(), JSONObject.class);
layout.setDsl(parsedDSL);
} catch (ParseException | JsonProcessingException e) {
throw new RuntimeException(e);
}
});
}
}

4
app/server/appsmith-server/src/main/java/com/appsmith/server/applications/git/ApplicationGitFileUtilsImpl.java
View File

@ -6,6 +6,7 @@ import com.appsmith.server.actioncollections.base.ActionCollectionService;
import com.appsmith.server.helpers.ArtifactGitFileUtils;
import com.appsmith.server.migrations.JsonSchemaMigration;
import com.appsmith.server.newactions.base.NewActionService;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.gson.Gson;
import org.springframework.stereotype.Component;
@ -15,10 +16,11 @@ public class ApplicationGitFileUtilsImpl extends ApplicationGitFileUtilsCEImpl
public ApplicationGitFileUtilsImpl(
Gson gson,
ObjectMapper objectMapper,
NewActionService newActionService,
FileInterface fileUtils,
JsonSchemaMigration jsonSchemaMigration,
ActionCollectionService actionCollectionService) {
super(gson, newActionService, fileUtils, jsonSchemaMigration, actionCollectionService);
super(gson, objectMapper, newActionService, fileUtils, jsonSchemaMigration, actionCollectionService);
}
}

11
app/server/appsmith-server/src/main/java/com/appsmith/server/dtos/ce/ApplicationJsonCE.java
View File

@ -10,6 +10,7 @@ import com.appsmith.server.constants.ArtifactType;
import com.appsmith.server.domains.ActionCollection;
import com.appsmith.server.domains.Application;
import com.appsmith.server.domains.Artifact;
import com.appsmith.server.domains.Context;
import com.appsmith.server.domains.CustomJSLib;
import com.appsmith.server.domains.NewAction;
import com.appsmith.server.domains.NewPage;
@ -123,6 +124,11 @@ public class ApplicationJsonCE implements ArtifactExchangeJsonCE {
return this.getExportedApplication();
}
@Override
public <T extends Artifact> void setArtifact(T application) {
this.exportedApplication = (Application) application;
}
@Override
public void setThemes(Theme unpublishedTheme, Theme publishedTheme) {
this.setEditModeTheme(unpublishedTheme);
@ -138,4 +144,9 @@ public class ApplicationJsonCE implements ArtifactExchangeJsonCE {
public List<NewPage> getContextList() {
return this.pageList;
}
@Override
public <T extends Context> void setContextList(List<T> contextList) {
this.pageList = (List<NewPage>) contextList;
}
}

4
app/server/appsmith-server/src/main/java/com/appsmith/server/dtos/ce/ArtifactExchangeJsonCE.java
View File

@ -30,6 +30,8 @@ public interface ArtifactExchangeJsonCE {
Artifact getArtifact();
<T extends Artifact> void setArtifact(T artifact);
default void setThemes(Theme unpublishedTheme, Theme publishedTheme) {}
default List<CustomJSLib> getCustomJSLibList() {
@ -68,4 +70,6 @@ public interface ArtifactExchangeJsonCE {
@JsonView(Views.Internal.class)
List<? extends Context> getContextList();
<T extends Context> void setContextList(List<T> contextList);
}

1
app/server/appsmith-server/src/main/java/com/appsmith/server/dtos/ce/MappedExportableResourcesCE_DTO.java
View File

@ -15,6 +15,7 @@ public class MappedExportableResourcesCE_DTO {
Map<String, String> datasourceIdToNameMap = new HashMap<>();
Map<String, Instant> datasourceNameToUpdatedAtMap = new HashMap<>();
Map<String, String> contextIdToNameMap = new HashMap<>();
Map<String, String> contextNameToGitSyncIdMap = new HashMap<>();
Map<String, String> actionIdToNameMap = new HashMap<>();
Map<String, String> collectionIdToNameMap = new HashMap<>();
}

24
app/server/appsmith-server/src/main/java/com/appsmith/server/git/autocommit/AutoCommitEventHandlerCEImpl.java
View File

@ -4,12 +4,14 @@ import com.appsmith.external.constants.AnalyticsEvents;
import com.appsmith.external.dtos.ModifiedResources;
import com.appsmith.external.git.GitExecutor;
import com.appsmith.external.git.constants.GitConstants.GitCommandConstants;
import com.appsmith.external.git.models.GitResourceType;
import com.appsmith.server.configurations.ProjectProperties;
import com.appsmith.server.constants.ArtifactType;
import com.appsmith.server.constants.FieldName;
import com.appsmith.server.domains.Layout;
import com.appsmith.server.domains.NewPage;
import com.appsmith.server.dtos.ApplicationJson;
import com.appsmith.server.dtos.PageDTO;
import com.appsmith.server.events.AutoCommitEvent;
import com.appsmith.server.exceptions.AppsmithError;
import com.appsmith.server.exceptions.AppsmithException;
@ -28,14 +30,16 @@ import org.springframework.scheduling.annotation.Async;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import reactor.core.scheduler.Schedulers;
import reactor.util.function.Tuple2;
import reactor.util.function.Tuples;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import static com.appsmith.external.git.constants.GitConstants.PAGE_LIST;
import static java.lang.Boolean.TRUE;
@ -212,9 +216,17 @@ public class AutoCommitEventHandlerCEImpl implements AutoCommitEventHandlerCE {
ApplicationJson to file system conversion will use this field to decide
which pages need to be written back to file system.
*/
Set<String> pageNamesSet = new HashSet<>(updatedPageNamesList);
Set<String> pageNamesSet =
updatedPageNamesList.stream().map(Tuple2::getT1).collect(Collectors.toSet());
Set<String> pageIdentifiersSet =
updatedPageNamesList.stream().map(Tuple2::getT2).collect(Collectors.toSet());
ModifiedResources modifiedResources = new ModifiedResources();
modifiedResources.putResource(PAGE_LIST, pageNamesSet);
modifiedResources
.getModifiedResourceIdentifiers()
.get(GitResourceType.CONTEXT_CONFIG)
.addAll(pageIdentifiersSet);
modifiedResources.setAllModified(true);
applicationJson.setModifiedResources(modifiedResources);
return applicationJson;
@ -236,7 +248,7 @@ public class AutoCommitEventHandlerCEImpl implements AutoCommitEventHandlerCE {
* @param latestSchemaVersion latest dsl schema version obtained from RTS
* @return list of names of the pages that have been migrated.
*/
private Mono<List<String>> migratePageDsl(List<NewPage> newPageList, Integer latestSchemaVersion) {
private Mono<List<Tuple2<String, String>>> migratePageDsl(List<NewPage> newPageList, Integer latestSchemaVersion) {
return Flux.fromIterable(newPageList)
.filter(newPage -> {
// filter the pages which have unpublished page with layouts and where dsl version is not latest
@ -249,8 +261,8 @@ public class AutoCommitEventHandlerCEImpl implements AutoCommitEventHandlerCE {
}
return false;
})
.map(NewPage::getUnpublishedPage)
.flatMap(pageDTO -> {
.flatMap(newPage -> {
PageDTO pageDTO = newPage.getUnpublishedPage();
Layout layout = pageDTO.getLayouts().get(0);
return dslMigrationUtils
.migratePageDsl(layout.getDsl())
@ -258,7 +270,7 @@ public class AutoCommitEventHandlerCEImpl implements AutoCommitEventHandlerCE {
layout.setDsl(migratedDsl);
return migratedDsl;
})
.thenReturn(pageDTO.getName());
.thenReturn(Tuples.of(pageDTO.getName(), newPage.getGitSyncId()));
})
.collectList();
}

7
app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/CommonGitFileUtils.java
View File

@ -10,6 +10,7 @@ import com.appsmith.server.migrations.JsonSchemaVersions;
import com.appsmith.server.newactions.base.NewActionService;
import com.appsmith.server.services.AnalyticsService;
import com.appsmith.server.services.SessionUserService;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.annotation.Import;
import org.springframework.stereotype.Component;
@ -27,7 +28,8 @@ public class CommonGitFileUtils extends CommonGitFileUtilsCE {
SessionUserService sessionUserService,
NewActionService newActionService,
ActionCollectionService actionCollectionService,
JsonSchemaVersions jsonSchemaVersions) {
JsonSchemaVersions jsonSchemaVersions,
ObjectMapper objectMapper) {
super(
applicationGitFileUtils,
fileUtils,
@ -36,6 +38,7 @@ public class CommonGitFileUtils extends CommonGitFileUtilsCE {
sessionUserService,
newActionService,
actionCollectionService,
jsonSchemaVersions);
jsonSchemaVersions,
objectMapper);
}
}

4
app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/ce/ArtifactGitFileUtilsCE.java
View File

@ -13,6 +13,8 @@ public interface ArtifactGitFileUtilsCE<T extends ArtifactGitReference> {
T createArtifactReferenceObject();
ArtifactExchangeJson createArtifactExchangeJsonObject();
void setArtifactDependentResources(ArtifactExchangeJson artifactExchangeJson, GitResourceMap gitResourceMap);
Mono<ArtifactExchangeJson> reconstructArtifactExchangeJsonFromFilesInRepository(
@ -24,4 +26,6 @@ public interface ArtifactGitFileUtilsCE<T extends ArtifactGitReference> {
Map<String, String> getConstantsMap();
Path getRepoSuffixPath(String workspaceId, String artifactId, String repoName, @NonNull String... args);
void setArtifactDependentPropertiesInJson(GitResourceMap gitResourceMap, ArtifactExchangeJson artifactExchangeJson);
}

276
app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/ce/CommonGitFileUtilsCE.java
View File

@ -7,6 +7,8 @@ import com.appsmith.external.git.models.GitResourceMap;
import com.appsmith.external.git.models.GitResourceType;
import com.appsmith.external.git.operations.FileOperations;
import com.appsmith.external.helpers.Stopwatch;
import com.appsmith.external.models.ActionConfiguration;
import com.appsmith.external.models.ActionDTO;
import com.appsmith.external.models.ApplicationGitReference;
import com.appsmith.external.models.ArtifactGitReference;
import com.appsmith.external.models.BaseDomain;
@ -22,6 +24,7 @@ import com.appsmith.server.domains.CustomJSLib;
import com.appsmith.server.domains.GitArtifactMetadata;
import com.appsmith.server.domains.NewAction;
import com.appsmith.server.domains.Theme;
import com.appsmith.server.dtos.ActionCollectionDTO;
import com.appsmith.server.dtos.ApplicationJson;
import com.appsmith.server.dtos.ArtifactExchangeJson;
import com.appsmith.server.dtos.PageDTO;
@ -32,10 +35,12 @@ import com.appsmith.server.migrations.JsonSchemaVersions;
import com.appsmith.server.newactions.base.NewActionService;
import com.appsmith.server.services.AnalyticsService;
import com.appsmith.server.services.SessionUserService;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.MapperFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.gson.Gson;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.json.JSONObject;
@ -52,19 +57,30 @@ import java.nio.file.Paths;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collector;
import java.util.stream.Collectors;
import static com.appsmith.external.git.constants.ce.GitConstantsCE.GitCommandConstantsCE.CHECKOUT_BRANCH;
import static com.appsmith.external.git.constants.ce.GitConstantsCE.RECONSTRUCT_PAGE;
import static com.appsmith.git.constants.CommonConstants.CLIENT_SCHEMA_VERSION;
import static com.appsmith.git.constants.CommonConstants.DELIMITER_PATH;
import static com.appsmith.git.constants.CommonConstants.FILE_FORMAT_VERSION;
import static com.appsmith.git.constants.CommonConstants.JSON_EXTENSION;
import static com.appsmith.git.constants.CommonConstants.JS_EXTENSION;
import static com.appsmith.git.constants.CommonConstants.METADATA;
import static com.appsmith.git.constants.CommonConstants.SERVER_SCHEMA_VERSION;
import static com.appsmith.git.constants.CommonConstants.TEXT_FILE_EXTENSION;
import static com.appsmith.git.constants.CommonConstants.THEME;
import static com.appsmith.git.constants.ce.GitDirectoriesCE.ACTION_COLLECTION_DIRECTORY;
import static com.appsmith.git.constants.ce.GitDirectoriesCE.ACTION_DIRECTORY;
import static com.appsmith.git.constants.ce.GitDirectoriesCE.DATASOURCE_DIRECTORY;
import static com.appsmith.git.constants.ce.GitDirectoriesCE.JS_LIB_DIRECTORY;
import static com.appsmith.git.constants.ce.GitDirectoriesCE.PAGE_DIRECTORY;
import static com.appsmith.git.files.FileUtilsCEImpl.getJsLibFileName;
import static org.springframework.util.StringUtils.hasText;
@Slf4j
@RequiredArgsConstructor
@Component
@Import({FileUtilsImpl.class})
public class CommonGitFileUtilsCE {
@ -83,6 +99,28 @@ public class CommonGitFileUtilsCE {
public final int INDEX_LOCK_FILE_STALE_TIME = 300;
private final JsonSchemaVersions jsonSchemaVersions;
protected final ObjectMapper objectMapper;
public CommonGitFileUtilsCE(
ArtifactGitFileUtils<ApplicationGitReference> applicationGitFileUtils,
FileInterface fileUtils,
FileOperations fileOperations,
AnalyticsService analyticsService,
SessionUserService sessionUserService,
NewActionService newActionService,
ActionCollectionService actionCollectionService,
JsonSchemaVersions jsonSchemaVersions,
ObjectMapper objectMapper) {
this.applicationGitFileUtils = applicationGitFileUtils;
this.fileUtils = fileUtils;
this.fileOperations = fileOperations;
this.analyticsService = analyticsService;
this.sessionUserService = sessionUserService;
this.newActionService = newActionService;
this.actionCollectionService = actionCollectionService;
this.jsonSchemaVersions = jsonSchemaVersions;
this.objectMapper = objectMapper.copy().disable(MapperFeature.USE_ANNOTATIONS);
}
private ArtifactGitFileUtils<?> getArtifactBasedFileHelper(ArtifactType artifactType) {
if (ArtifactType.APPLICATION.equals(artifactType)) {
@ -120,6 +158,19 @@ public class CommonGitFileUtilsCE {
}
}
public Mono<Path> saveArtifactToLocalRepoNew(
Path baseRepoSuffix, ArtifactExchangeJson artifactExchangeJson, String branchName)
throws IOException, GitAPIException {
// this should come from the specific files
GitResourceMap gitResourceMap = createGitResourceMap(artifactExchangeJson);
// Save application to git repo
return fileUtils
.saveArtifactToGitRepo(baseRepoSuffix, gitResourceMap, branchName)
.subscribeOn(Schedulers.boundedElastic());
}
public Mono<Path> saveArtifactToLocalRepoWithAnalytics(
Path baseRepoSuffix, ArtifactExchangeJson artifactExchangeJson, String branchName) {
@ -216,8 +267,9 @@ public class CommonGitFileUtilsCE {
if (datasourceList != null) {
datasourceList.forEach(datasource -> {
removeUnwantedFieldsFromDatasource(datasource);
final String filePath = DATASOURCE_DIRECTORY + DELIMITER_PATH + datasource.getName() + JSON_EXTENSION;
GitResourceIdentity identity =
new GitResourceIdentity(GitResourceType.DATASOURCE_CONFIG, datasource.getGitSyncId());
new GitResourceIdentity(GitResourceType.DATASOURCE_CONFIG, datasource.getGitSyncId(), filePath);
resourceMap.put(identity, datasource);
});
}
@ -230,7 +282,8 @@ public class CommonGitFileUtilsCE {
artifactExchangeJson.setThemes(theme, null);
// Remove internal fields from the themes
removeUnwantedFieldsFromBaseDomain(theme);
GitResourceIdentity identity = new GitResourceIdentity(GitResourceType.ROOT_CONFIG, THEME + JSON_EXTENSION);
final String filePath = THEME + JSON_EXTENSION;
GitResourceIdentity identity = new GitResourceIdentity(GitResourceType.ROOT_CONFIG, filePath, filePath);
resourceMap.put(identity, theme);
}
@ -240,7 +293,9 @@ public class CommonGitFileUtilsCE {
customJSLibList.forEach(jsLib -> {
removeUnwantedFieldsFromBaseDomain(jsLib);
String jsLibFileName = getJsLibFileName(jsLib.getUidString());
GitResourceIdentity identity = new GitResourceIdentity(GitResourceType.JSLIB_CONFIG, jsLibFileName);
final String filePath = JS_LIB_DIRECTORY + DELIMITER_PATH + jsLibFileName + JSON_EXTENSION;
GitResourceIdentity identity =
new GitResourceIdentity(GitResourceType.JSLIB_CONFIG, jsLibFileName, filePath);
resourceMap.put(identity, jsLib);
});
}
@ -265,58 +320,47 @@ public class CommonGitFileUtilsCE {
.peek(newAction -> newActionService.generateActionByViewMode(newAction, false))
.forEach(newAction -> {
removeUnwantedFieldFromAction(newAction);
String body = newAction.getUnpublishedAction().getActionConfiguration() != null
&& newAction
.getUnpublishedAction()
.getActionConfiguration()
.getBody()
!= null
? newAction
.getUnpublishedAction()
.getActionConfiguration()
.getBody()
: "";
ActionDTO action = newAction.getUnpublishedAction();
final String actionFileName = action.getValidName().replace(".", "-");
final String filePathPrefix = PAGE_DIRECTORY
+ DELIMITER_PATH
+ action.calculateContextId()
+ DELIMITER_PATH
+ ACTION_DIRECTORY
+ DELIMITER_PATH
+ actionFileName
+ DELIMITER_PATH;
String body = action.getActionConfiguration() != null
&& action.getActionConfiguration().getBody() != null
? action.getActionConfiguration().getBody()
: null;
// This is a special case where we are handling REMOTE type plugins based actions such as Twilio
// The user configured values are stored in an attribute called formData which is a map unlike the
// body
if (PluginType.REMOTE.equals(newAction.getPluginType())
&& newAction.getUnpublishedAction().getActionConfiguration() != null
&& newAction
.getUnpublishedAction()
.getActionConfiguration()
.getFormData()
!= null) {
body = new Gson()
.toJson(
newAction
.getUnpublishedAction()
.getActionConfiguration()
.getFormData(),
Map.class);
newAction
.getUnpublishedAction()
.getActionConfiguration()
.setFormData(null);
&& action.getActionConfiguration() != null
&& action.getActionConfiguration().getFormData() != null) {
body = new Gson().toJson(action.getActionConfiguration().getFormData(), Map.class);
action.getActionConfiguration().setFormData(null);
}
// This is a special case where we are handling JS actions as we don't want to commit the body of JS
// actions
if (PluginType.JS.equals(newAction.getPluginType())) {
if (newAction.getUnpublishedAction().getActionConfiguration() != null) {
newAction
.getUnpublishedAction()
.getActionConfiguration()
.setBody(null);
newAction.getUnpublishedAction().setJsonPathKeys(null);
if (action.getActionConfiguration() != null) {
action.getActionConfiguration().setBody(null);
action.setJsonPathKeys(null);
}
} else {
} else if (body != null) {
// For the regular actions we save the body field to git repo
final String filePath = filePathPrefix + actionFileName + TEXT_FILE_EXTENSION;
GitResourceIdentity actionDataIdentity =
new GitResourceIdentity(GitResourceType.QUERY_DATA, newAction.getGitSyncId());
new GitResourceIdentity(GitResourceType.QUERY_DATA, newAction.getGitSyncId(), filePath);
resourceMap.put(actionDataIdentity, body);
}
final String filePath = filePathPrefix + METADATA + JSON_EXTENSION;
GitResourceIdentity actionConfigIdentity =
new GitResourceIdentity(GitResourceType.QUERY_CONFIG, newAction.getGitSyncId());
new GitResourceIdentity(GitResourceType.QUERY_CONFIG, newAction.getGitSyncId(), filePath);
resourceMap.put(actionConfigIdentity, newAction);
});
}
@ -335,18 +379,29 @@ public class CommonGitFileUtilsCE {
actionCollectionService.generateActionCollectionByViewMode(actionCollection, false))
.forEach(actionCollection -> {
removeUnwantedFieldFromActionCollection(actionCollection);
String body = actionCollection.getUnpublishedCollection().getBody() != null
? actionCollection.getUnpublishedCollection().getBody()
: "";
actionCollection.getUnpublishedCollection().setBody(null);
ActionCollectionDTO collection = actionCollection.getUnpublishedCollection();
final String filePathPrefix = PAGE_DIRECTORY
+ DELIMITER_PATH
+ collection.calculateContextId()
+ DELIMITER_PATH
+ ACTION_COLLECTION_DIRECTORY
+ DELIMITER_PATH
+ collection.getName()
+ DELIMITER_PATH;
String body = collection.getBody();
collection.setBody(null);
GitResourceIdentity collectionConfigIdentity =
new GitResourceIdentity(GitResourceType.JSOBJECT_CONFIG, actionCollection.getGitSyncId());
String configFilePath = filePathPrefix + METADATA + JSON_EXTENSION;
GitResourceIdentity collectionConfigIdentity = new GitResourceIdentity(
GitResourceType.JSOBJECT_CONFIG, actionCollection.getGitSyncId(), configFilePath);
resourceMap.put(collectionConfigIdentity, actionCollection);
GitResourceIdentity collectionDataIdentity =
new GitResourceIdentity(GitResourceType.JSOBJECT_DATA, actionCollection.getGitSyncId());
resourceMap.put(collectionDataIdentity, body);
if (body != null) {
String dataFilePath = filePathPrefix + collection.getName() + JS_EXTENSION;
GitResourceIdentity collectionDataIdentity = new GitResourceIdentity(
GitResourceType.JSOBJECT_DATA, actionCollection.getGitSyncId(), dataFilePath);
resourceMap.put(collectionDataIdentity, body);
}
});
}
@ -366,6 +421,125 @@ public class CommonGitFileUtilsCE {
removeUnwantedFieldsFromBaseDomain(actionCollection);
}
public ArtifactExchangeJson createArtifactExchangeJson(GitResourceMap gitResourceMap, ArtifactType artifactType) {
ArtifactGitFileUtils<?> artifactGitFileUtils = getArtifactBasedFileHelper(artifactType);
ArtifactExchangeJson artifactExchangeJson = artifactGitFileUtils.createArtifactExchangeJsonObject();
artifactGitFileUtils.setArtifactDependentPropertiesInJson(gitResourceMap, artifactExchangeJson);
setArtifactIndependentPropertiesInJson(gitResourceMap, artifactExchangeJson);
return artifactExchangeJson;
}
protected void setArtifactIndependentPropertiesInJson(
GitResourceMap gitResourceMap, ArtifactExchangeJson artifactExchangeJson) {
Map<GitResourceIdentity, Object> resourceMap = gitResourceMap.getGitResourceMap();
// datasources
List<DatasourceStorage> datasourceList = resourceMap.entrySet().stream()
.filter(entry -> {
GitResourceIdentity key = entry.getKey();
return GitResourceType.DATASOURCE_CONFIG.equals(key.getResourceType());
})
.map(Map.Entry::getValue)
.map(value -> objectMapper.convertValue(value, DatasourceStorage.class))
.collect(Collectors.toList());
artifactExchangeJson.setDatasourceList(datasourceList);
// themes
final String themeFilePath = THEME + JSON_EXTENSION;
GitResourceIdentity themeIdentity =
new GitResourceIdentity(GitResourceType.ROOT_CONFIG, themeFilePath, themeFilePath);
Object themeObject = resourceMap.get(themeIdentity);
Theme theme = objectMapper.convertValue(themeObject, Theme.class);
artifactExchangeJson.setThemes(theme, null);
// custom js libs
List<CustomJSLib> jsLibList = resourceMap.entrySet().stream()
.filter(entry -> {
GitResourceIdentity key = entry.getKey();
return GitResourceType.JSLIB_CONFIG.equals(key.getResourceType());
})
.map(Map.Entry::getValue)
.map(value -> objectMapper.convertValue(value, CustomJSLib.class))
.collect(Collectors.toList());
artifactExchangeJson.setCustomJSLibList(jsLibList);
// actions
final Set<GitResourceType> queryTypes = Set.of(GitResourceType.QUERY_CONFIG, GitResourceType.QUERY_DATA);
List<NewAction> actionList = resourceMap.entrySet().stream()
.filter(entry -> {
GitResourceIdentity key = entry.getKey();
return queryTypes.contains(key.getResourceType());
})
.collect(collectByGitSyncId())
.entrySet()
.parallelStream()
.map(entry -> {
Object config = entry.getValue().get(GitResourceType.QUERY_CONFIG);
NewAction newAction = objectMapper.convertValue(config, NewAction.class);
ActionDTO actionDTO = newAction.getUnpublishedAction();
Object data = entry.getValue().get(GitResourceType.QUERY_DATA);
ActionConfiguration actionConfiguration = actionDTO.getActionConfiguration();
if (actionConfiguration == null) {
// This shouldn't happen but safe-guarding just in case
actionConfiguration = new ActionConfiguration();
}
if (PluginType.REMOTE.equals(newAction.getPluginType())) {
Map<String, Object> formData = objectMapper.convertValue(data, new TypeReference<>() {});
actionConfiguration.setFormData(formData);
} else if (data != null) {
String body = String.valueOf(data);
actionConfiguration.setBody(body);
}
return newAction;
})
.collect(Collectors.toList());
artifactExchangeJson.setActionList(actionList);
// action collections
final Set<GitResourceType> jsObjectTypes =
Set.of(GitResourceType.JSOBJECT_CONFIG, GitResourceType.JSOBJECT_DATA);
List<ActionCollection> collectionList = resourceMap.entrySet().stream()
.filter(entry -> {
GitResourceIdentity key = entry.getKey();
return jsObjectTypes.contains(key.getResourceType());
})
.collect(collectByGitSyncId())
.entrySet()
.parallelStream()
.map(entry -> {
Object config = entry.getValue().get(GitResourceType.JSOBJECT_CONFIG);
ActionCollection actionCollection = objectMapper.convertValue(config, ActionCollection.class);
Object data = entry.getValue().get(GitResourceType.JSOBJECT_DATA);
String body = String.valueOf(data);
actionCollection.getUnpublishedCollection().setBody(body);
return actionCollection;
})
.collect(Collectors.toList());
artifactExchangeJson.setActionCollectionList(collectionList);
}
private Collector<Map.Entry<GitResourceIdentity, Object>, ?, Map<String, HashMap<GitResourceType, Object>>>
collectByGitSyncId() {
return Collectors.toMap(
entry -> entry.getKey().getResourceIdentifier(),
entry -> {
HashMap<GitResourceType, Object> map = new HashMap<>();
map.put(entry.getKey().getResourceType(), entry.getValue());
return map;
},
(x, y) -> {
x.putAll(y);
return x;
});
}
private void setDatasourcesInArtifactReference(
ArtifactExchangeJson artifactExchangeJson, ArtifactGitReference artifactGitReference) {
Map<String, Object> resourceMap = new HashMap<>();

6
app/server/appsmith-server/src/main/java/com/appsmith/server/jslibs/exportable/CustomJSLibExportableServiceCEImpl.java
View File

@ -1,5 +1,6 @@
package com.appsmith.server.jslibs.exportable;
import com.appsmith.external.git.models.GitResourceType;
import com.appsmith.server.constants.FieldName;
import com.appsmith.server.domains.Application;
import com.appsmith.server.domains.Artifact;
@ -77,6 +78,11 @@ public class CustomJSLibExportableServiceCEImpl implements ExportableServiceCE<C
artifactExchangeJson
.getModifiedResources()
.putResource(FieldName.CUSTOM_JS_LIB_LIST, updatedCustomJSLibSet);
artifactExchangeJson
.getModifiedResources()
.getModifiedResourceIdentifiers()
.get(GitResourceType.JSLIB_CONFIG)
.addAll(updatedCustomJSLibSet);
/**
* Previously it was a Set and as Set is an unordered collection of elements that

17
app/server/appsmith-server/src/main/java/com/appsmith/server/newactions/exportable/NewActionExportableServiceCEImpl.java
View File

@ -1,5 +1,6 @@
package com.appsmith.server.newactions.exportable;
import com.appsmith.external.git.models.GitResourceType;
import com.appsmith.external.models.ActionDTO;
import com.appsmith.server.acl.AclPermission;
import com.appsmith.server.constants.FieldName;
@ -70,6 +71,7 @@ public class NewActionExportableServiceCEImpl implements ExportableServiceCE<New
List<NewAction> actionList = tuple.getT1();
Set<String> dbNamesUsedInActions = tuple.getT2();
Set<String> updatedActionSet = new HashSet<>();
Set<String> updatedIdentities = new HashSet<>();
actionList.forEach(newAction -> {
ActionDTO unpublishedActionDTO = newAction.getUnpublishedAction();
ActionDTO publishedActionDTO = newAction.getPublishedAction();
@ -85,9 +87,12 @@ public class NewActionExportableServiceCEImpl implements ExportableServiceCE<New
actionDTO,
exportingMetaDTO.getArtifactLastCommittedAt());
String contextListPath = artifactBasedExportableService.getContextListPath();
boolean isContextUpdated = ImportExportUtils.isContextNameInUpdatedList(
artifactExchangeJson, contextNameAtIdReference, contextListPath);
String contextGitSyncId = mappedExportableResourcesDTO
.getContextNameToGitSyncIdMap()
.get(contextNameAtIdReference);
boolean isContextUpdated = artifactExchangeJson
.getModifiedResources()
.isResourceUpdatedNew(GitResourceType.CONTEXT_CONFIG, contextGitSyncId);
Instant newActionUpdatedAt = newAction.getUpdatedAt();
boolean isNewActionUpdated = exportingMetaDTO.isClientSchemaMigrated()
|| exportingMetaDTO.isServerSchemaMigrated()
@ -98,10 +103,16 @@ public class NewActionExportableServiceCEImpl implements ExportableServiceCE<New
|| exportingMetaDTO.getArtifactLastCommittedAt().isBefore(newActionUpdatedAt);
if (isNewActionUpdated && newActionName != null) {
updatedActionSet.add(newActionName);
updatedIdentities.add(newAction.getGitSyncId());
}
newAction.sanitiseToExportDBObject();
});
artifactExchangeJson.getModifiedResources().putResource(FieldName.ACTION_LIST, updatedActionSet);
artifactExchangeJson
.getModifiedResources()
.getModifiedResourceIdentifiers()
.get(GitResourceType.QUERY_CONFIG)
.addAll(updatedIdentities);
artifactExchangeJson.setActionList(actionList);
// This is where we're removing global datasources that are unused in this application

13
app/server/appsmith-server/src/main/java/com/appsmith/server/newpages/exportable/NewPageExportableServiceCEImpl.java
View File

@ -1,5 +1,6 @@
package com.appsmith.server.newpages.exportable;
import com.appsmith.external.git.models.GitResourceType;
import com.appsmith.server.acl.AclPermission;
import com.appsmith.server.constants.FieldName;
import com.appsmith.server.constants.SerialiseArtifactObjective;
@ -66,7 +67,8 @@ public class NewPageExportableServiceCEImpl implements ExportableServiceCE<NewPa
// Extract mongoEscapedWidgets from pages and save it to applicationJson object as this
// field is JsonIgnored. Also remove any ids those are present in the page objects
Set<String> updatedPageSet = new HashSet<String>();
Set<String> updatedPageSet = new HashSet<>();
Set<String> updatedIdentities = new HashSet<>();
// check the application object for the page reference in the page list
// Exclude the deleted pages that are present in view mode because the app is not
@ -79,6 +81,9 @@ public class NewPageExportableServiceCEImpl implements ExportableServiceCE<NewPa
.put(
newPage.getId() + EDIT,
newPage.getUnpublishedPage().getName());
mappedExportableResourcesDTO
.getContextNameToGitSyncIdMap()
.put(newPage.getUnpublishedPage().getName(), newPage.getGitSyncId());
PageDTO unpublishedPageDTO = newPage.getUnpublishedPage();
if (!CollectionUtils.isEmpty(unpublishedPageDTO.getLayouts())) {
unpublishedPageDTO.getLayouts().forEach(layout -> {
@ -114,11 +119,17 @@ public class NewPageExportableServiceCEImpl implements ExportableServiceCE<NewPa
: null;
if (isNewPageUpdated && newPageName != null) {
updatedPageSet.add(newPageName);
updatedIdentities.add(newPage.getGitSyncId());
}
newPage.sanitiseToExportDBObject();
});
applicationJson.setPageList(newPageList);
applicationJson.getModifiedResources().putResource(FieldName.PAGE_LIST, updatedPageSet);
applicationJson
.getModifiedResources()
.getModifiedResourceIdentifiers()
.get(GitResourceType.CONTEXT_CONFIG)
.addAll(updatedIdentities);
return newPageList;
})

1
app/server/appsmith-server/src/test/java/com/appsmith/server/git/autocommit/AutoCommitEventHandlerImplTest.java
View File

@ -161,6 +161,7 @@ public class AutoCommitEventHandlerImplTest {
pageDTO.setName("Page 1");
pageDTO.setLayouts(List.of(layout));
NewPage newPage = new NewPage();
newPage.setGitSyncId("p1");
newPage.setUnpublishedPage(pageDTO);
ApplicationJson applicationJson = new ApplicationJson();
applicationJson.setPageList(List.of(newPage));

64
app/server/appsmith-server/src/test/java/com/appsmith/server/git/resourcemap/ExchangeJsonConversionTests.java
View File

@ -1,18 +1,26 @@
package com.appsmith.server.git.resourcemap;
import com.appsmith.external.git.GitExecutor;
import com.appsmith.external.git.models.GitResourceIdentity;
import com.appsmith.external.git.models.GitResourceMap;
import com.appsmith.server.constants.ArtifactType;
import com.appsmith.server.dtos.ArtifactExchangeJson;
import com.appsmith.server.git.resourcemap.templates.contexts.ExchangeJsonContext;
import com.appsmith.server.git.resourcemap.templates.providers.ExchangeJsonTestTemplateProvider;
import com.appsmith.server.helpers.CommonGitFileUtils;
import com.appsmith.server.migrations.JsonSchemaMigration;
import com.google.gson.Gson;
import com.fasterxml.jackson.databind.MapperFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.commons.io.FileUtils;
import org.assertj.core.api.Assertions;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.junit.jupiter.api.TestInstance;
import org.junit.jupiter.api.TestTemplate;
import org.junit.jupiter.api.extension.RegisterExtension;
import org.mockito.Mockito;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.mock.mockito.SpyBean;
import org.springframework.core.io.ClassPathResource;
import reactor.core.publisher.Mono;
import reactor.test.StepVerifier;
@ -20,6 +28,8 @@ import reactor.util.function.Tuple2;
import java.io.IOException;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Map;
import static org.assertj.core.api.Assertions.assertThat;
@ -33,7 +43,7 @@ public class ExchangeJsonConversionTests {
public ExchangeJsonTestTemplateProvider templateProvider;
@Autowired
Gson gson;
ObjectMapper objectMapper;
@Autowired
JsonSchemaMigration jsonSchemaMigration;
@ -41,16 +51,20 @@ public class ExchangeJsonConversionTests {
@Autowired
CommonGitFileUtils commonGitFileUtils;
@SpyBean
GitExecutor gitExecutor;
@TestTemplate
public void testConvertArtifactJsonToGitResourceMap_whenArtifactIsFullyPopulated_returnsCorrespondingResourceMap(
ExchangeJsonContext context) throws IOException {
Mono<? extends ArtifactExchangeJson> artifactJsonMono =
createArtifactJson(context).cache();
Mono<? extends ArtifactExchangeJson> artifactJsonMono = createArtifactJson(context);
Mono<? extends ArtifactExchangeJson> artifactJsonCloneMono = createArtifactJson(context);
Mono<? extends Tuple2<GitResourceMap, ? extends ArtifactExchangeJson>> gitResourceMapAndArtifactJsonMono =
artifactJsonMono
.map(artifactJson -> commonGitFileUtils.createGitResourceMap(artifactJson))
.zipWith(artifactJsonMono);
.zipWith(artifactJsonCloneMono);
StepVerifier.create(gitResourceMapAndArtifactJsonMono)
.assertNext(tuple2 -> {
@ -87,8 +101,46 @@ public class ExchangeJsonConversionTests {
Class<? extends ArtifactExchangeJson> exchangeJsonType = context.getArtifactExchangeJsonType();
ArtifactExchangeJson artifactExchangeJson = gson.fromJson(artifactJson, exchangeJsonType);
ArtifactExchangeJson artifactExchangeJson =
objectMapper.copy().disable(MapperFeature.USE_ANNOTATIONS).readValue(artifactJson, exchangeJsonType);
return jsonSchemaMigration.migrateArtifactExchangeJsonToLatestSchema(artifactExchangeJson, null, null);
}
@TestTemplate
public void testConvertGitResourceMapToArtifactExchangeJson_whenArtifactIsFullyPopulated_returnsCorrespondingJson(
ExchangeJsonContext context) throws IOException {
ArtifactExchangeJson originalArtifactJson = createArtifactJson(context).block();
GitResourceMap gitResourceMap = commonGitFileUtils.createGitResourceMap(originalArtifactJson);
ArtifactExchangeJson artifactExchangeJson =
commonGitFileUtils.createArtifactExchangeJson(gitResourceMap, ArtifactType.APPLICATION);
assertThat(artifactExchangeJson).isNotNull();
templateProvider.assertResourceComparisons(originalArtifactJson, artifactExchangeJson);
}
@TestTemplate
public void testSerializeArtifactExchangeJson_whenArtifactIsFullyPopulated_returnsCorrespondingBaseRepoPath(
ExchangeJsonContext context) throws IOException, GitAPIException {
ArtifactExchangeJson originalArtifactJson = createArtifactJson(context).block();
Mockito.doReturn(Mono.just(true)).when(gitExecutor).resetToLastCommit(Mockito.any(), Mockito.anyString());
Files.createDirectories(Path.of("./container-volumes/git-storage/test123"));
Mono<Path> responseMono =
commonGitFileUtils.saveArtifactToLocalRepoNew(Path.of("test123"), originalArtifactJson, "irrelevant");
StepVerifier.create(responseMono)
.assertNext(response -> {
Assertions.assertThat(response).isNotNull();
})
.verifyComplete();
FileUtils.deleteDirectory(
Path.of("./container-volumes/git-storage/test123").toFile());
}
}

66
app/server/appsmith-server/src/test/java/com/appsmith/server/git/resourcemap/templates/providers/ce/ExchangeJsonTestTemplateProviderCE.java
View File

@ -2,7 +2,12 @@ package com.appsmith.server.git.resourcemap.templates.providers.ce;
import com.appsmith.external.git.models.GitResourceIdentity;
import com.appsmith.external.git.models.GitResourceType;
import com.appsmith.external.models.DatasourceStorage;
import com.appsmith.external.models.PluginType;
import com.appsmith.server.domains.ActionCollection;
import com.appsmith.server.domains.Context;
import com.appsmith.server.domains.CustomJSLib;
import com.appsmith.server.domains.NewAction;
import com.appsmith.server.dtos.ApplicationJson;
import com.appsmith.server.dtos.ArtifactExchangeJson;
import com.appsmith.server.git.resourcemap.templates.contexts.ExchangeJsonContext;
@ -27,7 +32,7 @@ public class ExchangeJsonTestTemplateProviderCE implements TestTemplateInvocatio
@Override
public Stream<TestTemplateInvocationContext> provideTestTemplateInvocationContexts(
ExtensionContext extensionContext) {
ExchangeJsonContext context = new ExchangeJsonContext("valid-application.json", ApplicationJson.class, 23);
ExchangeJsonContext context = new ExchangeJsonContext("valid-application.json", ApplicationJson.class, 22);
return Stream.of(context);
}
@ -68,7 +73,14 @@ public class ExchangeJsonTestTemplateProviderCE implements TestTemplateInvocatio
List<Object> jsObjectDataResources = getResourceListByType(resourceMap, GitResourceType.JSOBJECT_DATA);
long resourceMapJsObjectDataCount = jsObjectDataResources.size();
assertThat(resourceMapJsObjectDataCount).isEqualTo(jsonJsObjectCount);
int jsonJsObjectDataCount = exchangeJson.getActionCollectionList() != null
? exchangeJson.getActionCollectionList().parallelStream()
.filter(collection ->
collection.getUnpublishedCollection().getBody() != null)
.collect(Collectors.toList())
.size()
: 0;
assertThat(resourceMapJsObjectDataCount).isEqualTo(jsonJsObjectDataCount);
List<Object> actionConfigResources = getResourceListByType(resourceMap, GitResourceType.QUERY_CONFIG);
long resourceMapActionConfigCount = actionConfigResources.size();
@ -82,7 +94,17 @@ public class ExchangeJsonTestTemplateProviderCE implements TestTemplateInvocatio
long jsonActionDataCount = 0;
if (exchangeJson.getActionList() != null) {
jsonActionDataCount = exchangeJson.getActionList().stream()
.filter(action -> !PluginType.JS.equals(action.getPluginType()))
.filter(action -> !PluginType.JS.equals(action.getPluginType())
&& action.getUnpublishedAction().getActionConfiguration() != null
&& !(action.getUnpublishedAction()
.getActionConfiguration()
.getBody()
== null
|| (action.getPluginType().equals(PluginType.REMOTE)
&& action.getUnpublishedAction()
.getActionConfiguration()
.getFormData()
== null)))
.count();
}
assertThat(resourceMapActionDataCount).isEqualTo(jsonActionDataCount);
@ -111,4 +133,42 @@ public class ExchangeJsonTestTemplateProviderCE implements TestTemplateInvocatio
.map(Map.Entry::getValue)
.collect(Collectors.toList());
}
public void assertResourceComparisons(
ArtifactExchangeJson originalExchangeJson, ArtifactExchangeJson convertedExchangeJson) {
List<DatasourceStorage> datasourceResources = convertedExchangeJson.getDatasourceList();
long convertedDatasourceCount = datasourceResources.size();
int jsonDatasourceCount = originalExchangeJson.getDatasourceList() != null
? originalExchangeJson.getDatasourceList().size()
: 0;
assertThat(convertedDatasourceCount).isEqualTo(jsonDatasourceCount);
List<CustomJSLib> jsLibResources = convertedExchangeJson.getCustomJSLibList();
long convertedJsLibCount = jsLibResources.size();
int jsonJsLibCount = originalExchangeJson.getCustomJSLibList() != null
? originalExchangeJson.getCustomJSLibList().size()
: 0;
assertThat(convertedJsLibCount).isEqualTo(jsonJsLibCount);
List<? extends Context> contextResources = convertedExchangeJson.getContextList();
long convertedContextCount = contextResources.size();
int jsonContextCount = originalExchangeJson.getContextList() != null
? originalExchangeJson.getContextList().size()
: 0;
assertThat(convertedContextCount).isEqualTo(jsonContextCount);
List<ActionCollection> jsObjectResources = convertedExchangeJson.getActionCollectionList();
long convertedJsObjectCount = jsObjectResources.size();
int jsonJsObjectCount = originalExchangeJson.getActionCollectionList() != null
? originalExchangeJson.getActionCollectionList().size()
: 0;
assertThat(convertedJsObjectCount).isEqualTo(jsonJsObjectCount);
List<NewAction> actionResources = convertedExchangeJson.getActionList();
long convertedActionCount = actionResources.size();
int jsonActionCount = originalExchangeJson.getActionList() != null
? originalExchangeJson.getActionList().size()
: 0;
assertThat(convertedActionCount).isEqualTo(jsonActionCount);
}
}