feat: Separate the js object code and metadata when used with git sync (#19156)

As per current DB structure actions are embedded inside the JSObjects.
Every new method in JSObjects results in new action getting created in
DB. Consider user updates the JSObject.myFun1 then user will have
changes at 2 locations

Queries directory under myFun1 file
Body of JSObject1
We should not commit these duplicates and JSObject body should be
committed as is. These steps surely makes it easier to resolve the
conflicts even if occurs at same line.
This commit is contained in:
Anagh Hegde 2023-01-19 20:12:22 +05:30 committed by GitHub
parent ce2b5faa92
commit 52e4d32a3f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 212 additions and 97 deletions

View File

@ -2,11 +2,13 @@ package com.appsmith.git.constants;
public class CommonConstants {
// This field will be useful when we migrate fields within JSON files (currently this will be useful for Git feature)
public static Integer fileFormatVersion = 2;
public static Integer fileFormatVersion = 3;
public static String FILE_FORMAT_VERSION = "fileFormatVersion";
public static final String CANVAS = "canvas";
public static final String APPLICATION = "application";
public static final String THEME = "theme";
public static final String METADATA = "metadata";
public static final String JSON_EXTENSION = ".json";
public static final String JS_EXTENSION = ".js";
public static final String JS_BODY = "body";
}

View File

@ -84,45 +84,67 @@ public class FileUtilsImpl implements FileInterface {
private final Scheduler scheduler = Schedulers.boundedElastic();
/**
Application will be stored in the following structure:
Application will be stored in the following structure:
For v1:
repo_name
application.json
metadata.json
datasource
datasource1Name.json
datasource2Name.json
queries (Only requirement here is the filename should be unique)
action1_page1
action2_page2
jsobjects (Only requirement here is the filename should be unique)
jsobject1_page1
jsobject2_page2
pages
page1
page2
For v1:
repo_name
application.json
metadata.json
datasource
datasource1Name.json
datasource2Name.json
queries (Only requirement here is the filename should be unique)
action1_page1
action2_page2
jsobjects (Only requirement here is the filename should be unique)
jsobject1_page1
jsobject2_page2
pages
page1
page2
For v2:
repo_name
application.json
metadata.json
theme
publishedTheme.json
editModeTheme.json
pages
page1
canvas.json
queries
Query1.json
Query2.json
jsobjects
JSObject1.json
page2
page3
datasources
datasource1.json
datasource2.json
For v2:
repo_name
application.json
metadata.json
theme
publishedTheme.json
editModeTheme.json
pages
page1
canvas.json
queries
Query1.json
Query2.json
jsobjects
JSObject1.json
page2
page3
datasources
datasource1.json
datasource2.json
For v3:
repo_name
application.json
metadata.json
theme
publishedTheme.json
editModeTheme.json
pages
page1
canvas.json
queries
Query1.json
jsobjects
JSObject1
JSObject1.js
Metadata.json
page2
page3
datasources
datasource1.json
datasource2.json
*/
@ -167,15 +189,15 @@ public class FileUtilsImpl implements FileInterface {
deleteDirectory(baseRepo.resolve(ACTION_COLLECTION_DIRECTORY));
// Save application
saveFile(applicationGitReference.getApplication(), baseRepo.resolve(CommonConstants.APPLICATION + CommonConstants.JSON_EXTENSION), gson);
saveResource(applicationGitReference.getApplication(), baseRepo.resolve(CommonConstants.APPLICATION + CommonConstants.JSON_EXTENSION), gson);
// Save application metadata
JsonObject metadata = gson.fromJson(gson.toJson(applicationGitReference.getMetadata()), JsonObject.class);
metadata.addProperty(CommonConstants.FILE_FORMAT_VERSION, CommonConstants.fileFormatVersion);
saveFile(metadata, baseRepo.resolve(CommonConstants.METADATA + CommonConstants.JSON_EXTENSION), gson);
saveResource(metadata, baseRepo.resolve(CommonConstants.METADATA + CommonConstants.JSON_EXTENSION), gson);
// Save application theme
saveFile(applicationGitReference.getTheme(), baseRepo.resolve(CommonConstants.THEME + CommonConstants.JSON_EXTENSION), gson);
saveResource(applicationGitReference.getTheme(), baseRepo.resolve(CommonConstants.THEME + CommonConstants.JSON_EXTENSION), gson);
// Save pages
Path pageDirectory = baseRepo.resolve(PAGE_DIRECTORY);
@ -187,7 +209,7 @@ public class FileUtilsImpl implements FileInterface {
Path pageSpecificDirectory = pageDirectory.resolve(pageName);
Boolean isResourceUpdated = updatedResources.get(PAGE_LIST).contains(pageName);
if(Boolean.TRUE.equals(isResourceUpdated)) {
saveFile(pageResource.getValue(), pageSpecificDirectory.resolve(CommonConstants.CANVAS + CommonConstants.JSON_EXTENSION), gson);
saveResource(pageResource.getValue(), pageSpecificDirectory.resolve(CommonConstants.CANVAS + CommonConstants.JSON_EXTENSION), gson);
}
validPages.add(pageName);
}
@ -206,7 +228,7 @@ public class FileUtilsImpl implements FileInterface {
Path jsLibSpecificFile =
jsLibDirectory.resolve(fileNameWithExtension);
if (isResourceUpdated) {
saveFile(jsLibEntry.getValue(), jsLibSpecificFile, gson);
saveResource(jsLibEntry.getValue(), jsLibSpecificFile, gson);
}
validJsLibs.add(fileNameWithExtension);
});
@ -239,7 +261,7 @@ public class FileUtilsImpl implements FileInterface {
}
validActionsMap.get(pageName).add(queryName + CommonConstants.JSON_EXTENSION);
if(Boolean.TRUE.equals(isResourceUpdated)) {
saveFile(
saveResource(
resource.getValue(),
pageSpecificDirectory.resolve(ACTION_DIRECTORY).resolve(queryName + CommonConstants.JSON_EXTENSION),
gson
@ -257,37 +279,42 @@ public class FileUtilsImpl implements FileInterface {
// Save JSObjects
for (Map.Entry<String, Object> resource : applicationGitReference.getActionCollections().entrySet()) {
// JSObjectName_pageName => nomenclature for the keys
// TODO
// JSObjectName => for app level JSObjects, this is not implemented yet
// TODO JSObjectName => for app level JSObjects, this is not implemented yet
String[] names = resource.getKey().split(NAME_SEPARATOR);
if (names.length > 1 && StringUtils.hasLength(names[1])) {
final String actionCollectionName = names[0];
final String pageName = names[1];
Path pageSpecificDirectory = pageDirectory.resolve(pageName);
Path actionCollectionSpecificDirectory = pageSpecificDirectory.resolve(ACTION_COLLECTION_DIRECTORY);
if(!validActionCollectionsMap.containsKey(pageName)) {
validActionCollectionsMap.put(pageName, new HashSet<>());
}
validActionCollectionsMap.get(pageName).add(actionCollectionName + CommonConstants.JSON_EXTENSION);
validActionCollectionsMap.get(pageName).add(actionCollectionName);
Boolean isResourceUpdated = updatedResources.get(ACTION_COLLECTION_LIST).contains(resource.getKey());
if(Boolean.TRUE.equals(isResourceUpdated)) {
saveFile(
saveActionCollection(
resource.getValue(),
pageSpecificDirectory.resolve(ACTION_COLLECTION_DIRECTORY).resolve(actionCollectionName + CommonConstants.JSON_EXTENSION),
applicationGitReference.getActionCollectionBody().get(resource.getKey()),
actionCollectionName,
actionCollectionSpecificDirectory.resolve(actionCollectionName),
gson
);
// Delete the resource from the old file structure v2
deleteFile(actionCollectionSpecificDirectory.resolve(actionCollectionName + CommonConstants.JSON_EXTENSION));
}
}
}
// Verify if the old files are deleted
validActionCollectionsMap.forEach((pageName, validActionCollectionNames) -> {
Path pageSpecificDirectory = pageDirectory.resolve(pageName);
scanAndDeleteFileForDeletedResources(validActionCollectionNames, pageSpecificDirectory.resolve(ACTION_COLLECTION_DIRECTORY));
scanAndDeleteDirectoryForDeletedResources(validActionCollectionNames, pageSpecificDirectory.resolve(ACTION_COLLECTION_DIRECTORY));
});
// Save datasources ref
for (Map.Entry<String, Object> resource : applicationGitReference.getDatasources().entrySet()) {
saveFile(resource.getValue(), baseRepo.resolve(DATASOURCE_DIRECTORY).resolve(resource.getKey() + CommonConstants.JSON_EXTENSION), gson);
saveResource(resource.getValue(), baseRepo.resolve(DATASOURCE_DIRECTORY).resolve(resource.getKey() + CommonConstants.JSON_EXTENSION), gson);
validFileNames.add(resource.getKey() + CommonConstants.JSON_EXTENSION);
}
// Scan datasource directory and delete any unwanted files if present
@ -307,19 +334,57 @@ public class FileUtilsImpl implements FileInterface {
* @param gson
* @return if the file operation is successful
*/
private boolean saveFile(Object sourceEntity, Path path, Gson gson) {
private boolean saveResource(Object sourceEntity, Path path, Gson gson) {
try {
Files.createDirectories(path.getParent());
try (BufferedWriter fileWriter = Files.newBufferedWriter(path, StandardCharsets.UTF_8)) {
gson.toJson(sourceEntity, fileWriter);
return true;
}
return writeToFile(sourceEntity, path, gson);
} catch (IOException e) {
log.debug(e.getMessage());
}
return false;
}
/**
* This method is used to write actionCollection specific resource to file system. We write the data in two steps
* 1. Actual js code
* 2. Metadata of the actionCollection
* @param sourceEntity the metadata of the action collection
* @param body actual js code written by the user
* @param resourceName name of the action collection
* @param path file path where the resource will be stored
* @param gson
* @return if the file operation is successful
*/
private boolean saveActionCollection(Object sourceEntity, String body, String resourceName, Path path, Gson gson) {
try {
Files.createDirectories(path);
// Write the js Object body to .js file to make conflict handling easier
Path bodyPath = path.resolve(resourceName + CommonConstants.JS_EXTENSION);
writeStringToFile(body, bodyPath);
// Write metadata for the jsObject
Path metadataPath = path.resolve(CommonConstants.METADATA + CommonConstants.JSON_EXTENSION);
return writeToFile(sourceEntity, metadataPath, gson);
} catch (IOException e) {
log.debug(e.getMessage());
}
return false;
}
private boolean writeStringToFile(String data, Path path) throws IOException {
try (BufferedWriter fileWriter = Files.newBufferedWriter(path, StandardCharsets.UTF_8)) {
fileWriter.write(data);
return true;
}
}
private boolean writeToFile(Object sourceEntity, Path path, Gson gson) throws IOException {
try (BufferedWriter fileWriter = Files.newBufferedWriter(path, StandardCharsets.UTF_8)) {
gson.toJson(sourceEntity, fileWriter);
return true;
}
}
/**
* This method will delete the JSON resource available in local git directory on subsequent commit made after the
* deletion of respective resource from DB
@ -417,8 +482,8 @@ public class FileUtilsImpl implements FileInterface {
// Instance creator is required while de-serialising using Gson as key instance can't be invoked with
// no-args constructor
Gson gson = new GsonBuilder()
.registerTypeAdapter(DatasourceStructure.Key.class, new DatasourceStructure.KeyInstanceCreator())
.create();
.registerTypeAdapter(DatasourceStructure.Key.class, new DatasourceStructure.KeyInstanceCreator())
.create();
ApplicationGitReference applicationGitReference = fetchApplicationReference(baseRepoPath, gson);
processStopwatch.stopAndLogTimeInMillis();
@ -522,6 +587,42 @@ public class FileUtilsImpl implements FileInterface {
return resource;
}
/**
* This method will read the content of the file as a plain text and does not apply the gson to json transformation
* @param filePath file path for files on which read operation will be performed
* @return content of the file in the path
*/
private String readFileAsString(Path filePath) {
String data = "";
try {
data = FileUtils.readFileToString(filePath.toFile(), "UTF-8");
} catch (IOException e) {
log.error(" Error while reading the file from git repo {0} ", e.getMessage());
}
return data;
}
/**
* This method is to read the content for action and actionCollection or any nested resources which has the new structure - v3
* Where the user queries and the metadata is split into to different files
* @param directoryPath file path for files on which read operation will be performed
* @return resources stored in the directory
*/
private Map<String, Object> readActionCollection(Path directoryPath, Gson gson, String keySuffix, Map<String, String> actionCollectionBodyMap) {
Map<String, Object> resource = new HashMap<>();
File directory = directoryPath.toFile();
if (directory.isDirectory()) {
for (File dirFile : Objects.requireNonNull(directory.listFiles())) {
String resourceName = dirFile.getName();
String body = readFileAsString(directoryPath.resolve(resourceName).resolve( resourceName + CommonConstants.JS_EXTENSION));
Object file = readFile(directoryPath.resolve(resourceName).resolve( CommonConstants.METADATA + CommonConstants.JSON_EXTENSION), gson);
actionCollectionBodyMap.put(resourceName + keySuffix, body);
resource.put(resourceName + keySuffix, file);
}
}
return resource;
}
private ApplicationGitReference fetchApplicationReference(Path baseRepoPath, Gson gson) {
ApplicationGitReference applicationGitReference = new ApplicationGitReference();
// Extract application metadata from the json
@ -549,26 +650,8 @@ public class FileUtilsImpl implements FileInterface {
break;
case 2:
// Extract pages and nested actions and actionCollections
File directory = pageDirectory.toFile();
Map<String, Object> pageMap = new HashMap<>();
Map<String, Object> actionMap = new HashMap<>();
Map<String, Object> actionCollectionMap = new HashMap<>();
// TODO same approach should be followed for modules(app level actions, actionCollections, widgets etc)
if (directory.isDirectory()) {
// Loop through all the directories and nested directories inside the pages directory to extract
// pages, actions and actionCollections from the JSON files
for (File page : Objects.requireNonNull(directory.listFiles())) {
pageMap.put(page.getName(), readFile(page.toPath().resolve(CommonConstants.CANVAS + CommonConstants.JSON_EXTENSION), gson));
actionMap.putAll(readFiles(page.toPath().resolve(ACTION_DIRECTORY), gson, page.getName()));
actionCollectionMap.putAll(readFiles(page.toPath().resolve(ACTION_COLLECTION_DIRECTORY), gson, page.getName()));
}
}
applicationGitReference.setActions(actionMap);
applicationGitReference.setActionCollections(actionCollectionMap);
applicationGitReference.setPages(pageMap);
// Extract datasources
applicationGitReference.setDatasources(readFiles(baseRepoPath.resolve(DATASOURCE_DIRECTORY), gson, ""));
case 3:
updateGitApplicationReference(baseRepoPath, gson, applicationGitReference, pageDirectory, fileFormatVersion);
break;
default:
@ -582,6 +665,36 @@ public class FileUtilsImpl implements FileInterface {
return applicationGitReference;
}
private void updateGitApplicationReference(Path baseRepoPath, Gson gson, ApplicationGitReference applicationGitReference, Path pageDirectory, int fileFormatVersion) {
// Extract pages and nested actions and actionCollections
File directory = pageDirectory.toFile();
Map<String, Object> pageMap = new HashMap<>();
Map<String, Object> actionMap = new HashMap<>();
Map<String, Object> actionBodyMap = new HashMap<>();
Map<String, Object> actionCollectionMap = new HashMap<>();
Map<String, String> actionCollectionBodyMap = new HashMap<>();
// TODO same approach should be followed for modules(app level actions, actionCollections, widgets etc)
if (directory.isDirectory()) {
// Loop through all the directories and nested directories inside the pages directory to extract
// pages, actions and actionCollections from the JSON files
for (File page : Objects.requireNonNull(directory.listFiles())) {
pageMap.put(page.getName(), readFile(page.toPath().resolve(CommonConstants.CANVAS + CommonConstants.JSON_EXTENSION), gson));
actionMap.putAll(readFiles(page.toPath().resolve(ACTION_DIRECTORY), gson, page.getName()));
if (fileFormatVersion == 3) {
actionCollectionMap.putAll(readActionCollection(page.toPath().resolve(ACTION_COLLECTION_DIRECTORY), gson, page.getName(), actionCollectionBodyMap));
} else {
actionCollectionMap.putAll(readFiles(page.toPath().resolve(ACTION_COLLECTION_DIRECTORY), gson, page.getName()));
}
}
}
applicationGitReference.setActions(actionMap);
applicationGitReference.setActionCollections(actionCollectionMap);
applicationGitReference.setActionCollectionBody(actionCollectionBodyMap);
applicationGitReference.setPages(pageMap);
// Extract datasources
applicationGitReference.setDatasources(readFiles(baseRepoPath.resolve(DATASOURCE_DIRECTORY), gson, ""));
}
private Integer getFileFormatVersion(Object metadata) {
if (metadata == null) {
return 1;

View File

@ -19,6 +19,7 @@ public class ApplicationGitReference {
Object theme;
Map<String, Object> actions;
Map<String, Object> actionCollections;
Map<String, String> actionCollectionBody;
Map<String, Object> pages;
Map<String, Object> datasources;
Map<String, Object> jsLibraries;

View File

@ -1,7 +1,6 @@
package com.appsmith.server.controllers.ce;
import com.appsmith.server.constants.FieldName;
import com.appsmith.server.domains.Application;
import com.appsmith.server.dtos.ApplicationImportDTO;
import com.appsmith.server.dtos.ApplicationTemplate;
import com.appsmith.server.dtos.ResponseDTO;

View File

@ -36,8 +36,6 @@ import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseStatus;
import reactor.core.publisher.Mono;
import java.util.ArrayList;
import java.util.EnumSet;
import java.util.List;
import java.util.Map;

View File

@ -1,7 +1,7 @@
package com.appsmith.server.domains;
import com.appsmith.external.models.BaseDomain;
import com.appsmith.server.dtos.ActionCollectionDTO;
import com.appsmith.external.models.BaseDomain;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;

View File

@ -1,13 +1,13 @@
package com.appsmith.server.dtos;
import com.appsmith.server.constants.FieldName;
import com.appsmith.server.exceptions.AppsmithError;
import com.appsmith.external.exceptions.ErrorDTO;
import com.appsmith.external.models.ActionDTO;
import com.appsmith.external.models.DefaultResources;
import com.appsmith.external.models.JSValue;
import com.appsmith.server.constants.FieldName;
import com.appsmith.server.domains.ActionCollection;
import com.appsmith.external.models.PluginType;
import com.appsmith.server.exceptions.AppsmithError;
import com.appsmith.external.exceptions.ErrorDTO;
import com.appsmith.server.domains.ActionCollection;
import com.fasterxml.jackson.annotation.JsonFormat;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;

View File

@ -194,6 +194,7 @@ public class GitFileUtils {
// Insert JSOObjects and also assign the keys which later will be used for saving the resource in actual filepath
// JSObjectName_pageName => nomenclature for the keys
Map<String, String> resourceMapBody = new HashMap<>();
applicationJson
.getActionCollectionList()
.stream()
@ -207,9 +208,13 @@ public class GitFileUtils {
: actionCollection.getPublishedCollection().getName() + NAME_SEPARATOR + actionCollection.getPublishedCollection().getPageId();
removeUnwantedFieldFromActionCollection(actionCollection);
String body = actionCollection.getUnpublishedCollection().getBody() != null ? actionCollection.getUnpublishedCollection().getBody() : "";
actionCollection.getUnpublishedCollection().setBody(null);
resourceMapBody.put(prefix, body);
resourceMap.put(prefix, actionCollection);
});
applicationReference.setActionCollections(new HashMap<>(resourceMap));
applicationReference.setActionCollectionBody(new HashMap<>(resourceMapBody));
applicationReference.setUpdatedResources(updatedResources);
resourceMap.clear();
@ -401,10 +406,17 @@ public class GitFileUtils {
if (CollectionUtils.isNullOrEmpty(applicationReference.getActionCollections())) {
applicationJson.setActionCollectionList(new ArrayList<>());
} else {
Map<String, String> actionCollectionBody = applicationReference.getActionCollectionBody();
List<ActionCollection> actionCollections = getApplicationResource(applicationReference.getActionCollections(), ActionCollection.class);
// Remove null values if present
org.apache.commons.collections.CollectionUtils.filter(actionCollections, PredicateUtils.notNullPredicate());
actionCollections.forEach(actionCollection -> {
// Set the js object body to the unpublished collection
// Since file version v3 we are splitting the js object code and metadata separately
String keyName = actionCollection.getUnpublishedCollection().getName() + actionCollection.getUnpublishedCollection().getPageId();
if (actionCollectionBody!= null && actionCollectionBody.containsKey(keyName)) {
actionCollection.getUnpublishedCollection().setBody(actionCollectionBody.get(keyName));
}
// As we are publishing the app and then committing to git we expect the published and unpublished
// actionCollectionDTO will be same, so we create a deep copy for the published version for
// actionCollection from unpublishedActionCollectionDTO

View File

@ -59,9 +59,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import static com.appsmith.server.acl.AclPermission.MANAGE_PAGES;
import static com.appsmith.server.acl.AclPermission.READ_COMMENTS;
import static com.appsmith.server.acl.AclPermission.READ_PAGES;
import static com.appsmith.server.acl.AclPermission.READ_THREADS;
import static com.appsmith.server.constants.CommentConstants.APPSMITH_BOT_NAME;
import static com.appsmith.server.constants.CommentConstants.APPSMITH_BOT_USERNAME;

View File

@ -3,11 +3,9 @@ package com.appsmith.server.services.ce;
import com.appsmith.server.configurations.GoogleRecaptchaConfig;
import com.appsmith.server.exceptions.AppsmithError;
import com.appsmith.server.exceptions.AppsmithException;
import com.appsmith.server.services.CaptchaService;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils;
import org.springframework.web.reactive.function.client.WebClient;
import reactor.core.publisher.Mono;

View File

@ -1,6 +1,5 @@
package com.appsmith.server.services.ce;
import com.appsmith.server.acl.AclPermission;
import com.appsmith.server.constants.FieldName;
import com.appsmith.server.domains.Layout;
import com.appsmith.server.dtos.PageDTO;
@ -18,9 +17,6 @@ import reactor.core.publisher.Mono;
import java.util.ArrayList;
import java.util.List;
import static com.appsmith.server.acl.AclPermission.MANAGE_PAGES;
import static com.appsmith.server.acl.AclPermission.READ_PAGES;
@Slf4j
public class LayoutServiceCEImpl implements LayoutServiceCE {

View File

@ -6,12 +6,10 @@ import com.appsmith.server.configurations.CloudServicesConfig;
import com.appsmith.server.dtos.ProviderPaginatedDTO;
import com.appsmith.server.exceptions.AppsmithError;
import com.appsmith.server.exceptions.AppsmithException;
import com.appsmith.server.services.MarketplaceService;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap;
import org.springframework.util.StringUtils;

View File

@ -10,9 +10,9 @@ import com.appsmith.external.models.Datasource;
import com.appsmith.external.models.DatasourceConfiguration;
import com.appsmith.external.models.DefaultResources;
import com.appsmith.external.models.JSValue;
import com.appsmith.external.models.PluginType;
import com.appsmith.server.constants.FieldName;
import com.appsmith.server.domains.ActionCollection;
import com.appsmith.external.models.PluginType;
import com.appsmith.server.domains.Application;
import com.appsmith.server.domains.ApplicationPage;
import com.appsmith.server.domains.GitApplicationMetadata;