Protei/PromucFlow_constructor
2
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

chore: Introduce changes to minimize git footprint (#32595)

Browse Source
This commit is contained in:
Nidhi 2024-04-16 21:34:24 +05:30 committed by GitHub
parent bc3d46d8c1
commit c51e4b810d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
69 changed files with 1053 additions and 445 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
app/client/cypress/fixtures/PartialImportExport/CustomJsLibsExportedOnly.json
View File

@ -1 +1 @@
{"clientSchemaVersion":1,"serverSchemaVersion":7,"customJSLibList":[{"name":"jsonwebtoken","accessor":["jsonwebtoken"],"url":"/libraries/jsonwebtoken@8.5.1.js","version":"8.5.1","defs":"{\"!name\":\"LIB/jsonwebtoken\",\"jsonwebtoken\":{\"decode\":{\"!type\":\"fn()\",\"prototype\":{}},\"verify\":{\"!type\":\"fn()\",\"prototype\":{}},\"sign\":{\"!type\":\"fn()\",\"prototype\":{}},\"JsonWebTokenError\":{\"!type\":\"fn()\",\"prototype\":{\"message\":{\"!type\":\"string\"},\"toString\":{\"!type\":\"fn()\"}}},\"NotBeforeError\":{\"!type\":\"fn()\",\"prototype\":{}},\"TokenExpiredError\":{\"!type\":\"fn()\",\"prototype\":{}}}}","userPermissions":[],"uidString":"jsonwebtoken_/libraries/jsonwebtoken@8.5.1.js","new":true}],"widgets":""}
{"artifactJsonType":"APPLICATION","clientSchemaVersion":1,"serverSchemaVersion":7,"customJSLibList":[{"name":"jsonwebtoken","accessor":["jsonwebtoken"],"url":"/libraries/jsonwebtoken@8.5.1.js","version":"8.5.1","defs":"{\"!name\":\"LIB/jsonwebtoken\",\"jsonwebtoken\":{\"decode\":{\"!type\":\"fn()\",\"prototype\":{}},\"verify\":{\"!type\":\"fn()\",\"prototype\":{}},\"sign\":{\"!type\":\"fn()\",\"prototype\":{}},\"JsonWebTokenError\":{\"!type\":\"fn()\",\"prototype\":{\"message\":{\"!type\":\"string\"},\"toString\":{\"!type\":\"fn()\"}}},\"NotBeforeError\":{\"!type\":\"fn()\",\"prototype\":{}},\"TokenExpiredError\":{\"!type\":\"fn()\",\"prototype\":{}}}}","userPermissions":[],"uidString":"jsonwebtoken_/libraries/jsonwebtoken@8.5.1.js","new":true}],"widgets":""}

2
app/client/cypress/fixtures/PartialImportExport/DatasourceExportedOnly.json
View File

@ -1 +1 @@
{"clientSchemaVersion":1,"serverSchemaVersion":7,"datasourceList":[{"datasourceConfiguration":{"connection":{"mode":"READ_WRITE","ssl":{"authType":"DEFAULT"}},"endpoints":[{"host":"mockdb.internal.appsmith.com"}]},"name":"Users","pluginId":"postgres-plugin","messages":[],"isAutoGenerated":false,"isMock":true,"isValid":true,"embedded":false,"new":true}],"widgets":""}
{"artifactJsonType":"APPLICATION","clientSchemaVersion":1,"serverSchemaVersion":7,"datasourceList":[{"datasourceConfiguration":{"connection":{"mode":"READ_WRITE","ssl":{"authType":"DEFAULT"}},"endpoints":[{"host":"mockdb.internal.appsmith.com"}]},"name":"Users","pluginId":"postgres-plugin","messages":[],"isAutoGenerated":false,"isMock":true,"isValid":true,"new":true}],"widgets":""}

2
app/client/cypress/fixtures/PartialImportExport/JSExportedOnly.json
View File

@ -1 +1 @@
{"clientSchemaVersion":1,"serverSchemaVersion":7,"actionList":[],"actionCollectionList":[{"id":"Home_JSObject1","unpublishedCollection":{"name":"JSObject1","pageId":"Home","pluginId":"js-plugin","pluginType":"JS","actions":[],"archivedActions":[],"body":"export default {\n\tmyVar1: [],\n\tmyVar2: {},\n\taddNumbers (a, b) {\n\t\treturn a+b;\n\t},\n\tasync myFun2 () {\n\t\t//\tuse async-await or promises\n\t\t//\tawait storeValue('varName', 'hello world')\n\t}\n}","variables":[{"name":"myVar1","value":"[]"},{"name":"myVar2","value":"{}"}],"userPermissions":[],"userExecutableName":"JSObject1"},"new":false}],"widgets":""}
{"artifactJsonType":"APPLICATION","clientSchemaVersion":1,"serverSchemaVersion":7,"actionList":[],"actionCollectionList":[{"id":"Home_JSObject1","unpublishedCollection":{"name":"JSObject1","pageId":"Home","pluginId":"js-plugin","pluginType":"JS","actions":[],"archivedActions":[],"body":"export default {\n\tmyVar1: [],\n\tmyVar2: {},\n\taddNumbers (a, b) {\n\t\treturn a+b;\n\t},\n\tasync myFun2 () {\n\t\t//\tuse async-await or promises\n\t\t//\tawait storeValue('varName', 'hello world')\n\t}\n}","variables":[{"name":"myVar1","value":"[]"},{"name":"myVar2","value":"{}"}],"userPermissions":[]},"new":false}],"widgets":""}

2
app/client/cypress/fixtures/PartialImportExport/QueriesExportedOnly.json
View File

File diff suppressed because one or more lines are too long

1
app/server/appsmith-git/pom.xml
View File

@ -8,7 +8,6 @@
<artifactId>integrated</artifactId>
<version>1.0-SNAPSHOT</version>
</parent>
<groupId>com.appsmith</groupId>
<artifactId>appsmith-git</artifactId>
<version>1.0-SNAPSHOT</version>
<packaging>jar</packaging>

360
app/server/appsmith-git/src/main/java/com/appsmith/git/helpers/ce/FileUtilsCEImpl.java → app/server/appsmith-git/src/main/java/com/appsmith/git/files/FileUtilsCEImpl.java
View File

@ -1,35 +1,25 @@
package com.appsmith.git.helpers.ce;
package com.appsmith.git.files;
import com.appsmith.external.converters.ISOStringToInstantConverter;
import com.appsmith.external.dtos.ModifiedResources;
import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginError;
import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginException;
import com.appsmith.external.git.FileInterface;
import com.appsmith.external.git.GitExecutor;
import com.appsmith.external.git.constants.GitSpan;
import com.appsmith.external.git.operations.FileOperations;
import com.appsmith.external.helpers.ObservationHelper;
import com.appsmith.external.helpers.Stopwatch;
import com.appsmith.external.models.ApplicationGitReference;
import com.appsmith.external.models.ArtifactGitReference;
import com.appsmith.external.models.BaseDomain;
import com.appsmith.external.models.DatasourceStructure;
import com.appsmith.git.configurations.GitServiceConfig;
import com.appsmith.git.constants.CommonConstants;
import com.appsmith.git.converters.GsonDoubleToLongConverter;
import com.appsmith.git.converters.GsonUnorderedToOrderedConverter;
import com.appsmith.git.helpers.DSLTransformerHelper;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.stream.JsonReader;
import io.micrometer.tracing.Span;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.json.JSONArray;
import org.json.JSONObject;
import org.springframework.context.annotation.Import;
import org.springframework.stereotype.Component;
@ -42,21 +32,15 @@ import reactor.core.scheduler.Schedulers;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.StringWriter;
import java.nio.charset.StandardCharsets;
import java.nio.file.DirectoryNotEmptyException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.attribute.BasicFileAttributes;
import java.nio.file.attribute.FileTime;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.time.Instant;
import java.util.Arrays;
import java.util.Base64;
import java.util.HashMap;
import java.util.HashSet;
@ -65,7 +49,6 @@ import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.regex.Pattern;
import java.util.stream.Stream;
import static com.appsmith.external.git.constants.GitConstants.ACTION_COLLECTION_LIST;
import static com.appsmith.external.git.constants.GitConstants.ACTION_LIST;
@ -73,10 +56,8 @@ import static com.appsmith.external.git.constants.GitConstants.CUSTOM_JS_LIB_LIS
import static com.appsmith.external.git.constants.GitConstants.NAME_SEPARATOR;
import static com.appsmith.external.git.constants.GitConstants.PAGE_LIST;
import static com.appsmith.external.git.constants.ce.GitConstantsCE.GitMetricConstantsCE.ACTION_COLLECTION_BODY;
import static com.appsmith.external.git.constants.ce.GitConstantsCE.GitMetricConstantsCE.METADATA;
import static com.appsmith.external.git.constants.ce.GitConstantsCE.GitMetricConstantsCE.NEW_ACTION_BODY;
import static com.appsmith.external.git.constants.ce.GitConstantsCE.GitMetricConstantsCE.RESOURCE_TYPE;
import static com.appsmith.external.git.constants.ce.GitConstantsCE.GitMetricConstantsCE.WIDGETS;
import static com.appsmith.git.constants.GitDirectories.ACTION_COLLECTION_DIRECTORY;
import static com.appsmith.git.constants.GitDirectories.ACTION_DIRECTORY;
import static com.appsmith.git.constants.GitDirectories.DATASOURCE_DIRECTORY;
@ -91,7 +72,7 @@ public class FileUtilsCEImpl implements FileInterface {
private final GitServiceConfig gitServiceConfig;
private final GitExecutor gitExecutor;
private final Gson gson;
private final FileOperations fileOperations;
private final ObservationHelper observationHelper;
private static final String EDIT_MODE_URL_TEMPLATE = "{{editModeUrl}}";
@ -108,26 +89,11 @@ public class FileUtilsCEImpl implements FileInterface {
public FileUtilsCEImpl(
GitServiceConfig gitServiceConfig,
GitExecutor gitExecutor,
GsonBuilder gsonBuilder,
FileOperations fileOperations,
ObservationHelper observationHelper) {
this.gitServiceConfig = gitServiceConfig;
this.gitExecutor = gitExecutor;
// Gson to pretty format JSON file
// Keep Long type as is by default GSON have behavior to convert to Double
// Convert unordered set to ordered one
this.gson = gsonBuilder
.registerTypeAdapter(Double.class, new GsonDoubleToLongConverter())
.registerTypeAdapter(Set.class, new GsonUnorderedToOrderedConverter())
.registerTypeAdapter(Map.class, new GsonUnorderedToOrderedConverter())
.registerTypeAdapter(Instant.class, new ISOStringToInstantConverter())
// Instance creator is required while de-serialising using Gson as key instance can't be invoked
// with no-args constructor
.registerTypeAdapter(DatasourceStructure.Key.class, new DatasourceStructure.KeyInstanceCreator())
.disableHtmlEscaping()
.setPrettyPrinting()
.create();
this.fileOperations = fileOperations;
this.observationHelper = observationHelper;
}
@ -255,8 +221,8 @@ public class FileUtilsCEImpl implements FileInterface {
ModifiedResources modifiedResources = applicationGitReference.getModifiedResources();
// Remove unwanted directories which was present in v1 of the git file format version
deleteDirectory(baseRepo.resolve(ACTION_DIRECTORY));
deleteDirectory(baseRepo.resolve(ACTION_COLLECTION_DIRECTORY));
fileOperations.deleteDirectory(baseRepo.resolve(ACTION_DIRECTORY));
fileOperations.deleteDirectory(baseRepo.resolve(ACTION_COLLECTION_DIRECTORY));
// Save application
saveResource(
@ -264,9 +230,7 @@ public class FileUtilsCEImpl implements FileInterface {
baseRepo.resolve(CommonConstants.APPLICATION + CommonConstants.JSON_EXTENSION));
// Save application metadata
JsonObject metadata = gson.fromJson(gson.toJson(applicationGitReference.getMetadata()), JsonObject.class);
metadata.addProperty(CommonConstants.FILE_FORMAT_VERSION, CommonConstants.fileFormatVersion);
saveResource(metadata, baseRepo.resolve(CommonConstants.METADATA + CommonConstants.JSON_EXTENSION));
fileOperations.saveMetadataResource(applicationGitReference, baseRepo);
// Save application theme
saveResource(
@ -307,19 +271,20 @@ public class FileUtilsCEImpl implements FileInterface {
Path path = Paths.get(
String.valueOf(pageSpecificDirectory.resolve(CommonConstants.WIDGETS)), childPath);
validWidgetToParentMap.put(widgetName, path.toFile().toString());
saveWidgets(jsonObject, widgetName, path);
fileOperations.saveWidgets(jsonObject, widgetName, path);
});
// Remove deleted widgets from the file system
deleteWidgets(
pageSpecificDirectory.resolve(CommonConstants.WIDGETS).toFile(), validWidgetToParentMap);
// Remove the canvas.json from the file system since the value is stored in the page.json
deleteFile(pageSpecificDirectory.resolve(CommonConstants.CANVAS + CommonConstants.JSON_EXTENSION));
fileOperations.deleteFile(
pageSpecificDirectory.resolve(CommonConstants.CANVAS + CommonConstants.JSON_EXTENSION));
}
validPages.add(pageName);
}
scanAndDeleteDirectoryForDeletedResources(validPages, baseRepo.resolve(PAGE_DIRECTORY));
fileOperations.scanAndDeleteDirectoryForDeletedResources(validPages, baseRepo.resolve(PAGE_DIRECTORY));
// Save JS Libs if there's at least one change
if (modifiedResources != null
@ -341,7 +306,7 @@ public class FileUtilsCEImpl implements FileInterface {
}
validJsLibs.add(fileNameWithExtension);
});
scanAndDeleteFileForDeletedResources(validJsLibs, jsLibDirectory);
fileOperations.scanAndDeleteFileForDeletedResources(validJsLibs, jsLibDirectory);
}
// Create HashMap for valid actions and actionCollections
@ -381,7 +346,7 @@ public class FileUtilsCEImpl implements FileInterface {
queryName,
actionSpecificDirectory.resolve(queryName));
// Delete the resource from the old file structure v2
deleteFile(pageSpecificDirectory
fileOperations.deleteFile(pageSpecificDirectory
.resolve(ACTION_DIRECTORY)
.resolve(queryName + CommonConstants.JSON_EXTENSION));
}
@ -390,7 +355,7 @@ public class FileUtilsCEImpl implements FileInterface {
validActionsMap.forEach((pageName, validActionNames) -> {
Path pageSpecificDirectory = pageDirectory.resolve(pageName);
scanAndDeleteDirectoryForDeletedResources(
fileOperations.scanAndDeleteDirectoryForDeletedResources(
validActionNames, pageSpecificDirectory.resolve(ACTION_DIRECTORY));
});
@ -419,7 +384,7 @@ public class FileUtilsCEImpl implements FileInterface {
actionCollectionName,
actionCollectionSpecificDirectory.resolve(actionCollectionName));
// Delete the resource from the old file structure v2
deleteFile(actionCollectionSpecificDirectory.resolve(
fileOperations.deleteFile(actionCollectionSpecificDirectory.resolve(
actionCollectionName + CommonConstants.JSON_EXTENSION));
}
}
@ -428,7 +393,7 @@ public class FileUtilsCEImpl implements FileInterface {
// Verify if the old files are deleted
validActionCollectionsMap.forEach((pageName, validActionCollectionNames) -> {
Path pageSpecificDirectory = pageDirectory.resolve(pageName);
scanAndDeleteDirectoryForDeletedResources(
fileOperations.scanAndDeleteDirectoryForDeletedResources(
validActionCollectionNames, pageSpecificDirectory.resolve(ACTION_COLLECTION_DIRECTORY));
});
@ -442,7 +407,8 @@ public class FileUtilsCEImpl implements FileInterface {
}
// Scan datasource directory and delete any unwanted files if present
if (!applicationGitReference.getDatasources().isEmpty()) {
scanAndDeleteFileForDeletedResources(validDatasourceFileNames, baseRepo.resolve(DATASOURCE_DIRECTORY));
fileOperations.scanAndDeleteFileForDeletedResources(
validDatasourceFileNames, baseRepo.resolve(DATASOURCE_DIRECTORY));
}
return validPages;
@ -458,7 +424,7 @@ public class FileUtilsCEImpl implements FileInterface {
protected boolean saveResource(Object sourceEntity, Path path) {
try {
Files.createDirectories(path.getParent());
return writeToFile(sourceEntity, path);
return fileOperations.writeToFile(sourceEntity, path);
} catch (IOException e) {
log.error("Error while writing resource to file {} with {}", path, e.getMessage());
log.debug(e.getMessage());
@ -466,22 +432,6 @@ public class FileUtilsCEImpl implements FileInterface {
return false;
}
private void saveWidgets(JSONObject sourceEntity, String resourceName, Path path) {
Span span = observationHelper.createSpan(GitSpan.FILE_WRITE);
try {
Files.createDirectories(path);
String resourceType = WIDGETS;
span.tag(RESOURCE_TYPE, resourceType);
observationHelper.startSpan(span, true);
writeStringToFile(sourceEntity.toString(4), path.resolve(resourceName + CommonConstants.JSON_EXTENSION));
} catch (IOException e) {
log.debug("Error while writings widgets data to file, {}", e.getMessage());
} finally {
observationHelper.endSpan(span, true);
}
}
/**
* This method is used to write actionCollection specific resource to file system. We write the data in two steps
* 1. Actual js code
@ -508,7 +458,7 @@ public class FileUtilsCEImpl implements FileInterface {
// Write metadata for the jsObject
Path metadataPath = path.resolve(CommonConstants.METADATA + CommonConstants.JSON_EXTENSION);
return writeToFile(sourceEntity, metadataPath);
return fileOperations.writeToFile(sourceEntity, metadataPath);
} catch (IOException e) {
log.debug(e.getMessage());
} finally {
@ -544,7 +494,7 @@ public class FileUtilsCEImpl implements FileInterface {
// Write metadata for the actions
Path metadataPath = path.resolve(CommonConstants.METADATA + CommonConstants.JSON_EXTENSION);
return writeToFile(sourceEntity, metadataPath);
return fileOperations.writeToFile(sourceEntity, metadataPath);
} catch (IOException e) {
log.error("Error while reading file {} with message {} with cause", path, e.getMessage(), e.getCause());
} finally {
@ -553,101 +503,9 @@ public class FileUtilsCEImpl implements FileInterface {
return false;
}
private boolean writeStringToFile(String data, Path path) throws IOException {
private void writeStringToFile(String sourceEntity, Path path) throws IOException {
try (BufferedWriter fileWriter = Files.newBufferedWriter(path, StandardCharsets.UTF_8)) {
fileWriter.write(data);
return true;
}
}
private boolean writeToFile(Object sourceEntity, Path path) throws IOException {
Span span = observationHelper.createSpan(GitSpan.FILE_WRITE);
String resourceType = sourceEntity.getClass().getSimpleName();
if (!(sourceEntity instanceof BaseDomain)) {
resourceType = METADATA;
}
span.tag(RESOURCE_TYPE, resourceType);
observationHelper.startSpan(span, true);
try (BufferedWriter fileWriter = Files.newBufferedWriter(path, StandardCharsets.UTF_8)) {
gson.toJson(sourceEntity, fileWriter);
return true;
} finally {
observationHelper.endSpan(span, true);
}
}
/**
* This method will delete the JSON resource available in local git directory on subsequent commit made after the
* deletion of respective resource from DB
*
* @param validResources resources those are still available in DB
* @param resourceDirectory directory which needs to be scanned for possible file deletion operations
*/
public void scanAndDeleteFileForDeletedResources(Set<String> validResources, Path resourceDirectory) {
// Scan resource directory and delete any unwanted file if present
// unwanted file : corresponding resource from DB has been deleted
if (resourceDirectory.toFile().exists()) {
try (Stream<Path> paths = Files.walk(resourceDirectory)) {
paths.filter(pathLocal -> Files.isRegularFile(pathLocal)
&& !validResources.contains(
pathLocal.getFileName().toString()))
.forEach(this::deleteFile);
} catch (IOException e) {
log.error("Error while scanning directory: {}, with error {}", resourceDirectory, e.getMessage());
}
}
}
/**
* This method will delete the JSON resource directory available in local git directory on subsequent commit made after the
* deletion of respective resource from DB
*
* @param validResources resources those are still available in DB
* @param resourceDirectory directory which needs to be scanned for possible file deletion operations
*/
public void scanAndDeleteDirectoryForDeletedResources(Set<String> validResources, Path resourceDirectory) {
// Scan resource directory and delete any unwanted directory if present
// unwanted directory : corresponding resource from DB has been deleted
if (resourceDirectory.toFile().exists()) {
try (Stream<Path> paths = Files.walk(resourceDirectory, 1)) {
paths.filter(path -> Files.isDirectory(path)
&& !path.equals(resourceDirectory)
&& !validResources.contains(path.getFileName().toString()))
.forEach(this::deleteDirectory);
} catch (IOException e) {
log.error("Error while scanning directory {} with error {}", resourceDirectory, e.getMessage());
}
}
}
/**
* This method will delete the directory and all its contents
*
* @param directory
*/
private void deleteDirectory(Path directory) {
if (directory.toFile().exists()) {
try {
FileUtils.deleteDirectory(directory.toFile());
} catch (IOException e) {
log.error("Unable to delete directory for path {} with message {}", directory, e.getMessage());
}
}
}
/**
* This method will delete the file from local repo
*
* @param filePath file that needs to be deleted
*/
private void deleteFile(Path filePath) {
try {
Files.deleteIfExists(filePath);
} catch (DirectoryNotEmptyException e) {
log.error("Unable to delete non-empty directory at {} with cause", filePath, e.getMessage());
} catch (IOException e) {
log.error("Unable to delete file {} with {}", filePath, e.getMessage());
fileWriter.write(sourceEntity);
}
}
@ -747,73 +605,6 @@ public class FileUtilsCEImpl implements FileInterface {
});
}
/**
* This method will be used to read and dehydrate the json file present from the local git repo
*
* @param filePath file on which the read operation will be performed
* @return resource stored in the JSON file
*/
public Object readFile(Path filePath) {
Span span = observationHelper.createSpan(GitSpan.FILE_READ);
observationHelper.startSpan(span, true);
Object file;
try (JsonReader reader = new JsonReader(new FileReader(filePath.toFile()))) {
file = gson.fromJson(reader, Object.class);
} catch (Exception e) {
log.error("Error while reading file {} with message {} with cause", filePath, e.getMessage(), e.getCause());
return null;
} finally {
observationHelper.endSpan(span, true);
}
return file;
}
/**
* This method will be used to read and dehydrate the json files present from the local git repo
*
* @param directoryPath directory path for files on which read operation will be performed
* @return resources stored in the directory
*/
protected Map<String, Object> readFiles(Path directoryPath, String keySuffix) {
Map<String, Object> resource = new HashMap<>();
File directory = directoryPath.toFile();
if (directory.isDirectory()) {
Arrays.stream(Objects.requireNonNull(directory.listFiles())).forEach(file -> {
try (JsonReader reader = new JsonReader(new FileReader(file))) {
resource.put(file.getName() + keySuffix, gson.fromJson(reader, Object.class));
} catch (Exception e) {
log.error(
"Error while reading file {} with message {} with cause",
file.toPath(),
e.getMessage(),
e.getCause());
}
});
}
return resource;
}
/**
* This method will read the content of the file as a plain text and does not apply the gson to json transformation
*
* @param filePath file path for files on which read operation will be performed
* @return content of the file in the path
*/
private String readFileAsString(Path filePath) {
Span span = observationHelper.createSpan(GitSpan.FILE_READ);
observationHelper.startSpan(span, true);
String data = CommonConstants.EMPTY_STRING;
try {
data = FileUtils.readFileToString(filePath.toFile(), "UTF-8");
} catch (IOException e) {
log.error("Error while reading the file from git repo {} ", e.getMessage());
} finally {
observationHelper.endSpan(span, true);
}
return data;
}
/**
* This method is to read the content for action and actionCollection or any nested resources which has the new structure - v3
* Where the user written JS Object code and the metadata is split into to different files
@ -832,9 +623,9 @@ public class FileUtilsCEImpl implements FileInterface {
directoryPath.resolve(resourceName).resolve(resourceName + CommonConstants.JS_EXTENSION);
String body = CommonConstants.EMPTY_STRING;
if (resourcePath.toFile().exists()) {
body = readFileAsString(resourcePath);
body = fileOperations.readFileAsString(resourcePath);
}
Object file = readFile(directoryPath
Object file = fileOperations.readFile(directoryPath
.resolve(resourceName)
.resolve(CommonConstants.METADATA + CommonConstants.JSON_EXTENSION));
actionCollectionBodyMap.put(resourceName + keySuffix, body);
@ -862,9 +653,9 @@ public class FileUtilsCEImpl implements FileInterface {
Path queryPath =
directoryPath.resolve(resourceName).resolve(resourceName + CommonConstants.TEXT_FILE_EXTENSION);
if (queryPath.toFile().exists()) {
body = readFileAsString(queryPath);
body = fileOperations.readFileAsString(queryPath);
}
Object file = readFile(directoryPath
Object file = fileOperations.readFile(directoryPath
.resolve(resourceName)
.resolve(CommonConstants.METADATA + CommonConstants.JSON_EXTENSION));
actionCollectionBodyMap.put(resourceName + keySuffix, body);
@ -875,38 +666,40 @@ public class FileUtilsCEImpl implements FileInterface {
}
private Object readPageMetadata(Path directoryPath) {
return readFile(directoryPath.resolve(directoryPath.toFile().getName() + CommonConstants.JSON_EXTENSION));
return fileOperations.readFile(
directoryPath.resolve(directoryPath.toFile().getName() + CommonConstants.JSON_EXTENSION));
}
private ApplicationGitReference fetchApplicationReference(Path baseRepoPath) {
ApplicationGitReference applicationGitReference = new ApplicationGitReference();
// Extract application metadata from the json
Object metadata = readFile(baseRepoPath.resolve(CommonConstants.METADATA + CommonConstants.JSON_EXTENSION));
Integer fileFormatVersion = getFileFormatVersion(metadata);
Object metadata = fileOperations.readFile(
baseRepoPath.resolve(CommonConstants.METADATA + CommonConstants.JSON_EXTENSION));
Integer fileFormatVersion = fileOperations.getFileFormatVersion(metadata);
// Check if fileFormat of the saved files in repo is compatible
if (!isFileFormatCompatible(fileFormatVersion)) {
throw new AppsmithPluginException(AppsmithPluginError.INCOMPATIBLE_FILE_FORMAT);
}
// Extract application data from the json
applicationGitReference.setApplication(
readFile(baseRepoPath.resolve(CommonConstants.APPLICATION + CommonConstants.JSON_EXTENSION)));
applicationGitReference.setApplication(fileOperations.readFile(
baseRepoPath.resolve(CommonConstants.APPLICATION + CommonConstants.JSON_EXTENSION)));
applicationGitReference.setTheme(
readFile(baseRepoPath.resolve(CommonConstants.THEME + CommonConstants.JSON_EXTENSION)));
fileOperations.readFile(baseRepoPath.resolve(CommonConstants.THEME + CommonConstants.JSON_EXTENSION)));
Path pageDirectory = baseRepoPath.resolve(PAGE_DIRECTORY);
// Reconstruct application from given file format
switch (fileFormatVersion) {
case 1:
// Extract actions
applicationGitReference.setActions(
readFiles(baseRepoPath.resolve(ACTION_DIRECTORY), CommonConstants.EMPTY_STRING));
fileOperations.readFiles(baseRepoPath.resolve(ACTION_DIRECTORY), CommonConstants.EMPTY_STRING));
// Extract actionCollections
applicationGitReference.setActionCollections(
readFiles(baseRepoPath.resolve(ACTION_COLLECTION_DIRECTORY), CommonConstants.EMPTY_STRING));
applicationGitReference.setActionCollections(fileOperations.readFiles(
baseRepoPath.resolve(ACTION_COLLECTION_DIRECTORY), CommonConstants.EMPTY_STRING));
// Extract pages
applicationGitReference.setPages(readFiles(pageDirectory, CommonConstants.EMPTY_STRING));
applicationGitReference.setPages(fileOperations.readFiles(pageDirectory, CommonConstants.EMPTY_STRING));
// Extract datasources
applicationGitReference.setDatasources(
readFiles(baseRepoPath.resolve(DATASOURCE_DIRECTORY), CommonConstants.EMPTY_STRING));
applicationGitReference.setDatasources(fileOperations.readFiles(
baseRepoPath.resolve(DATASOURCE_DIRECTORY), CommonConstants.EMPTY_STRING));
break;
case 2:
@ -925,7 +718,7 @@ public class FileUtilsCEImpl implements FileInterface {
applicationGitReference.setMetadata(metadata);
Path jsLibDirectory = baseRepoPath.resolve(JS_LIB_DIRECTORY);
Map<String, Object> jsLibrariesMap = readFiles(jsLibDirectory, CommonConstants.EMPTY_STRING);
Map<String, Object> jsLibrariesMap = fileOperations.readFiles(jsLibDirectory, CommonConstants.EMPTY_STRING);
applicationGitReference.setJsLibraries(jsLibrariesMap);
return applicationGitReference;
@ -950,13 +743,14 @@ public class FileUtilsCEImpl implements FileInterface {
for (File page : Objects.requireNonNull(directory.listFiles())) {
pageMap.put(
page.getName(),
readFile(page.toPath().resolve(CommonConstants.CANVAS + CommonConstants.JSON_EXTENSION)));
fileOperations.readFile(
page.toPath().resolve(CommonConstants.CANVAS + CommonConstants.JSON_EXTENSION)));
if (fileFormatVersion >= 4) {
actionMap.putAll(
readAction(page.toPath().resolve(ACTION_DIRECTORY), page.getName(), actionBodyMap));
} else {
actionMap.putAll(readFiles(page.toPath().resolve(ACTION_DIRECTORY), page.getName()));
actionMap.putAll(fileOperations.readFiles(page.toPath().resolve(ACTION_DIRECTORY), page.getName()));
}
if (fileFormatVersion >= 3) {
@ -965,8 +759,8 @@ public class FileUtilsCEImpl implements FileInterface {
page.getName(),
actionCollectionBodyMap));
} else {
actionCollectionMap.putAll(
readFiles(page.toPath().resolve(ACTION_COLLECTION_DIRECTORY), page.getName()));
actionCollectionMap.putAll(fileOperations.readFiles(
page.toPath().resolve(ACTION_COLLECTION_DIRECTORY), page.getName()));
}
}
}
@ -977,16 +771,7 @@ public class FileUtilsCEImpl implements FileInterface {
applicationGitReference.setPages(pageMap);
// Extract datasources
applicationGitReference.setDatasources(
readFiles(baseRepoPath.resolve(DATASOURCE_DIRECTORY), CommonConstants.EMPTY_STRING));
}
private Integer getFileFormatVersion(Object metadata) {
if (metadata == null) {
return 1;
}
JsonObject json = gson.fromJson(gson.toJson(metadata), JsonObject.class);
JsonElement fileFormatVersion = json.get(CommonConstants.FILE_FORMAT_VERSION);
return fileFormatVersion.getAsInt();
fileOperations.readFiles(baseRepoPath.resolve(DATASOURCE_DIRECTORY), CommonConstants.EMPTY_STRING));
}
public static boolean isFileFormatCompatible(int savedFileFormat) {
@ -1005,7 +790,6 @@ public class FileUtilsCEImpl implements FileInterface {
Map<String, Object> actionMap = new HashMap<>();
Map<String, String> actionBodyMap = new HashMap<>();
Map<String, Object> actionCollectionMap = new HashMap<>();
Map<String, Object> moduleInstanceMap = new HashMap<>();
Map<String, String> actionCollectionBodyMap = new HashMap<>();
if (directory.isDirectory()) {
// Loop through all the directories and nested directories inside the pages directory to extract
@ -1014,7 +798,7 @@ public class FileUtilsCEImpl implements FileInterface {
if (page.isDirectory()) {
pageMap.put(page.getName(), readPageMetadata(page.toPath()));
JSONObject mainContainer = getMainContainer(pageMap.get(page.getName()));
JSONObject mainContainer = fileOperations.getMainContainer(pageMap.get(page.getName()));
// Read widgets data recursively from the widgets directory
Map<String, JSONObject> widgetsData = readWidgetsData(
@ -1042,7 +826,7 @@ public class FileUtilsCEImpl implements FileInterface {
applicationGitReference.setPageDsl(pageDsl);
// Extract datasources
applicationGitReference.setDatasources(
readFiles(baseRepoPath.resolve(DATASOURCE_DIRECTORY), CommonConstants.EMPTY_STRING));
fileOperations.readFiles(baseRepoPath.resolve(DATASOURCE_DIRECTORY), CommonConstants.EMPTY_STRING));
}
private Map<String, JSONObject> readWidgetsData(String directoryPath) {
@ -1107,48 +891,34 @@ public class FileUtilsCEImpl implements FileInterface {
// The check here is to validate if the parent is correct or not
if (!validWidgetToParentMap.containsKey(name)) {
if (file.isDirectory()) {
deleteDirectory(file.toPath());
fileOperations.deleteDirectory(file.toPath());
} else {
deleteFile(file.toPath());
fileOperations.deleteFile(file.toPath());
}
} else if (!file.getParentFile().getPath().equals(validWidgetToParentMap.get(name))
&& !file.getPath().equals(validWidgetToParentMap.get(name))) {
if (file.isDirectory()) {
deleteDirectory(file.toPath());
fileOperations.deleteDirectory(file.toPath());
} else {
deleteFile(file.toPath());
fileOperations.deleteFile(file.toPath());
}
}
}
}
private JSONObject getMainContainer(Object pageJson) {
JSONObject pageJSON = new JSONObject(gson.toJson(pageJson));
JSONArray layouts = pageJSON.getJSONObject("unpublishedPage").getJSONArray("layouts");
return layouts.getJSONObject(0).getJSONObject("dsl");
}
@Override
public Mono<Long> deleteIndexLockFile(Path path, int validTimeInSeconds) {
// Check the time created of the index.lock file
// If the File is stale for more than validTime, then delete the file
try {
BasicFileAttributes attr = Files.readAttributes(path, BasicFileAttributes.class);
FileTime fileTime = attr.creationTime();
Instant now = Instant.now();
Instant validCreateTime = now.minusSeconds(validTimeInSeconds);
if (fileTime.toInstant().isBefore(validCreateTime)) {
// Add base repo path
path = Paths.get(path + ".lock");
deleteFile(path);
return Mono.just(now.minusMillis(fileTime.toMillis()).getEpochSecond());
} else {
return Mono.just(0L);
return fileOperations.deleteIndexLockFile(path, validTimeInSeconds);
}
} catch (IOException ex) {
log.error("Error reading index.lock file: {}", ex.getMessage());
return Mono.just(0L);
@Override
public void scanAndDeleteFileForDeletedResources(Set<String> validResources, Path resourceDirectory) {
fileOperations.scanAndDeleteFileForDeletedResources(validResources, resourceDirectory);
}
@Override
public void scanAndDeleteDirectoryForDeletedResources(Set<String> validResources, Path resourceDirectory) {
fileOperations.scanAndDeleteDirectoryForDeletedResources(validResources, resourceDirectory);
}
/**
@ -1193,7 +963,7 @@ public class FileUtilsCEImpl implements FileInterface {
.map(isSwitched -> {
Path baseRepoPath =
Paths.get(gitServiceConfig.getGitRootPath()).resolve(baseRepoSuffix);
Object metadata = readFile(
Object metadata = fileOperations.readFile(
baseRepoPath.resolve(CommonConstants.METADATA + CommonConstants.JSON_EXTENSION));
return metadata;
});

11
app/server/appsmith-git/src/main/java/com/appsmith/git/helpers/FileUtilsImpl.java → app/server/appsmith-git/src/main/java/com/appsmith/git/files/FileUtilsImpl.java
View File

@ -1,27 +1,28 @@
package com.appsmith.git.helpers;
package com.appsmith.git.files;
import com.appsmith.external.git.FileInterface;
import com.appsmith.external.git.GitExecutor;
import com.appsmith.external.git.operations.FileOperations;
import com.appsmith.external.helpers.ObservationHelper;
import com.appsmith.git.configurations.GitServiceConfig;
import com.appsmith.git.helpers.ce.FileUtilsCEImpl;
import com.google.gson.GsonBuilder;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.annotation.Import;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Component;
@Slf4j
@Getter
@Component
@Primary
@Import({GitServiceConfig.class})
public class FileUtilsImpl extends FileUtilsCEImpl implements FileInterface {
public FileUtilsImpl(
GitServiceConfig gitServiceConfig,
GitExecutor gitExecutor,
GsonBuilder gsonBuilder,
FileOperations fileOperations,
ObservationHelper observationHelper) {
super(gitServiceConfig, gitExecutor, gsonBuilder, observationHelper);
super(gitServiceConfig, gitExecutor, fileOperations, observationHelper);
}
}

359
app/server/appsmith-git/src/main/java/com/appsmith/git/files/operations/FileOperationsCEImpl.java Normal file
View File

@ -0,0 +1,359 @@
package com.appsmith.git.files.operations;
import com.appsmith.external.converters.ISOStringToInstantConverter;
import com.appsmith.external.git.GitExecutor;
import com.appsmith.external.git.constants.GitSpan;
import com.appsmith.external.git.operations.FileOperationsCE;
import com.appsmith.external.helpers.ObservationHelper;
import com.appsmith.external.models.ApplicationGitReference;
import com.appsmith.external.models.BaseDomain;
import com.appsmith.external.models.DatasourceStructure;
import com.appsmith.git.configurations.GitServiceConfig;
import com.appsmith.git.constants.CommonConstants;
import com.appsmith.git.converters.GsonDoubleToLongConverter;
import com.appsmith.git.converters.GsonUnorderedToOrderedConverter;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import io.micrometer.tracing.Span;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.json.JSONArray;
import org.json.JSONObject;
import org.springframework.context.annotation.Import;
import org.springframework.stereotype.Component;
import reactor.core.publisher.Mono;
import reactor.core.scheduler.Scheduler;
import reactor.core.scheduler.Schedulers;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.DirectoryNotEmptyException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.attribute.BasicFileAttributes;
import java.nio.file.attribute.FileTime;
import java.time.Instant;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.regex.Pattern;
import java.util.stream.Stream;
import static com.appsmith.external.git.constants.ce.GitConstantsCE.GitMetricConstantsCE.METADATA;
import static com.appsmith.external.git.constants.ce.GitConstantsCE.GitMetricConstantsCE.RESOURCE_TYPE;
import static com.appsmith.external.git.constants.ce.GitConstantsCE.GitMetricConstantsCE.WIDGETS;
@Slf4j
@Getter
@Component
@Import({GitServiceConfig.class})
public class FileOperationsCEImpl implements FileOperationsCE {
private final GitServiceConfig gitServiceConfig;
private final GitExecutor gitExecutor;
private final Gson gson;
protected final ObservationHelper observationHelper;
private static final String EDIT_MODE_URL_TEMPLATE = "{{editModeUrl}}";
private static final String VIEW_MODE_URL_TEMPLATE = "{{viewModeUrl}}";
private static final Pattern ALLOWED_FILE_EXTENSION_PATTERN =
Pattern.compile("(.*?)\\.(md|MD|git|gitignore|github|yml|yaml)$");
private final Scheduler scheduler = Schedulers.boundedElastic();
private static final String CANVAS_WIDGET = "(Canvas)[0-9]*.";
public FileOperationsCEImpl(
GitServiceConfig gitServiceConfig,
GitExecutor gitExecutor,
GsonBuilder gsonBuilder,
ObservationHelper observationHelper) {
this.gitServiceConfig = gitServiceConfig;
this.gitExecutor = gitExecutor;
// Gson to pretty format JSON file
// Keep Long type as is by default GSON have behavior to convert to Double
// Convert unordered set to ordered one
this.gson = gsonBuilder
.registerTypeAdapter(Double.class, new GsonDoubleToLongConverter())
.registerTypeAdapter(Set.class, new GsonUnorderedToOrderedConverter())
.registerTypeAdapter(Map.class, new GsonUnorderedToOrderedConverter())
.registerTypeAdapter(Instant.class, new ISOStringToInstantConverter())
// Instance creator is required while de-serialising using Gson as key instance can't be invoked
// with no-args constructor
.registerTypeAdapter(DatasourceStructure.Key.class, new DatasourceStructure.KeyInstanceCreator())
.disableHtmlEscaping()
.setPrettyPrinting()
.create();
this.observationHelper = observationHelper;
}
@Override
public void saveMetadataResource(ApplicationGitReference applicationGitReference, Path baseRepo) {
JsonObject metadata = gson.fromJson(gson.toJson(applicationGitReference.getMetadata()), JsonObject.class);
metadata.addProperty(CommonConstants.FILE_FORMAT_VERSION, CommonConstants.fileFormatVersion);
saveResource(metadata, baseRepo.resolve(CommonConstants.METADATA + CommonConstants.JSON_EXTENSION));
}
/**
* This method will be used to store the DB resource to JSON file
*
* @param sourceEntity resource extracted from DB to be stored in file
* @param path file path where the resource to be stored
* @return if the file operation is successful
*/
@Override
public boolean saveResource(Object sourceEntity, Path path) {
try {
Files.createDirectories(path.getParent());
return writeToFile(sourceEntity, path);
} catch (IOException e) {
log.error("Error while writing resource to file {} with {}", path, e.getMessage());
log.debug(e.getMessage());
}
return false;
}
@Override
public void saveWidgets(JSONObject sourceEntity, String resourceName, Path path) {
Span span = observationHelper.createSpan(GitSpan.FILE_WRITE);
try {
Files.createDirectories(path);
String resourceType = WIDGETS;
span.tag(RESOURCE_TYPE, resourceType);
observationHelper.startSpan(span, true);
writeStringToFile(sourceEntity.toString(4), path.resolve(resourceName + CommonConstants.JSON_EXTENSION));
} catch (IOException e) {
log.debug("Error while writings widgets data to file, {}", e.getMessage());
} finally {
observationHelper.endSpan(span, true);
}
}
@Override
public void writeStringToFile(String sourceEntity, Path path) throws IOException {
try (BufferedWriter fileWriter = Files.newBufferedWriter(path, StandardCharsets.UTF_8)) {
fileWriter.write(sourceEntity);
}
}
@Override
public boolean writeToFile(Object sourceEntity, Path path) throws IOException {
Span span = observationHelper.createSpan(GitSpan.FILE_WRITE);
String resourceType = sourceEntity.getClass().getSimpleName();
if (!(sourceEntity instanceof BaseDomain)) {
resourceType = METADATA;
}
span.tag(RESOURCE_TYPE, resourceType);
observationHelper.startSpan(span, true);
try (BufferedWriter fileWriter = Files.newBufferedWriter(path, StandardCharsets.UTF_8)) {
gson.toJson(sourceEntity, fileWriter);
return true;
} finally {
observationHelper.endSpan(span, true);
}
}
/**
* This method will delete the JSON resource available in local git directory on subsequent commit made after the
* deletion of respective resource from DB
*
* @param validResources resources those are still available in DB
* @param resourceDirectory directory which needs to be scanned for possible file deletion operations
*/
@Override
public void scanAndDeleteFileForDeletedResources(Set<String> validResources, Path resourceDirectory) {
// Scan resource directory and delete any unwanted file if present
// unwanted file : corresponding resource from DB has been deleted
if (resourceDirectory.toFile().exists()) {
try (Stream<Path> paths = Files.walk(resourceDirectory)) {
paths.filter(pathLocal -> Files.isRegularFile(pathLocal)
&& !validResources.contains(
pathLocal.getFileName().toString()))
.forEach(this::deleteFile);
} catch (IOException e) {
log.error("Error while scanning directory: {}, with error {}", resourceDirectory, e.getMessage());
}
}
}
/**
* This method will delete the JSON resource directory available in local git directory on subsequent commit made after the
* deletion of respective resource from DB
*
* @param validResources resources those are still available in DB
* @param resourceDirectory directory which needs to be scanned for possible file deletion operations
*/
@Override
public void scanAndDeleteDirectoryForDeletedResources(Set<String> validResources, Path resourceDirectory) {
// Scan resource directory and delete any unwanted directory if present
// unwanted directory : corresponding resource from DB has been deleted
if (resourceDirectory.toFile().exists()) {
try (Stream<Path> paths = Files.walk(resourceDirectory, 1)) {
paths.filter(path -> Files.isDirectory(path)
&& !path.equals(resourceDirectory)
&& !validResources.contains(path.getFileName().toString()))
.forEach(this::deleteDirectory);
} catch (IOException e) {
log.error("Error while scanning directory {} with error {}", resourceDirectory, e.getMessage());
}
}
}
/**
* This method will delete the directory and all its contents
*
* @param directory
*/
@Override
public void deleteDirectory(Path directory) {
if (directory.toFile().exists()) {
try {
FileUtils.deleteDirectory(directory.toFile());
} catch (IOException e) {
log.error("Unable to delete directory for path {} with message {}", directory, e.getMessage());
}
}
}
/**
* This method will delete the file from local repo
*
* @param filePath file that needs to be deleted
*/
@Override
public void deleteFile(Path filePath) {
try {
Files.deleteIfExists(filePath);
} catch (DirectoryNotEmptyException e) {
log.error("Unable to delete non-empty directory at {} with cause", filePath, e.getMessage());
} catch (IOException e) {
log.error("Unable to delete file {} with {}", filePath, e.getMessage());
}
}
/**
* This method will be used to read and dehydrate the json file present from the local git repo
*
* @param filePath file on which the read operation will be performed
* @return resource stored in the JSON file
*/
@Override
public Object readFile(Path filePath) {
Span span = observationHelper.createSpan(GitSpan.FILE_READ);
observationHelper.startSpan(span, true);
Object file;
try (FileReader reader = new FileReader(filePath.toFile())) {
file = gson.fromJson(reader, Object.class);
} catch (Exception e) {
log.error("Error while reading file {} with message {} with cause", filePath, e.getMessage(), e.getCause());
return null;
} finally {
observationHelper.endSpan(span, true);
}
return file;
}
/**
* This method will be used to read and dehydrate the json files present from the local git repo
*
* @param directoryPath directory path for files on which read operation will be performed
* @return resources stored in the directory
*/
@Override
public Map<String, Object> readFiles(Path directoryPath, String keySuffix) {
Map<String, Object> resource = new HashMap<>();
File directory = directoryPath.toFile();
if (directory.isDirectory()) {
Arrays.stream(Objects.requireNonNull(directory.listFiles())).forEach(file -> {
try (FileReader reader = new FileReader(file)) {
resource.put(file.getName() + keySuffix, gson.fromJson(reader, Object.class));
} catch (Exception e) {
log.error(
"Error while reading file {} with message {} with cause",
file.toPath(),
e.getMessage(),
e.getCause());
}
});
}
return resource;
}
/**
* This method will read the content of the file as a plain text and does not apply the gson to json transformation
*
* @param filePath file path for files on which read operation will be performed
* @return content of the file in the path
*/
@Override
public String readFileAsString(Path filePath) {
Span span = observationHelper.createSpan(GitSpan.FILE_READ);
observationHelper.startSpan(span, true);
String data = CommonConstants.EMPTY_STRING;
try {
data = FileUtils.readFileToString(filePath.toFile(), "UTF-8");
} catch (IOException e) {
log.error("Error while reading the file from git repo {} ", e.getMessage());
} finally {
observationHelper.endSpan(span, true);
}
return data;
}
@Override
public Integer getFileFormatVersion(Object metadata) {
if (metadata == null) {
return 1;
}
JsonObject json = gson.fromJson(gson.toJson(metadata), JsonObject.class);
JsonElement fileFormatVersion = json.get(CommonConstants.FILE_FORMAT_VERSION);
return fileFormatVersion.getAsInt();
}
@Override
public JSONObject getMainContainer(Object pageJson) {
JSONObject pageJSON = new JSONObject(gson.toJson(pageJson));
JSONArray layouts = pageJSON.getJSONObject("unpublishedPage").getJSONArray("layouts");
return layouts.getJSONObject(0).getJSONObject("dsl");
}
@Override
public Mono<Long> deleteIndexLockFile(Path path, int validTimeInSeconds) {
// Check the time created of the index.lock file
// If the File is stale for more than validTime, then delete the file
try {
BasicFileAttributes attr = Files.readAttributes(path, BasicFileAttributes.class);
FileTime fileTime = attr.creationTime();
Instant now = Instant.now();
Instant validCreateTime = now.minusSeconds(validTimeInSeconds);
if (fileTime.toInstant().isBefore(validCreateTime)) {
// Add base repo path
path = Paths.get(path + ".lock");
deleteFile(path);
return Mono.just(now.minusMillis(fileTime.toMillis()).getEpochSecond());
} else {
return Mono.just(0L);
}
} catch (IOException ex) {
log.error("Error reading index.lock file: {}", ex.getMessage());
return Mono.just(0L);
}
}
}

193
app/server/appsmith-git/src/main/java/com/appsmith/git/files/operations/FileOperationsCEv2Impl.java Normal file
View File

@ -0,0 +1,193 @@
package com.appsmith.git.files.operations;
import com.appsmith.external.annotations.FeatureFlagged;
import com.appsmith.external.enums.FeatureFlagEnum;
import com.appsmith.external.git.GitExecutor;
import com.appsmith.external.git.constants.GitSpan;
import com.appsmith.external.git.operations.FileOperationsCE;
import com.appsmith.external.helpers.ObservationHelper;
import com.appsmith.external.models.ApplicationGitReference;
import com.appsmith.external.models.BaseDomain;
import com.appsmith.external.views.Git;
import com.appsmith.git.configurations.GitServiceConfig;
import com.appsmith.git.constants.CommonConstants;
import com.appsmith.util.SerializationUtils;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.PrettyPrinter;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectReader;
import com.fasterxml.jackson.databind.ObjectWriter;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.gson.GsonBuilder;
import io.micrometer.tracing.Span;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.json.JSONObject;
import org.springframework.context.annotation.Import;
import org.springframework.stereotype.Component;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
import static com.appsmith.external.git.constants.ce.GitConstantsCE.GitMetricConstantsCE.METADATA;
import static com.appsmith.external.git.constants.ce.GitConstantsCE.GitMetricConstantsCE.RESOURCE_TYPE;
import static com.appsmith.external.git.constants.ce.GitConstantsCE.GitMetricConstantsCE.WIDGETS;
@Slf4j
@Getter
@Component
@Import({GitServiceConfig.class})
public class FileOperationsCEv2Impl extends FileOperationsCEImpl implements FileOperationsCE {
protected final ObjectMapper objectMapper;
protected final ObjectReader objectReader;
protected final ObjectWriter objectWriter;
private final ObservationHelper observationHelper;
public FileOperationsCEv2Impl(
GitServiceConfig gitServiceConfig,
GitExecutor gitExecutor,
GsonBuilder gsonBuilder,
PrettyPrinter prettyPrinter,
ObservationHelper observationHelper) {
super(gitServiceConfig, gitExecutor, gsonBuilder, observationHelper);
this.objectMapper = SerializationUtils.getBasicObjectMapper(prettyPrinter);
this.objectReader = objectMapper.readerWithView(Git.class);
this.objectWriter = objectMapper.writerWithView(Git.class);
this.observationHelper = observationHelper;
}
@FeatureFlagged(featureFlagName = FeatureFlagEnum.release_git_cleanup_feature_enabled)
@Override
public void saveMetadataResource(ApplicationGitReference applicationGitReference, Path baseRepo) {
ObjectNode metadata = objectMapper.valueToTree(applicationGitReference.getMetadata());
metadata.put(CommonConstants.FILE_FORMAT_VERSION, CommonConstants.fileFormatVersion);
saveResource(metadata, baseRepo.resolve(CommonConstants.METADATA + CommonConstants.JSON_EXTENSION));
}
@FeatureFlagged(featureFlagName = FeatureFlagEnum.release_git_cleanup_feature_enabled)
@Override
public void saveWidgets(JSONObject sourceEntity, String resourceName, Path path) {
Span span = observationHelper.createSpan(GitSpan.FILE_WRITE);
try {
Files.createDirectories(path);
String resourceType = WIDGETS;
span.tag(RESOURCE_TYPE, resourceType);
observationHelper.startSpan(span, true);
writeToFile(
objectReader.readTree(sourceEntity.toString()),
path.resolve(resourceName + CommonConstants.JSON_EXTENSION));
} catch (IOException e) {
log.debug("Error while writings widgets data to file, {}", e.getMessage());
} finally {
observationHelper.endSpan(span, true);
}
}
@FeatureFlagged(featureFlagName = FeatureFlagEnum.release_git_cleanup_feature_enabled)
@Override
public boolean writeToFile(Object sourceEntity, Path path) throws IOException {
Span span = observationHelper.createSpan(GitSpan.FILE_WRITE);
String resourceType = sourceEntity.getClass().getSimpleName();
if (!(sourceEntity instanceof BaseDomain)) {
resourceType = METADATA;
}
span.tag(RESOURCE_TYPE, resourceType);
observationHelper.startSpan(span, true);
try (BufferedWriter fileWriter = Files.newBufferedWriter(path, StandardCharsets.UTF_8)) {
objectWriter.writeValue(fileWriter, sourceEntity);
return true;
} finally {
observationHelper.endSpan(span, true);
}
}
/**
* This method will be used to read and dehydrate the json file present from the local git repo
*
* @param filePath file on which the read operation will be performed
* @return resource stored in the JSON file
*/
@FeatureFlagged(featureFlagName = FeatureFlagEnum.release_git_cleanup_feature_enabled)
@Override
public Object readFile(Path filePath) {
Span span = observationHelper.createSpan(GitSpan.FILE_READ);
observationHelper.startSpan(span, true);
Object file;
try (FileReader reader = new FileReader(filePath.toFile())) {
file = objectReader.readValue(reader, Object.class);
} catch (Exception e) {
log.error("Error while reading file {} with message {} with cause", filePath, e.getMessage(), e.getCause());
return null;
} finally {
observationHelper.endSpan(span, true);
}
return file;
}
/**
* This method will be used to read and dehydrate the json files present from the local git repo
*
* @param directoryPath directory path for files on which read operation will be performed
* @return resources stored in the directory
*/
@FeatureFlagged(featureFlagName = FeatureFlagEnum.release_git_cleanup_feature_enabled)
@Override
public Map<String, Object> readFiles(Path directoryPath, String keySuffix) {
Map<String, Object> resource = new HashMap<>();
File directory = directoryPath.toFile();
if (directory.isDirectory()) {
Arrays.stream(Objects.requireNonNull(directory.listFiles())).forEach(file -> {
try (FileReader reader = new FileReader(file)) {
resource.put(file.getName() + keySuffix, objectReader.readValue(reader, Object.class));
} catch (Exception e) {
log.error(
"Error while reading file {} with message {} with cause",
file.toPath(),
e.getMessage(),
e.getCause());
}
});
}
return resource;
}
@FeatureFlagged(featureFlagName = FeatureFlagEnum.release_git_cleanup_feature_enabled)
@Override
public Integer getFileFormatVersion(Object metadata) {
if (metadata == null) {
return 1;
}
JsonNode json = objectMapper.valueToTree(metadata);
int fileFormatVersion = json.get(CommonConstants.FILE_FORMAT_VERSION).asInt();
return fileFormatVersion;
}
@FeatureFlagged(featureFlagName = FeatureFlagEnum.release_git_cleanup_feature_enabled)
@Override
public JSONObject getMainContainer(Object pageJson) {
JsonNode pageJSON = objectMapper.valueToTree(pageJson);
try {
return new JSONObject(objectMapper.writeValueAsString(
pageJSON.get("unpublishedPage").get("layouts").get(0).get("dsl")));
} catch (JsonProcessingException e) {
throw new RuntimeException(e);
}
}
}

25
app/server/appsmith-git/src/main/java/com/appsmith/git/files/operations/FileOperationsImpl.java Normal file
View File

@ -0,0 +1,25 @@
package com.appsmith.git.files.operations;
import com.appsmith.external.git.GitExecutor;
import com.appsmith.external.git.operations.FileOperations;
import com.appsmith.external.helpers.ObservationHelper;
import com.appsmith.git.configurations.GitServiceConfig;
import com.fasterxml.jackson.core.PrettyPrinter;
import com.google.gson.GsonBuilder;
import org.springframework.context.annotation.Import;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Component;
@Component
@Primary
@Import({GitServiceConfig.class})
public class FileOperationsImpl extends FileOperationsCEv2Impl implements FileOperations {
public FileOperationsImpl(
GitServiceConfig gitServiceConfig,
GitExecutor gitExecutor,
GsonBuilder gsonBuilder,
PrettyPrinter prettyPrinter,
ObservationHelper observationHelper) {
super(gitServiceConfig, gitExecutor, gsonBuilder, prettyPrinter, observationHelper);
}
}

2
app/server/appsmith-git/src/main/java/com/appsmith/git/service/GitExecutorImpl.java
View File

@ -5,9 +5,11 @@ import com.appsmith.git.configurations.GitServiceConfig;
import com.appsmith.git.service.ce.GitExecutorCEImpl;
import io.micrometer.observation.ObservationRegistry;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Component;
@Component
@Primary
@Slf4j
public class GitExecutorImpl extends GitExecutorCEImpl implements GitExecutor {

9
app/server/appsmith-git/src/test/java/com/appsmith/git/helpers/FileUtilsImplTest.java
View File

@ -1,8 +1,11 @@
package com.appsmith.git.helpers;
import com.appsmith.external.git.operations.FileOperations;
import com.appsmith.external.helpers.ObservationHelper;
import com.appsmith.external.models.ApplicationGitReference;
import com.appsmith.git.configurations.GitServiceConfig;
import com.appsmith.git.files.FileUtilsImpl;
import com.appsmith.git.files.operations.FileOperationsImpl;
import com.appsmith.git.service.GitExecutorImpl;
import com.google.gson.GsonBuilder;
import org.apache.commons.io.FileUtils;
@ -36,6 +39,8 @@ public class FileUtilsImplTest {
@MockBean
private GitExecutorImpl gitExecutor;
private FileOperations fileOperations;
private GitServiceConfig gitServiceConfig;
private static final String localTestDirectory = "localTestDirectory";
private static final Path localTestDirectoryPath = Path.of(localTestDirectory);
@ -44,7 +49,9 @@ public class FileUtilsImplTest {
public void setUp() {
gitServiceConfig = new GitServiceConfig();
gitServiceConfig.setGitRootPath(localTestDirectoryPath.toString());
fileUtils = new FileUtilsImpl(gitServiceConfig, gitExecutor, new GsonBuilder(), ObservationHelper.NOOP);
fileOperations =
new FileOperationsImpl(gitServiceConfig, gitExecutor, new GsonBuilder(), null, ObservationHelper.NOOP);
fileUtils = new FileUtilsImpl(gitServiceConfig, gitExecutor, fileOperations, ObservationHelper.NOOP);
}
@AfterEach

8
app/server/appsmith-interfaces/pom.xml
View File

@ -88,11 +88,13 @@
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>${jackson-bom.version}</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.datatype</groupId>
<artifactId>jackson-datatype-jsr310</artifactId>
<version>${jackson-bom.version}</version>
<scope>compile</scope>
</dependency>
<dependency>
@ -184,6 +186,12 @@
<artifactId>jjwt-jackson</artifactId>
<!-- or jjwt-gson if Gson is preferred -->
<version>${jjwt.version}</version>
<exclusions>
<exclusion>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>
<!--Test dependencies-->

4
app/server/appsmith-interfaces/src/main/java/com/appsmith/external/datatypes/ArrayType.java vendored
View File

@ -4,6 +4,7 @@ import com.appsmith.external.constants.DataType;
import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginError;
import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginException;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.StreamReadFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import net.minidev.json.JSONArray;
import net.minidev.json.parser.JSONParser;
@ -11,7 +12,8 @@ import reactor.core.Exceptions;
public class ArrayType implements AppsmithType {
private static final ObjectMapper objectMapper = new ObjectMapper();
private static final ObjectMapper objectMapper =
new ObjectMapper().enable(StreamReadFeature.INCLUDE_SOURCE_IN_LOCATION.mappedFeature());
@Override
public boolean test(String s) {

4
app/server/appsmith-interfaces/src/main/java/com/appsmith/external/datatypes/DateType.java vendored
View File

@ -4,6 +4,7 @@ import com.appsmith.external.constants.DataType;
import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginError;
import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginException;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.StreamReadFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import reactor.core.Exceptions;
@ -15,7 +16,8 @@ import java.util.regex.Matcher;
public class DateType implements AppsmithType {
private static final ObjectMapper objectMapper = new ObjectMapper();
private static final ObjectMapper objectMapper =
new ObjectMapper().enable(StreamReadFeature.INCLUDE_SOURCE_IN_LOCATION.mappedFeature());
@Override
public boolean test(String s) {

4
app/server/appsmith-interfaces/src/main/java/com/appsmith/external/datatypes/JsonObjectType.java vendored
View File

@ -4,6 +4,7 @@ import com.appsmith.external.constants.DataType;
import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginError;
import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginException;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.StreamReadFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.gson.Gson;
import com.google.gson.JsonObject;
@ -21,7 +22,8 @@ import java.util.regex.Matcher;
public class JsonObjectType implements AppsmithType {
private static final TypeAdapter<JsonObject> strictGsonObjectAdapter = new Gson().getAdapter(JsonObject.class);
private static final ObjectMapper objectMapper = new ObjectMapper();
private static final ObjectMapper objectMapper =
new ObjectMapper().enable(StreamReadFeature.INCLUDE_SOURCE_IN_LOCATION.mappedFeature());
@Override
public boolean test(String s) {

4
app/server/appsmith-interfaces/src/main/java/com/appsmith/external/datatypes/StringType.java vendored
View File

@ -4,6 +4,7 @@ import com.appsmith.external.constants.DataType;
import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginError;
import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginException;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.StreamReadFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import reactor.core.Exceptions;
@ -11,7 +12,8 @@ import java.util.regex.Matcher;
public class StringType implements AppsmithType {
private static final ObjectMapper objectMapper = new ObjectMapper();
private static final ObjectMapper objectMapper =
new ObjectMapper().enable(StreamReadFeature.INCLUDE_SOURCE_IN_LOCATION.mappedFeature());
@Override
public boolean test(String s) {

4
app/server/appsmith-interfaces/src/main/java/com/appsmith/external/datatypes/TimeType.java vendored
View File

@ -4,6 +4,7 @@ import com.appsmith.external.constants.DataType;
import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginError;
import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginException;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.StreamReadFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import reactor.core.Exceptions;
@ -15,7 +16,8 @@ import java.util.regex.Matcher;
public class TimeType implements AppsmithType {
private static final ObjectMapper objectMapper = new ObjectMapper();
private static final ObjectMapper objectMapper =
new ObjectMapper().enable(StreamReadFeature.INCLUDE_SOURCE_IN_LOCATION.mappedFeature());
@Override
public boolean test(String s) {

4
app/server/appsmith-interfaces/src/main/java/com/appsmith/external/datatypes/TimestampType.java vendored
View File

@ -4,6 +4,7 @@ import com.appsmith.external.constants.DataType;
import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginError;
import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginException;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.StreamReadFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import reactor.core.Exceptions;
@ -15,7 +16,8 @@ import java.util.regex.Matcher;
public class TimestampType implements AppsmithType {
private static final ObjectMapper objectMapper = new ObjectMapper();
private static final ObjectMapper objectMapper =
new ObjectMapper().enable(StreamReadFeature.INCLUDE_SOURCE_IN_LOCATION.mappedFeature());
@Override
public boolean test(String s) {

1
app/server/appsmith-interfaces/src/main/java/com/appsmith/external/enums/FeatureFlagEnum.java vendored
View File

@ -15,5 +15,6 @@ public enum FeatureFlagEnum {
release_embed_hide_share_settings_enabled,
rollout_datasource_test_rate_limit_enabled,
release_git_autocommit_feature_enabled,
release_git_cleanup_feature_enabled,
// Add EE flags below this line, to avoid conflicts.
}

9
app/server/appsmith-interfaces/src/main/java/com/appsmith/external/git/FileInterface.java vendored
View File

@ -7,13 +7,14 @@ import reactor.core.publisher.Mono;
import java.io.IOException;
import java.nio.file.Path;
import java.util.Set;
public interface FileInterface {
/**
* This method is use to store the serialised application to git repo, directory path structure we are going to follow :
* ./container-volumes/git-repo/workspaceId/defaultApplicationId/repoName/{application_data}
* @param baseRepoSuffix path suffix used to create a repo path
* @param applicationGitReference application reference object from which entire application can be rehydrated
* @param artifactGitReference application reference object from which entire application can be rehydrated
* @return Path to where the application is stored
*
* Application will be stored in the following structure :
@ -84,9 +85,13 @@ public interface FileInterface {
* This will check if the cloned repo is empty. The check excludes files like Readme files
*
* @param baseRepoSuffix path suffix used to create a branch repo path as per worktree implementation
* @return success if the clone repo doesnt contain any files
* @return success if the clone repo doesn't contain any files
*/
Mono<Boolean> checkIfDirectoryIsEmpty(Path baseRepoSuffix) throws IOException;
Mono<Long> deleteIndexLockFile(Path path, int validTimeInSeconds);
void scanAndDeleteFileForDeletedResources(Set<String> validResources, Path resourceDirectory);
void scanAndDeleteDirectoryForDeletedResources(Set<String> validResources, Path resourceDirectory);
}

3
app/server/appsmith-interfaces/src/main/java/com/appsmith/external/git/operations/FileOperations.java vendored Normal file
View File

@ -0,0 +1,3 @@
package com.appsmith.external.git.operations;
public interface FileOperations extends FileOperationsCE {}

42
app/server/appsmith-interfaces/src/main/java/com/appsmith/external/git/operations/FileOperationsCE.java vendored Normal file
View File

@ -0,0 +1,42 @@
package com.appsmith.external.git.operations;
import com.appsmith.external.models.ApplicationGitReference;
import org.json.JSONObject;
import reactor.core.publisher.Mono;
import java.io.IOException;
import java.nio.file.Path;
import java.util.Map;
import java.util.Set;
public interface FileOperationsCE {
void saveMetadataResource(ApplicationGitReference applicationGitReference, Path baseRepo);
boolean saveResource(Object sourceEntity, Path path);
void saveWidgets(JSONObject sourceEntity, String resourceName, Path path);
void writeStringToFile(String sourceEntity, Path path) throws IOException;
boolean writeToFile(Object sourceEntity, Path path) throws IOException;
void scanAndDeleteFileForDeletedResources(Set<String> validResources, Path resourceDirectory);
void scanAndDeleteDirectoryForDeletedResources(Set<String> validResources, Path resourceDirectory);
void deleteDirectory(Path directory);
void deleteFile(Path filePath);
Object readFile(Path filePath);
Map<String, Object> readFiles(Path directoryPath, String keySuffix);
String readFileAsString(Path filePath);
Integer getFileFormatVersion(Object metadata);
JSONObject getMainContainer(Object pageJson);
Mono<Long> deleteIndexLockFile(Path path, int validTimeInSeconds);
}

4
app/server/appsmith-interfaces/src/main/java/com/appsmith/external/helpers/DataTypeStringUtils.java vendored
View File

@ -9,6 +9,7 @@ import com.appsmith.external.models.Param;
import com.appsmith.external.models.ParsedDataType;
import com.appsmith.external.plugins.SmartSubstitutionInterface;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.StreamReadFeature;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
@ -53,7 +54,8 @@ public class DataTypeStringUtils {
public static Pattern placeholderPattern = Pattern.compile(APPSMITH_SUBSTITUTION_PLACEHOLDER);
private static ObjectMapper objectMapper = new ObjectMapper();
private static ObjectMapper objectMapper =
new ObjectMapper().enable(StreamReadFeature.INCLUDE_SOURCE_IN_LOCATION.mappedFeature());
private static final TypeAdapter<JsonObject> strictGsonObjectAdapter = new Gson().getAdapter(JsonObject.class);

4
app/server/appsmith-interfaces/src/main/java/com/appsmith/external/helpers/PluginUtils.java vendored
View File

@ -11,6 +11,7 @@ import com.appsmith.external.models.Endpoint;
import com.appsmith.external.models.Param;
import com.appsmith.external.models.Property;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.StreamReadFeature;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.extern.slf4j.Slf4j;
@ -46,7 +47,8 @@ import static com.appsmith.external.constants.CommonFieldName.VALUE;
@Slf4j
public class PluginUtils {
private static final ObjectMapper objectMapper = new ObjectMapper();
private static final ObjectMapper objectMapper =
new ObjectMapper().enable(StreamReadFeature.INCLUDE_SOURCE_IN_LOCATION.mappedFeature());
public static final TypeReference<String> STRING_TYPE = new TypeReference<>() {
@Override
public Type getType() {

3
app/server/appsmith-interfaces/src/main/java/com/appsmith/external/helpers/restApiUtils/helpers/DataUtils.java vendored
View File

@ -8,6 +8,7 @@ import com.appsmith.external.models.ActionConfiguration;
import com.appsmith.external.models.ApiContentType;
import com.appsmith.external.models.Property;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.StreamReadFeature;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
@ -76,7 +77,7 @@ public class DataUtils {
}
public DataUtils() {
this.objectMapper = new ObjectMapper();
this.objectMapper = new ObjectMapper().enable(StreamReadFeature.INCLUDE_SOURCE_IN_LOCATION.mappedFeature());
this.objectMapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES);
}

27
app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/ActionConfiguration.java vendored
View File

@ -1,6 +1,10 @@
package com.appsmith.external.models;
import com.appsmith.external.converters.HttpMethodConverter;
import com.appsmith.external.views.FromRequest;
import com.appsmith.external.views.Git;
import com.appsmith.external.views.Views;
import com.fasterxml.jackson.annotation.JsonView;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import com.google.gson.annotations.JsonAdapter;
@ -43,32 +47,51 @@ public class ActionConfiguration implements AppsmithDomain, ExecutableConfigurat
*/
@Range(min = MIN_TIMEOUT_VALUE, max = MAX_TIMEOUT_VALUE, message = TIMEOUT_OUT_OF_RANGE_MESSAGE)
@JsonView({Views.Public.class, FromRequest.class, Git.class})
Integer timeoutInMillisecond;
@JsonView({Views.Public.class, FromRequest.class, Git.class})
PaginationType paginationType = PaginationType.NONE;
// API fields
@JsonView({Views.Public.class, FromRequest.class, Git.class})
String path;
@JsonView({Views.Public.class, FromRequest.class, Git.class})
List<Property> headers;
@JsonView({Views.Public.class, FromRequest.class, Git.class})
List<Property> autoGeneratedHeaders;
@JsonView({Views.Public.class, FromRequest.class, Git.class})
Boolean encodeParamsToggle = true;
@JsonView({Views.Public.class, FromRequest.class, Git.class})
List<Property> queryParameters;
@JsonView({Views.Public.class, FromRequest.class, Git.class})
String body;
// For form-data input instead of json use the following
@JsonView({Views.Public.class, FromRequest.class, Git.class})
List<Property> bodyFormData;
// For route parameters extracted from rapid-api
@JsonView({Views.Public.class, FromRequest.class, Git.class})
List<Property> routeParameters;
// All the following adapters are registered so that we can serialize between enum HttpMethod,
// and what is now the class HttpMethod
@JsonSerialize(using = HttpMethodConverter.HttpMethodSerializer.class)
@JsonDeserialize(using = HttpMethodConverter.HttpMethodDeserializer.class)
@JsonAdapter(HttpMethodConverter.class)
@JsonView({Views.Public.class, FromRequest.class, Git.class})
HttpMethod httpMethod;
@JsonView({Views.Public.class, FromRequest.class, Git.class})
HttpProtocol httpVersion;
// Paginated API fields
@JsonView({Views.Public.class, FromRequest.class, Git.class})
String next;
@JsonView({Views.Public.class, FromRequest.class, Git.class})
String prev;
/**
@ -78,6 +101,7 @@ public class ActionConfiguration implements AppsmithDomain, ExecutableConfigurat
* cyclic dependency errors.
*/
@Transient
@JsonView({Views.Internal.class})
Set<String> selfReferencingDataPaths = new HashSet<>();
// DB action fields
@ -85,6 +109,7 @@ public class ActionConfiguration implements AppsmithDomain, ExecutableConfigurat
// JS action fields
// Body, the raw class data, is shared with API type actions
// Represents the values that need to be
@JsonView({Views.Public.class, FromRequest.class, Git.class})
List<JSValue> jsArguments;
// This property is being retained right now so that Git does not see commit changes, do not use
@Deprecated(forRemoval = true)
@ -97,12 +122,14 @@ public class ActionConfiguration implements AppsmithDomain, ExecutableConfigurat
* They will have to represented in a key-value format where the plugin
* understands what the keys stand for.
*/
@JsonView({Views.Public.class, FromRequest.class, Git.class})
List<Property> pluginSpecifiedTemplates;
/*
* After porting plugins to UQI, we should be able to use a map for referring to form data
* instead of a list of properties
*/
@JsonView({Views.Public.class, FromRequest.class, Git.class})
Map<String, Object> formData;
@Transient

5
app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/BaseDomain.java vendored
View File

@ -2,6 +2,7 @@ package com.appsmith.external.models;
import com.appsmith.external.helpers.Identifiable;
import com.appsmith.external.views.FromRequest;
import com.appsmith.external.views.Git;
import com.appsmith.external.views.Views;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonView;
@ -38,7 +39,7 @@ public abstract class BaseDomain implements Persistable<String>, AppsmithDomain,
private static final long serialVersionUID = 7459916000501322517L;
@Id
@JsonView({Views.Public.class, FromRequest.class})
@JsonView({Views.Public.class, FromRequest.class, Git.class})
private String id;
@JsonView(Views.Internal.class)
@ -94,7 +95,7 @@ public abstract class BaseDomain implements Persistable<String>, AppsmithDomain,
// This field will only be used for git related functionality to sync the action object across different instances.
// This field will be deprecated once we move to the new git sync implementation.
@JsonProperty(access = JsonProperty.Access.READ_ONLY)
@JsonView(Views.Internal.class)
@JsonView({Views.Internal.class, Git.class})
String gitSyncId;
public void sanitiseToExportDBObject() {

7
app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/Datasource.java vendored
View File

@ -1,6 +1,7 @@
package com.appsmith.external.models;
import com.appsmith.external.views.FromRequest;
import com.appsmith.external.views.Git;
import com.appsmith.external.views.Views;
import com.fasterxml.jackson.annotation.JsonView;
import lombok.Getter;
@ -29,10 +30,10 @@ public class Datasource extends BranchAwareDomain {
@Transient
public static final String DEFAULT_NAME_PREFIX = "Untitled datasource";
@JsonView({Views.Public.class, FromRequest.class})
@JsonView({Views.Public.class, FromRequest.class, Git.class})
String name;
@JsonView({Views.Public.class, FromRequest.class})
@JsonView({Views.Public.class, FromRequest.class, Git.class})
String pluginId;
// name of the plugin. used to log analytics events where pluginName is a required attribute
@ -77,7 +78,7 @@ public class Datasource extends BranchAwareDomain {
* while trying set createdAt and updatedAt properties on the null object
*/
@Transient
@JsonView(Views.Internal.class)
@JsonView({Views.Internal.class, Git.class})
Boolean isAutoGenerated = false;
/*

1
app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/DatasourceStorage.java vendored
View File

@ -143,6 +143,7 @@ public class DatasourceStorage extends BaseDomain {
this.setIsRecentlyCreated(null);
}
@JsonView({Views.Internal.class})
public boolean isEmbedded() {
/**
* We cannot just rely on datasourceId == null check because it will always be true for all cases when the

34
app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/ce/ActionCE_DTO.java vendored
View File

@ -17,6 +17,7 @@ import com.appsmith.external.models.PluginType;
import com.appsmith.external.models.Policy;
import com.appsmith.external.models.Property;
import com.appsmith.external.views.FromRequest;
import com.appsmith.external.views.Git;
import com.appsmith.external.views.Views;
import com.fasterxml.jackson.annotation.JsonFormat;
import com.fasterxml.jackson.annotation.JsonView;
@ -65,26 +66,26 @@ public class ActionCE_DTO implements Identifiable, Executable {
@JsonView({Views.Public.class, FromRequest.class})
String pluginId;
@JsonView({Views.Public.class, FromRequest.class})
@JsonView({Views.Public.class, FromRequest.class, Git.class})
String name;
// The FQN for an action will also include any collection it is a part of as collectionName.actionName
@JsonView({Views.Public.class, FromRequest.class})
@JsonView({Views.Public.class, FromRequest.class, Git.class})
String fullyQualifiedName;
@JsonView({Views.Public.class, FromRequest.class})
@JsonView({Views.Public.class, FromRequest.class, Git.class})
Datasource datasource;
@JsonView({Views.Public.class, FromRequest.class})
@JsonView({Views.Public.class, FromRequest.class, Git.class})
String pageId;
@JsonView({Views.Public.class, FromRequest.class})
@JsonView({Views.Public.class, FromRequest.class, Git.class})
CreatorContextType contextType;
@JsonView({Views.Public.class, FromRequest.class})
@JsonView({Views.Public.class, FromRequest.class, Git.class})
String collectionId;
@JsonView({Views.Public.class, FromRequest.class})
@JsonView({Views.Public.class, FromRequest.class, Git.class})
ActionConfiguration actionConfiguration;
// this attribute carries error messages while processing the actionCollection
@ -92,17 +93,17 @@ public class ActionCE_DTO implements Identifiable, Executable {
@JsonView(Views.Public.class)
List<ErrorDTO> errorReports;
@JsonView({Views.Public.class, FromRequest.class})
@JsonView({Views.Public.class, FromRequest.class, Git.class})
Boolean executeOnLoad;
@JsonView({Views.Public.class, FromRequest.class})
@JsonView({Views.Public.class, FromRequest.class, Git.class})
Boolean clientSideExecution;
/*
* This is a list of fields specified by the client to signify which fields have dynamic bindings in them.
* TODO: The server can use this field to simplify our Mustache substitutions in the future
*/
@JsonView({Views.Public.class, FromRequest.class})
@JsonView({Views.Public.class, FromRequest.class, Git.class})
List<Property> dynamicBindingPathList;
@JsonView(Views.Public.class)
@ -123,10 +124,10 @@ public class ActionCE_DTO implements Identifiable, Executable {
@JsonView(Views.Internal.class)
String cacheResponse;
@JsonView(Views.Internal.class)
@JsonView({Views.Internal.class, Git.class})
Boolean userSetOnLoad = false;
@JsonView({Views.Public.class, FromRequest.class})
@JsonView({Views.Public.class, FromRequest.class, Git.class})
Boolean confirmBeforeExecute = false;
@Transient
@ -175,7 +176,7 @@ public class ActionCE_DTO implements Identifiable, Executable {
ActionCreationSourceTypeEnum source;
@Override
@JsonView(Views.Public.class)
@JsonView({Views.Internal.class})
public String getValidName() {
if (this.fullyQualifiedName == null) {
return this.name;
@ -185,6 +186,7 @@ public class ActionCE_DTO implements Identifiable, Executable {
}
@Override
@JsonView({Views.Internal.class})
public Set<String> getExecutableNames() {
String validName = this.getValidName();
HashSet<String> validNames = new HashSet<>();
@ -230,6 +232,7 @@ public class ActionCE_DTO implements Identifiable, Executable {
}
@Override
@JsonView({Views.Internal.class})
public Set<String> getSelfReferencingDataPaths() {
if (this.getActionConfiguration() == null) {
return new HashSet<>();
@ -238,26 +241,31 @@ public class ActionCE_DTO implements Identifiable, Executable {
}
@Override
@JsonView({Views.Internal.class})
public ActionConfiguration getExecutableConfiguration() {
return this.getActionConfiguration();
}
@Override
@JsonView({Views.Internal.class})
public String getConfigurationPath() {
return this.getUserExecutableName() + ".actionConfiguration";
}
@Override
@JsonView({Views.Internal.class})
public String getCompleteDynamicBindingPath(String fieldPath) {
return this.getConfigurationPath() + "." + fieldPath;
}
@Override
@JsonView({Views.Internal.class})
public boolean hasExtractableBinding() {
return PluginType.JS.equals(this.getPluginType());
}
@Override
@JsonView({Views.Internal.class})
public DslExecutableDTO getDslExecutable() {
DslExecutableDTO dslExecutableDTO = new DslExecutableDTO();

4
app/server/appsmith-interfaces/src/main/java/com/appsmith/external/plugins/BasePlugin.java vendored
View File

@ -1,12 +1,14 @@
package com.appsmith.external.plugins;
import com.fasterxml.jackson.core.StreamReadFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.pf4j.Plugin;
import org.pf4j.PluginWrapper;
public abstract class BasePlugin extends Plugin {
protected static final ObjectMapper objectMapper = new ObjectMapper();
protected static final ObjectMapper objectMapper =
new ObjectMapper().enable(StreamReadFeature.INCLUDE_SOURCE_IN_LOCATION.mappedFeature());
public BasePlugin(PluginWrapper wrapper) {
super(wrapper);

3
app/server/appsmith-interfaces/src/main/java/com/appsmith/external/services/ce/FilterDataServiceCE.java vendored
View File

@ -8,6 +8,7 @@ import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginError;
import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginException;
import com.appsmith.external.models.Condition;
import com.appsmith.external.models.UQIDataFilterParams;
import com.fasterxml.jackson.core.StreamReadFeature;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
@ -97,7 +98,7 @@ public class FilterDataServiceCE implements IFilterDataServiceCE {
public FilterDataServiceCE() {
objectMapper = new ObjectMapper();
objectMapper = new ObjectMapper().enable(StreamReadFeature.INCLUDE_SOURCE_IN_LOCATION.mappedFeature());
try {
connection = DriverManager.getConnection(URL);

3
app/server/appsmith-interfaces/src/main/java/com/appsmith/external/views/Git.java vendored Normal file
View File

@ -0,0 +1,3 @@
package com.appsmith.external.views;
public interface Git {}

66
app/server/appsmith-interfaces/src/main/java/com/appsmith/util/JSONPrettyPrinter.java Normal file
View File

@ -0,0 +1,66 @@
package com.appsmith.util;
import com.fasterxml.jackson.core.util.DefaultIndenter;
import com.fasterxml.jackson.core.util.DefaultPrettyPrinter;
import com.fasterxml.jackson.core.util.Separators;
import org.springframework.stereotype.Component;
@Component
public class JSONPrettyPrinter extends DefaultPrettyPrinter {
public JSONPrettyPrinter() {
super();
/*
[
a,
b
]
instead of
[ a, b ]
*/
this._arrayIndenter = DefaultIndenter.SYSTEM_LINEFEED_INSTANCE;
/*
{
k1: v1,
k2: v2
}
instead of
{ k1: v1, k2: v2 }
*/
this._objectIndenter = DefaultIndenter.SYSTEM_LINEFEED_INSTANCE;
// {} instead of { }
this._objectEmptySeparator = "";
// [] instead of [ ]
this._arrayEmptySeparator = "";
// { k: v } instead of { k : v }
this._objectFieldValueSeparatorWithSpaces = _separators.getObjectFieldValueSeparator() + " ";
this._separators = this._separators
.withObjectFieldValueSpacing(Separators.Spacing.AFTER)
.withObjectEmptySeparator("")
.withArrayEmptySeparator("");
}
public JSONPrettyPrinter(DefaultPrettyPrinter base) {
super(base);
this._arrayIndenter = DefaultIndenter.SYSTEM_LINEFEED_INSTANCE;
this._objectIndenter = DefaultIndenter.SYSTEM_LINEFEED_INSTANCE;
this._objectEmptySeparator = "";
this._arrayEmptySeparator = "";
this._objectFieldValueSeparatorWithSpaces = _separators.getObjectFieldValueSeparator() + " ";
this._separators = this._separators
.withObjectFieldValueSpacing(Separators.Spacing.AFTER)
.withObjectEmptySeparator("")
.withArrayEmptySeparator("");
}
@Override
public JSONPrettyPrinter createInstance() {
if (getClass() != JSONPrettyPrinter.class) {
throw new IllegalStateException(
"Failed `createInstance()`: " + getClass().getName() + " does not override method; it has to");
}
return new JSONPrettyPrinter(this);
}
}

37
app/server/appsmith-interfaces/src/main/java/com/appsmith/util/SerializationUtils.java
View File

@ -5,9 +5,13 @@ import com.appsmith.external.converters.ISOStringToInstantConverter;
import com.appsmith.external.models.DatasourceStructure;
import com.appsmith.external.views.Views;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.core.PrettyPrinter;
import com.fasterxml.jackson.core.StreamReadFeature;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.MapperFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import com.fasterxml.jackson.databind.cfg.JsonNodeFeature;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import org.springframework.boot.autoconfigure.gson.GsonBuilderCustomizer;
import org.springframework.http.HttpMethod;
@ -24,12 +28,29 @@ public class SerializationUtils {
HTTP_METHOD_MODULE = new HttpMethodConverter.HttpMethodModule();
}
public static ObjectMapper configureObjectMapper(ObjectMapper objectMapper) {
objectMapper.registerModule(JAVA_TIME_MODULE);
objectMapper.registerModule(HTTP_METHOD_MODULE);
objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
objectMapper.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false);
objectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL);
public static ObjectMapper getBasicObjectMapper(PrettyPrinter prettyPrinter) {
ObjectMapper objectMapper = new ObjectMapper();
objectMapper
.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false)
.enable(StreamReadFeature.INCLUDE_SOURCE_IN_LOCATION.mappedFeature())
.registerModules(JAVA_TIME_MODULE, HTTP_METHOD_MODULE)
.setSerializationInclusion(JsonInclude.Include.NON_NULL);
if (prettyPrinter != null) {
objectMapper
.setDefaultPrettyPrinter(prettyPrinter)
.configure(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS, true)
.configure(JsonNodeFeature.WRITE_PROPERTIES_SORTED, true)
.configure(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY, true)
.enable(SerializationFeature.INDENT_OUTPUT);
}
return objectMapper;
}
public static ObjectMapper getDefaultObjectMapper(PrettyPrinter prettyPrinter) {
ObjectMapper objectMapper = getBasicObjectMapper(prettyPrinter);
/*
Setting Views.Public as default view class for the serializer.
@ -41,10 +62,6 @@ public class SerializationUtils {
return objectMapper;
}
public static ObjectMapper getDefaultObjectMapper() {
return configureObjectMapper(new ObjectMapper());
}
public static GsonBuilderCustomizer typeAdapterRegistration() {
return builder -> {
builder.registerTypeAdapter(Instant.class, new ISOStringToInstantConverter());

4
app/server/appsmith-plugins/firestorePlugin/src/main/java/com/external/utils/WhereConditionUtils.java vendored
View File

@ -6,6 +6,7 @@ import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginError;
import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginException;
import com.appsmith.external.helpers.DataTypeStringUtils;
import com.external.plugins.exceptions.FirestoreErrorMessages;
import com.fasterxml.jackson.core.StreamReadFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.cloud.firestore.FieldPath;
import com.google.cloud.firestore.Query;
@ -19,7 +20,8 @@ import java.util.List;
public class WhereConditionUtils {
protected static final ObjectMapper objectMapper = new ObjectMapper();
protected static final ObjectMapper objectMapper =
new ObjectMapper().enable(StreamReadFeature.INCLUDE_SOURCE_IN_LOCATION.mappedFeature());
public static Query applyWhereConditional(Query query, String strPath, String operatorString, String strValue)
throws AppsmithPluginException {

4
app/server/appsmith-plugins/googleSheetsPlugin/src/main/java/com/external/config/GetDatasourceMetadataMethod.java vendored
View File

@ -6,6 +6,7 @@ import com.appsmith.external.models.DatasourceConfiguration;
import com.appsmith.external.models.Property;
import com.appsmith.util.WebClientUtils;
import com.external.constants.FieldName;
import com.fasterxml.jackson.core.StreamReadFeature;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.extern.slf4j.Slf4j;
@ -28,7 +29,8 @@ import static org.springframework.util.StringUtils.hasLength;
@Slf4j
public class GetDatasourceMetadataMethod {
protected static final ObjectMapper objectMapper = new ObjectMapper();
protected static final ObjectMapper objectMapper =
new ObjectMapper().enable(StreamReadFeature.INCLUDE_SOURCE_IN_LOCATION.mappedFeature());
public static Mono<DatasourceConfiguration> getDatasourceMetadata(DatasourceConfiguration datasourceConfiguration) {

4
app/server/appsmith-plugins/graphqlPlugin/src/main/java/com/external/utils/GraphQLDataTypeUtils.java vendored
View File

@ -3,6 +3,7 @@ package com.external.utils;
import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginError;
import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginException;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.StreamReadFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import graphql.parser.InvalidSyntaxException;
import graphql.parser.Parser;
@ -19,7 +20,8 @@ import static com.appsmith.external.helpers.SmartSubstitutionHelper.APPSMITH_SUB
public class GraphQLDataTypeUtils {
public static final String GRAPHQL_BODY_ENDS_WITH_PARAM_REGEX = "[\\w\\W]+:$";
public static final ObjectMapper objectMapper = new ObjectMapper();
public static final ObjectMapper objectMapper =
new ObjectMapper().enable(StreamReadFeature.INCLUDE_SOURCE_IN_LOCATION.mappedFeature());
public static String smartlyReplaceGraphQLQueryBodyPlaceholderWithValue(
String queryBody, String replacement, List<Map.Entry<String, String>> insertedParams) {

4
app/server/appsmith-plugins/mongoPlugin/src/main/java/com/external/plugins/commands/MongoCommand.java vendored
View File

@ -6,6 +6,7 @@ import com.appsmith.external.models.ActionConfiguration;
import com.appsmith.external.models.DatasourceStructure;
import com.external.plugins.exceptions.MongoPluginError;
import com.external.plugins.exceptions.MongoPluginErrorMessages;
import com.fasterxml.jackson.core.StreamReadFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.Getter;
import lombok.NoArgsConstructor;
@ -32,7 +33,8 @@ import static com.external.plugins.constants.FieldName.COLLECTION;
public abstract class MongoCommand {
String collection;
List<String> fieldNamesWithNoConfiguration;
protected static final ObjectMapper objectMapper = new ObjectMapper();
protected static final ObjectMapper objectMapper =
new ObjectMapper().enable(StreamReadFeature.INCLUDE_SOURCE_IN_LOCATION.mappedFeature());
public MongoCommand(ActionConfiguration actionConfiguration) {

4
app/server/appsmith-plugins/mysqlPlugin/src/main/java/com/external/plugins/datatypes/MySQLDateTimeType.java vendored
View File

@ -5,6 +5,7 @@ import com.appsmith.external.datatypes.AppsmithType;
import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginError;
import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginException;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.StreamReadFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import reactor.core.Exceptions;
@ -15,7 +16,8 @@ import java.time.format.DateTimeParseException;
import java.util.regex.Matcher;
public class MySQLDateTimeType implements AppsmithType {
private static final ObjectMapper objectMapper = new ObjectMapper();
private static final ObjectMapper objectMapper =
new ObjectMapper().enable(StreamReadFeature.INCLUDE_SOURCE_IN_LOCATION.mappedFeature());
@Override
public boolean test(String s) {

5
app/server/appsmith-plugins/postgresPlugin/pom.xml
View File

@ -49,6 +49,11 @@
<version>${testcontainers.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.assertj</groupId>
<artifactId>assertj-core</artifactId>
<scope>test</scope>
</dependency>
</dependencies>

16
app/server/appsmith-plugins/postgresPlugin/src/test/java/com/external/plugins/PostgresPluginTest.java vendored
View File

@ -21,12 +21,14 @@ import com.appsmith.external.models.SSLDetails;
import com.appsmith.external.services.SharedConfig;
import com.external.plugins.exceptions.PostgresErrorMessages;
import com.external.plugins.exceptions.PostgresPluginError;
import com.fasterxml.jackson.core.StreamReadFeature;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.JsonNodeType;
import com.fasterxml.jackson.databind.node.NullNode;
import com.zaxxer.hikari.HikariDataSource;
import org.junit.jupiter.api.Assertions;
import org.assertj.core.api.Assertions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.testcontainers.containers.PostgreSQLContainer;
@ -305,7 +307,9 @@ public class PostgresPluginTest {
Mono<HikariDataSource> dsConnectionMono = pluginExecutor.datasourceCreate(dsConfig);
StepVerifier.create(dsConnectionMono)
.assertNext(Assertions::assertNotNull)
.assertNext(value -> {
Assertions.assertThat(value).isNotNull();
})
.verifyComplete();
}
@ -383,6 +387,7 @@ public class PostgresPluginTest {
assertArrayEquals(
new String[] {"user_id"},
new ObjectMapper()
.enable(StreamReadFeature.INCLUDE_SOURCE_IN_LOCATION.mappedFeature())
.convertValue(node, LinkedHashMap.class)
.keySet()
.toArray());
@ -448,7 +453,7 @@ public class PostgresPluginTest {
assertEquals(
"1 years 5 mons 0 days 2 hours 0 mins 0.0 secs",
node.get("interval1").asText());
assertTrue(node.get("spouse_dob").isNull());
Assertions.assertThat(node.get("spouse_dob")).isEqualTo(NullNode.getInstance());
// Check the order of the columns.
assertArrayEquals(
@ -469,6 +474,7 @@ public class PostgresPluginTest {
"rating"
},
new ObjectMapper()
.enable(StreamReadFeature.INCLUDE_SOURCE_IN_LOCATION.mappedFeature())
.convertValue(node, LinkedHashMap.class)
.keySet()
.toArray());
@ -777,6 +783,7 @@ public class PostgresPluginTest {
"rating"
},
new ObjectMapper()
.enable(StreamReadFeature.INCLUDE_SOURCE_IN_LOCATION.mappedFeature())
.convertValue(node, LinkedHashMap.class)
.keySet()
.toArray());
@ -854,6 +861,7 @@ public class PostgresPluginTest {
"rating"
},
new ObjectMapper()
.enable(StreamReadFeature.INCLUDE_SOURCE_IN_LOCATION.mappedFeature())
.convertValue(node, LinkedHashMap.class)
.keySet()
.toArray());
@ -943,6 +951,7 @@ public class PostgresPluginTest {
"rating"
},
new ObjectMapper()
.enable(StreamReadFeature.INCLUDE_SOURCE_IN_LOCATION.mappedFeature())
.convertValue(node, LinkedHashMap.class)
.keySet()
.toArray());
@ -1617,6 +1626,7 @@ public class PostgresPluginTest {
assertArrayEquals(
new String[] {"numeric_string"},
new ObjectMapper()
.enable(StreamReadFeature.INCLUDE_SOURCE_IN_LOCATION.mappedFeature())
.convertValue(node, LinkedHashMap.class)
.keySet()
.toArray());

4
app/server/appsmith-plugins/saasPlugin/src/main/java/com/external/plugins/SaasPlugin.java vendored
View File

@ -18,6 +18,7 @@ import com.external.plugins.exceptions.SaaSErrorMessages;
import com.external.plugins.exceptions.SaaSPluginError;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.StreamReadFeature;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.MapperFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
@ -60,7 +61,8 @@ public class SaasPlugin extends BasePlugin {
// Setting max content length. This would've been coming from `spring.codec.max-in-memory-size` property if the
// `WebClient` instance was loaded as an auto-wired bean.
private final ExchangeStrategies EXCHANGE_STRATEGIES;
private final ObjectMapper saasObjectMapper = new ObjectMapper();
private final ObjectMapper saasObjectMapper =
new ObjectMapper().enable(StreamReadFeature.INCLUDE_SOURCE_IN_LOCATION.mappedFeature());
public SaasPluginExecutor(SharedConfig sharedConfig) {
this.sharedConfig = sharedConfig;

2
app/server/appsmith-server/src/main/java/com/appsmith/server/ServerApplication.java
View File

@ -24,11 +24,13 @@ import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
import org.springframework.boot.builder.SpringApplicationBuilder;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.scheduling.annotation.EnableScheduling;
import java.time.Duration;
@SpringBootApplication
@ComponentScan({"com.appsmith"})
@EnableScheduling
@Slf4j
public class ServerApplication {

9
app/server/appsmith-server/src/main/java/com/appsmith/server/configurations/CommonConfig.java
View File

@ -1,6 +1,8 @@
package com.appsmith.server.configurations;
import com.appsmith.util.JSONPrettyPrinter;
import com.appsmith.util.SerializationUtils;
import com.fasterxml.jackson.core.PrettyPrinter;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
@ -90,9 +92,14 @@ public class CommonConfig {
}
}
@Bean
public PrettyPrinter prettyPrinter() {
return new JSONPrettyPrinter();
}
@Bean
public ObjectMapper objectMapper() {
return SerializationUtils.getDefaultObjectMapper();
return SerializationUtils.getDefaultObjectMapper(null);
}
@Bean

3
app/server/appsmith-server/src/main/java/com/appsmith/server/configurations/ProjectProperties.java
View File

@ -16,13 +16,12 @@ import java.nio.file.Paths;
@Slf4j
public class ProjectProperties {
private static final String INFO_JSON_PATH = "/opt/appsmith/info.json";
private static final ObjectMapper objectMapper = new ObjectMapper();
public static final String EDITION = "CE";
private String version = "UNKNOWN";
private String commitSha = "UNKNOWN";
public ProjectProperties() {
public ProjectProperties(ObjectMapper objectMapper) {
try {
Path infoJsonPath = Paths.get(INFO_JSON_PATH);
if (Files.exists(infoJsonPath)) {

68
app/server/appsmith-server/src/main/java/com/appsmith/server/domains/Application.java
View File

@ -1,6 +1,7 @@
package com.appsmith.server.domains;
import com.appsmith.external.models.BaseDomain;
import com.appsmith.external.views.Git;
import com.appsmith.external.views.Views;
import com.appsmith.server.constants.ArtifactType;
import com.appsmith.server.dtos.CustomJSLibContextDTO;
@ -50,7 +51,7 @@ public class Application extends BaseDomain implements Artifact {
@JsonView(Views.Public.class)
Boolean isPublic = false;
@JsonView(Views.Public.class)
@JsonView({Views.Public.class, Git.class})
List<ApplicationPage> pages;
@JsonView(Views.Internal.class)
@ -61,7 +62,7 @@ public class Application extends BaseDomain implements Artifact {
Boolean viewMode = false;
@Transient
@JsonView(Views.Public.class)
@JsonView({Views.Public.class, Git.class})
boolean appIsExample = false;
@Transient
@ -71,22 +72,22 @@ public class Application extends BaseDomain implements Artifact {
@JsonView(Views.Internal.class)
String clonedFromApplicationId;
@JsonView(Views.Internal.class)
@JsonView({Views.Internal.class, Git.class})
ApplicationDetail unpublishedApplicationDetail;
@JsonView(Views.Internal.class)
ApplicationDetail publishedApplicationDetail;
@JsonView(Views.Public.class)
@JsonView({Views.Public.class, Git.class})
String color;
@JsonView(Views.Public.class)
@JsonView({Views.Public.class, Git.class})
String icon;
@JsonView(Views.Public.class)
private String slug;
@JsonView(Views.Internal.class)
@JsonView({Views.Internal.class, Git.class})
AppLayout unpublishedAppLayout;
@JsonView(Views.Internal.class)
@ -106,7 +107,7 @@ public class Application extends BaseDomain implements Artifact {
Instant lastDeployedAt; // when this application was last deployed
@JsonProperty(access = JsonProperty.Access.READ_ONLY)
@JsonView(Views.Public.class)
@JsonView({Views.Public.class, Git.class})
Integer evaluationVersion;
/**
@ -116,7 +117,7 @@ public class Application extends BaseDomain implements Artifact {
* so that they can update their application.
* Once updated, we should set applicationVersion to latest version as well.
*/
@JsonView(Views.Public.class)
@JsonView({Views.Public.class, Git.class})
Integer applicationVersion;
/**
@ -127,9 +128,10 @@ public class Application extends BaseDomain implements Artifact {
@JsonView(Views.Internal.class)
Instant lastEditedAt;
@JsonView(Views.Public.class)
@JsonView({Views.Public.class, Git.class})
EmbedSetting embedSetting;
@JsonView({Views.Public.class, Git.class})
Boolean collapseInvisibleWidgets;
/**
@ -171,10 +173,10 @@ public class Application extends BaseDomain implements Artifact {
// To convey current schema version for client and server. This will be used to check if we run the migration
// between 2 commits if the application is connected to git
@JsonView(Views.Internal.class)
@JsonView({Views.Internal.class, Git.class})
Integer clientSchemaVersion;
@JsonView(Views.Internal.class)
@JsonView({Views.Internal.class, Git.class})
Integer serverSchemaVersion;
@JsonView(Views.Internal.class)
@ -351,7 +353,7 @@ public class Application extends BaseDomain implements Artifact {
}
@Override
@JsonView(Views.Internal.class)
@JsonView({Views.Internal.class})
public ArtifactType getArtifactType() {
return ArtifactType.APPLICATION;
}
@ -360,7 +362,7 @@ public class Application extends BaseDomain implements Artifact {
@NoArgsConstructor
@AllArgsConstructor
public static class AppLayout implements Serializable {
@JsonView(Views.Public.class)
@JsonView({Views.Public.class, Git.class})
Type type;
public enum Type {
@ -378,13 +380,13 @@ public class Application extends BaseDomain implements Artifact {
@Data
public static class EmbedSetting {
@JsonView(Views.Public.class)
@JsonView({Views.Public.class, Git.class})
private String height;
@JsonView(Views.Public.class)
@JsonView({Views.Public.class, Git.class})
private String width;
@JsonView(Views.Public.class)
@JsonView({Views.Public.class, Git.class})
private Boolean showNavigationBar;
}
@ -393,31 +395,31 @@ public class Application extends BaseDomain implements Artifact {
*/
@Data
public static class NavigationSetting {
@JsonView(Views.Public.class)
@JsonView({Views.Public.class, Git.class})
private Boolean showNavbar;
@JsonView(Views.Public.class)
@JsonView({Views.Public.class, Git.class})
private String orientation;
@JsonView(Views.Public.class)
@JsonView({Views.Public.class, Git.class})
private String navStyle;
@JsonView(Views.Public.class)
@JsonView({Views.Public.class, Git.class})
private String position;
@JsonView(Views.Public.class)
@JsonView({Views.Public.class, Git.class})
private String itemStyle;
@JsonView(Views.Public.class)
@JsonView({Views.Public.class, Git.class})
private String colorStyle;
@JsonView(Views.Public.class)
@JsonView({Views.Public.class, Git.class})
private String logoAssetId;
@JsonView(Views.Public.class)
@JsonView({Views.Public.class, Git.class})
private String logoConfiguration;
@JsonView(Views.Public.class)
@JsonView({Views.Public.class, Git.class})
private Boolean showSignIn;
}
@ -427,7 +429,7 @@ public class Application extends BaseDomain implements Artifact {
@Data
@NoArgsConstructor
public static class AppPositioning {
@JsonView(Views.Public.class)
@JsonView({Views.Public.class, Git.class})
Type type;
public AppPositioning(Type type) {
@ -445,25 +447,25 @@ public class Application extends BaseDomain implements Artifact {
@NoArgsConstructor
public static class ThemeSetting {
@JsonView(Views.Public.class)
@JsonView({Views.Public.class, Git.class})
private String accentColor;
@JsonView(Views.Public.class)
@JsonView({Views.Public.class, Git.class})
private String borderRadius;
@JsonView(Views.Public.class)
@JsonView({Views.Public.class, Git.class})
private float sizing = 1;
@JsonView(Views.Public.class)
@JsonView({Views.Public.class, Git.class})
private float density = 1;
@JsonView(Views.Public.class)
@JsonView({Views.Public.class, Git.class})
private String fontFamily;
@JsonView(Views.Public.class)
@JsonView({Views.Public.class, Git.class})
Type colorMode;
@JsonView(Views.Public.class)
@JsonView({Views.Public.class, Git.class})
IconStyle iconStyle;
public ThemeSetting(Type colorMode) {

5
app/server/appsmith-server/src/main/java/com/appsmith/server/domains/ApplicationPage.java
View File

@ -1,5 +1,6 @@
package com.appsmith.server.domains;
import com.appsmith.external.views.Git;
import com.appsmith.external.views.Views;
import com.fasterxml.jackson.annotation.JsonView;
import lombok.EqualsAndHashCode;
@ -16,10 +17,10 @@ import org.springframework.data.annotation.Transient;
@EqualsAndHashCode
public class ApplicationPage {
@JsonView({Views.Public.class, Views.Export.class})
@JsonView({Views.Public.class, Views.Export.class, Git.class})
String id;
@JsonView(Views.Public.class)
@JsonView({Views.Public.class, Git.class})
Boolean isDefault;
@Transient

10
app/server/appsmith-server/src/main/java/com/appsmith/server/domains/Layout.java
View File

@ -3,6 +3,7 @@ package com.appsmith.server.domains;
import com.appsmith.external.dtos.DslExecutableDTO;
import com.appsmith.external.exceptions.ErrorDTO;
import com.appsmith.external.models.Policy;
import com.appsmith.external.views.Git;
import com.appsmith.external.views.Views;
import com.appsmith.server.helpers.CollectionUtils;
import com.appsmith.server.helpers.CompareDslActionDTO;
@ -38,7 +39,7 @@ public class Layout {
@JsonView(Views.Internal.class)
Boolean viewMode = false;
@JsonView({Views.Public.class, Views.Export.class})
@JsonView({Views.Public.class, Views.Export.class, Git.class})
JSONObject dsl;
@JsonView(Views.Internal.class)
@ -74,13 +75,14 @@ public class Layout {
@JsonView(Views.Internal.class)
Boolean validOnPageLoadActions = TRUE;
@JsonView({Views.Public.class, Views.Export.class})
private String id;
/*
* These fields (except for `id`) only exist here because their removal will cause a huge diff on all layouts in
* git-connected applications. So, instead, we keep them, but defunct. For all other practical purposes, these
* fields (again, except for `id`) don't exist.
*/
@JsonView({Views.Public.class, Views.Export.class})
private String id;
// BEGIN DEFUNCT FIELDS
@Deprecated(forRemoval = true)
@Transient
@ -105,7 +107,7 @@ public class Layout {
* If view mode, the dsl returned should be the publishedDSL, else if the edit mode is on (view mode = false)
* the dsl returned should be JSONObject dsl
*/
@JsonView({Views.Public.class, Views.Export.class})
@JsonView({Views.Public.class, Views.Export.class, Git.class})
public JSONObject getDsl() {
return viewMode ? publishedDsl : dsl;
}

3
app/server/appsmith-server/src/main/java/com/appsmith/server/domains/NewPage.java
View File

@ -1,6 +1,7 @@
package com.appsmith.server.domains;
import com.appsmith.external.models.BranchAwareDomain;
import com.appsmith.external.views.Git;
import com.appsmith.external.views.Views;
import com.appsmith.server.dtos.PageDTO;
import com.fasterxml.jackson.annotation.JsonView;
@ -19,7 +20,7 @@ public class NewPage extends BranchAwareDomain implements Context {
@JsonView(Views.Public.class)
String applicationId;
@JsonView(Views.Public.class)
@JsonView({Views.Public.class, Git.class})
PageDTO unpublishedPage;
@JsonView(Views.Public.class)

7
app/server/appsmith-server/src/main/java/com/appsmith/server/domains/Theme.java
View File

@ -1,6 +1,7 @@
package com.appsmith.server.domains;
import com.appsmith.external.models.BaseDomain;
import com.appsmith.external.views.Git;
import com.appsmith.external.views.Views;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonView;
@ -24,11 +25,11 @@ public class Theme extends BaseDomain {
// name will be used internally to identify system themes for import, export application and theme migration
// it'll never change. We need to remove this from API response in future when FE uses displayName everywhere
@JsonView({Views.Public.class})
@JsonView({Views.Public.class, Git.class})
private String name;
// displayName will be visible to users. Users can set their own input when saving/customising a theme
@JsonView({Views.Public.class})
@JsonView({Views.Public.class, Git.class})
private String displayName;
@JsonView(Views.Public.class)
@ -47,7 +48,7 @@ public class Theme extends BaseDomain {
private Map<String, Object> stylesheet;
@JsonProperty("isSystemTheme") // manually setting property name to make sure it's compatible with Gson
@JsonView({Views.Public.class})
@JsonView({Views.Public.class, Git.class})
private boolean isSystemTheme = false; // should be false by default
@Data

3
app/server/appsmith-server/src/main/java/com/appsmith/server/domains/ce/ActionCollectionCE.java
View File

@ -3,6 +3,7 @@ package com.appsmith.server.domains.ce;
import com.appsmith.external.models.BranchAwareDomain;
import com.appsmith.external.models.CreatorContextType;
import com.appsmith.external.models.DefaultResources;
import com.appsmith.external.views.Git;
import com.appsmith.external.views.Views;
import com.appsmith.server.dtos.ActionCollectionDTO;
import com.fasterxml.jackson.annotation.JsonView;
@ -30,7 +31,7 @@ public class ActionCollectionCE extends BranchAwareDomain {
@JsonView(Views.Public.class)
String workspaceId;
@JsonView(Views.Public.class)
@JsonView({Views.Public.class, Git.class})
ActionCollectionDTO unpublishedCollection;
@JsonView(Views.Public.class)

10
app/server/appsmith-server/src/main/java/com/appsmith/server/domains/ce/CustomJSLibCE.java
View File

@ -1,8 +1,11 @@
package com.appsmith.server.domains.ce;
import com.appsmith.external.models.BranchAwareDomain;
import com.appsmith.external.views.Git;
import com.appsmith.external.views.Views;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonView;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
@ -21,31 +24,38 @@ import java.util.Set;
@FieldNameConstants
public class CustomJSLibCE extends BranchAwareDomain {
/* Library name */
@JsonView({Views.Public.class, Git.class})
String name;
/**
* This string is used to uniquely identify a given library. We expect this to be universally unique for a given
* JS library
*/
@JsonView({Views.Public.class, Git.class})
String uidString;
/**
* These are the namespaces under which the library functions reside. User would access lib methods like
* `accessor.method`
*/
@JsonView({Views.Public.class, Git.class})
Set<String> accessor;
/* Library UMD src url */
@JsonView({Views.Public.class, Git.class})
String url;
/* Library documentation page URL */
@JsonView({Views.Public.class, Git.class})
String docsUrl;
/* Library version */
@JsonView({Views.Public.class, Git.class})
String version;
/* `Tern` tool definitions - it defines the methods exposed by the library. It helps us with auto-complete
feature i.e. the function name showing up as suggestion when user has partially typed it. */
@JsonView({Views.Public.class, Git.class})
String defs;
@JsonCreator(mode = JsonCreator.Mode.PROPERTIES)

7
app/server/appsmith-server/src/main/java/com/appsmith/server/domains/ce/NewActionCE.java
View File

@ -6,6 +6,7 @@ import com.appsmith.external.models.BranchAwareDomain;
import com.appsmith.external.models.Datasource;
import com.appsmith.external.models.Documentation;
import com.appsmith.external.models.PluginType;
import com.appsmith.external.views.Git;
import com.appsmith.external.views.Views;
import com.fasterxml.jackson.annotation.JsonView;
import lombok.Getter;
@ -28,17 +29,17 @@ public class NewActionCE extends BranchAwareDomain {
@JsonView(Views.Public.class)
String workspaceId;
@JsonView(Views.Public.class)
@JsonView({Views.Public.class, Git.class})
PluginType pluginType;
@JsonView(Views.Public.class)
@JsonView({Views.Public.class, Git.class})
String pluginId;
@JsonView(Views.Public.class)
Documentation documentation; // Documentation for the template using which this action was created
// Action specific fields that are allowed to change between published and unpublished versions
@JsonView(Views.Public.class)
@JsonView({Views.Public.class, Git.class})
ActionDTO unpublishedAction;
@JsonView(Views.Public.class)

11
app/server/appsmith-server/src/main/java/com/appsmith/server/dtos/PageDTO.java
View File

@ -2,6 +2,7 @@ package com.appsmith.server.dtos;
import com.appsmith.external.models.DefaultResources;
import com.appsmith.external.models.Policy;
import com.appsmith.external.views.Git;
import com.appsmith.external.views.Views;
import com.appsmith.server.domains.Layout;
import com.fasterxml.jackson.annotation.JsonFormat;
@ -26,19 +27,19 @@ import java.util.Set;
public class PageDTO {
@Transient
@JsonView(Views.Public.class)
@JsonView({Views.Public.class})
private String id;
@JsonView({Views.Public.class, Views.Export.class})
@JsonView({Views.Public.class, Views.Export.class, Git.class})
String name;
@JsonView(Views.Public.class)
@JsonView({Views.Public.class})
String icon;
@JsonView(Views.Public.class)
String description;
@JsonView({Views.Public.class, Views.Export.class})
@JsonView({Views.Public.class, Views.Export.class, Git.class})
String slug;
@JsonView(Views.Public.class)
@ -48,7 +49,7 @@ public class PageDTO {
@JsonView(Views.Public.class)
String applicationId;
@JsonView({Views.Public.class, Views.Export.class})
@JsonView({Views.Public.class, Views.Export.class, Git.class})
List<Layout> layouts;
@Transient

14
app/server/appsmith-server/src/main/java/com/appsmith/server/dtos/ce/ActionCollectionCE_DTO.java
View File

@ -6,6 +6,7 @@ import com.appsmith.external.models.CreatorContextType;
import com.appsmith.external.models.DefaultResources;
import com.appsmith.external.models.JSValue;
import com.appsmith.external.models.PluginType;
import com.appsmith.external.views.Git;
import com.appsmith.external.views.Views;
import com.appsmith.server.constants.FieldName;
import com.appsmith.server.domains.ActionCollection;
@ -47,17 +48,17 @@ public class ActionCollectionCE_DTO {
@JsonView(Views.Public.class)
String workspaceId;
@JsonView(Views.Public.class)
@JsonView({Views.Public.class, Git.class})
String name;
@JsonView(Views.Public.class)
@JsonView({Views.Public.class, Git.class})
String pageId;
@JsonView(Views.Public.class)
@JsonView({Views.Public.class, Git.class})
CreatorContextType contextType;
// This field will only be populated if this collection is bound to one plugin (eg: JS)
@JsonView(Views.Public.class)
@JsonView({Views.Public.class, Git.class})
String pluginId;
// this attribute carries error messages while processing the actionCollection
@ -66,7 +67,7 @@ public class ActionCollectionCE_DTO {
@JsonView(Views.Public.class)
List<ErrorDTO> errorReports;
@JsonView(Views.Public.class)
@JsonView({Views.Public.class, Git.class})
PluginType pluginType;
@JsonFormat(pattern = "yyyy-MM-dd'T'HH:mm:ss'Z'", timezone = "UTC")
@ -109,7 +110,7 @@ public class ActionCollectionCE_DTO {
String body;
// This list is currently used to record constants
@JsonView(Views.Public.class)
@JsonView({Views.Public.class, Git.class})
List<JSValue> variables;
// This will be used to store the defaultPageId but other fields like branchName, applicationId will act as
@ -158,6 +159,7 @@ public class ActionCollectionCE_DTO {
this.setUserPermissions(Set.of());
}
@JsonView({Views.Internal.class})
public String getUserExecutableName() {
return this.getName();
}

3
app/server/appsmith-server/src/main/java/com/appsmith/server/exports/internal/ExportServiceCEImpl.java
View File

@ -166,8 +166,7 @@ public class ExportServiceCEImpl implements ExportServiceCE {
exportableArtifact.makePristine();
exportableArtifact.sanitiseToExportDBObject();
// Disable exporting the exportableArtifact with datasource config once imported in
// destination
// instance
// destination instance
exportableArtifact.setExportWithConfiguration(null);
return artifactExchangeJson;
}));

2
app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/CommonGitFileUtils.java
View File

@ -1,7 +1,7 @@
package com.appsmith.server.helpers;
import com.appsmith.external.git.FileInterface;
import com.appsmith.git.helpers.FileUtilsImpl;
import com.appsmith.git.files.FileUtilsImpl;
import com.appsmith.server.applications.git.ApplicationGitFileUtils;
import com.appsmith.server.helpers.ce.CommonGitFileUtilsCE;
import com.appsmith.server.services.AnalyticsService;

2
app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/GitFileUtils.java
View File

@ -1,7 +1,7 @@
package com.appsmith.server.helpers;
import com.appsmith.external.git.FileInterface;
import com.appsmith.git.helpers.FileUtilsImpl;
import com.appsmith.git.files.FileUtilsImpl;
import com.appsmith.server.actioncollections.base.ActionCollectionService;
import com.appsmith.server.helpers.ce.GitFileUtilsCE;
import com.appsmith.server.newactions.base.NewActionService;

2
app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/ce/CommonGitFileUtilsCE.java
View File

@ -9,7 +9,7 @@ import com.appsmith.external.models.ArtifactGitReference;
import com.appsmith.external.models.BaseDomain;
import com.appsmith.external.models.DatasourceStorage;
import com.appsmith.git.constants.CommonConstants;
import com.appsmith.git.helpers.FileUtilsImpl;
import com.appsmith.git.files.FileUtilsImpl;
import com.appsmith.server.constants.ArtifactType;
import com.appsmith.server.constants.FieldName;
import com.appsmith.server.domains.ActionCollection;

2
app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/ce/GitFileUtilsCE.java
View File

@ -8,7 +8,7 @@ import com.appsmith.external.models.ApplicationGitReference;
import com.appsmith.external.models.BaseDomain;
import com.appsmith.external.models.DatasourceStorage;
import com.appsmith.external.models.PluginType;
import com.appsmith.git.helpers.FileUtilsImpl;
import com.appsmith.git.files.FileUtilsImpl;
import com.appsmith.server.actioncollections.base.ActionCollectionService;
import com.appsmith.server.constants.FieldName;
import com.appsmith.server.domains.ActionCollection;

5
app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/ApplicationTemplateServiceCEImpl.java
View File

@ -28,7 +28,6 @@ import com.appsmith.server.solutions.ReleaseNotesService;
import com.appsmith.util.WebClientUtils;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectWriter;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.reflect.TypeToken;
@ -381,9 +380,7 @@ public class ApplicationTemplateServiceCEImpl implements ApplicationTemplateServ
// The default mapper is registered with views.public.class and removes few
// attributes due to this
// The templates flow has different requirement hence not using the same
ObjectMapper ow = new ObjectMapper();
ow.registerModule(new JavaTimeModule());
ObjectWriter writer = ow.writer().withDefaultPrettyPrinter();
ObjectWriter writer = objectMapper.writerWithView(null);
payload = writer.writeValueAsString(communityTemplate);
} catch (Exception e) {
return Mono.error(e);

3
app/server/appsmith-server/src/main/java/com/appsmith/server/solutions/ActionExecutionSolutionImpl.java
View File

@ -7,7 +7,6 @@ import com.appsmith.server.helpers.PluginExecutorHelper;
import com.appsmith.server.newactions.base.NewActionService;
import com.appsmith.server.newpages.base.NewPageService;
import com.appsmith.server.plugins.base.PluginService;
import com.appsmith.server.repositories.NewActionRepository;
import com.appsmith.server.services.AnalyticsService;
import com.appsmith.server.services.AuthenticationValidator;
import com.appsmith.server.services.ConfigService;
@ -26,7 +25,6 @@ public class ActionExecutionSolutionImpl extends ActionExecutionSolutionCEImpl i
ActionPermission actionPermission,
ObservationRegistry observationRegistry,
ObjectMapper objectMapper,
NewActionRepository repository,
DatasourceService datasourceService,
PluginService pluginService,
DatasourceContextService datasourceContextService,
@ -46,7 +44,6 @@ public class ActionExecutionSolutionImpl extends ActionExecutionSolutionCEImpl i
actionPermission,
observationRegistry,
objectMapper,
repository,
datasourceService,
pluginService,
datasourceContextService,

4
app/server/appsmith-server/src/main/java/com/appsmith/server/solutions/ce/ActionExecutionSolutionCEImpl.java
View File

@ -38,7 +38,6 @@ import com.appsmith.server.helpers.PluginExecutorHelper;
import com.appsmith.server.newactions.base.NewActionService;
import com.appsmith.server.newpages.base.NewPageService;
import com.appsmith.server.plugins.base.PluginService;
import com.appsmith.server.repositories.NewActionRepository;
import com.appsmith.server.services.AnalyticsService;
import com.appsmith.server.services.AuthenticationValidator;
import com.appsmith.server.services.ConfigService;
@ -104,7 +103,6 @@ public class ActionExecutionSolutionCEImpl implements ActionExecutionSolutionCE
private final ActionPermission actionPermission;
private final ObservationRegistry observationRegistry;
private final ObjectMapper objectMapper;
private final NewActionRepository repository;
private final DatasourceService datasourceService;
private final PluginService pluginService;
private final DatasourceContextService datasourceContextService;
@ -132,7 +130,6 @@ public class ActionExecutionSolutionCEImpl implements ActionExecutionSolutionCE
ActionPermission actionPermission,
ObservationRegistry observationRegistry,
ObjectMapper objectMapper,
NewActionRepository repository,
DatasourceService datasourceService,
PluginService pluginService,
DatasourceContextService datasourceContextService,
@ -151,7 +148,6 @@ public class ActionExecutionSolutionCEImpl implements ActionExecutionSolutionCE
this.actionPermission = actionPermission;
this.observationRegistry = observationRegistry;
this.objectMapper = objectMapper;
this.repository = repository;
this.datasourceService = datasourceService;
this.pluginService = pluginService;
this.datasourceContextService = datasourceContextService;

15
app/server/appsmith-server/src/test/java/com/appsmith/server/controllers/ApplicationControllerTest.java
View File

@ -27,7 +27,10 @@ import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mockito;
import org.mockito.stubbing.Answer;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.autoconfigure.web.reactive.WebFluxTest;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.boot.autoconfigure.web.reactive.ReactiveMultipartAutoConfiguration;
import org.springframework.boot.test.autoconfigure.web.reactive.AutoConfigureWebTestClient;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.context.annotation.Import;
import org.springframework.core.io.ClassPathResource;
@ -44,7 +47,9 @@ import java.io.IOException;
import static org.mockito.ArgumentMatchers.any;
@ExtendWith(SpringExtension.class)
@WebFluxTest(ApplicationController.class)
@SpringBootTest
@AutoConfigureWebTestClient
@EnableAutoConfiguration(exclude = ReactiveMultipartAutoConfiguration.class)
@Import({SecurityTestConfig.class, RedisUtils.class, RedisTestContainerConfig.class})
public class ApplicationControllerTest {
@MockBean
@ -74,9 +79,6 @@ public class ApplicationControllerTest {
@MockBean
UserDataService userDataService;
@Autowired
private WebTestClient webTestClient;
@MockBean
AnalyticsService analyticsService;
@ -95,6 +97,9 @@ public class ApplicationControllerTest {
@MockBean
ProjectProperties projectProperties;
@Autowired
private WebTestClient webTestClient;
private String getFileName(int length) {
StringBuilder fileName = new StringBuilder();
for (int count = 0; count < length; count++) {

5
app/server/appsmith-server/src/test/java/com/appsmith/server/solutions/ce/ActionExecutionSolutionCEImplTest.java
View File

@ -18,7 +18,6 @@ import com.appsmith.server.helpers.PluginExecutorHelper;
import com.appsmith.server.newactions.base.NewActionService;
import com.appsmith.server.newpages.base.NewPageService;
import com.appsmith.server.plugins.base.PluginService;
import com.appsmith.server.repositories.NewActionRepository;
import com.appsmith.server.services.AnalyticsService;
import com.appsmith.server.services.AuthenticationValidator;
import com.appsmith.server.services.ConfigService;
@ -93,9 +92,6 @@ class ActionExecutionSolutionCEImplTest {
@SpyBean
ObjectMapper objectMapper;
@MockBean
NewActionRepository repository;
@SpyBean
DatasourceService datasourceService;
@ -149,7 +145,6 @@ class ActionExecutionSolutionCEImplTest {
actionPermission,
observationRegistry,
objectMapper,
repository,
datasourceService,
pluginService,
datasourceContextService,

2
app/server/pom.xml
View File

@ -27,6 +27,8 @@
<properties>
<deploy.disabled>true</deploy.disabled>
<jackson-bom.version>2.17.0</jackson-bom.version>
<jackson.version>2.17.0</jackson.version>
<java.version>17</java.version>
<javadoc.disabled>true</javadoc.disabled>
<logback.version>1.4.14</logback.version>

1
app/server/reactive-caching/pom.xml
View File

@ -107,6 +107,7 @@
<dependency>
<groupId>com.fasterxml.jackson.datatype</groupId>
<artifactId>jackson-datatype-jsr310</artifactId>
<version>${jackson-bom.version}</version>
<scope>test</scope>
</dependency>
</dependencies>