Protei/PromucFlow_constructor
2
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

chore: Allow all file I/Os for git in parallel (#36872)

Browse Source
This commit is contained in:
Nidhi 2024-10-21 17:45:12 +05:30 committed by GitHub
parent 631abe8e96
commit 1f72d758c0
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 45 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

86
app/server/appsmith-git/src/main/java/com/appsmith/git/files/FileUtilsCEImpl.java
View File

@ -47,6 +47,7 @@ import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Objects; import java.util.Objects;
import java.util.Set; import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import static com.appsmith.external.git.constants.GitConstants.ACTION_COLLECTION_LIST; import static com.appsmith.external.git.constants.GitConstants.ACTION_COLLECTION_LIST;
@ -243,7 +244,7 @@ public class FileUtilsCEImpl implements FileInterface {
Set<String> validPages = new HashSet<>(); Set<String> validPages = new HashSet<>();
for (Map.Entry<String, Object> pageResource : pageEntries) { for (Map.Entry<String, Object> pageResource : pageEntries) {
Map<String, String> validWidgetToParentMap = new HashMap<>(); Map<String, String> validWidgetToParentMap = new ConcurrentHashMap<>();
final String pageName = pageResource.getKey(); final String pageName = pageResource.getKey();
Path pageSpecificDirectory = pageDirectory.resolve(pageName); Path pageSpecificDirectory = pageDirectory.resolve(pageName);
boolean isResourceUpdated = boolean isResourceUpdated =
@ -255,7 +256,8 @@ public class FileUtilsCEImpl implements FileInterface {
pageSpecificDirectory.resolve(pageName + CommonConstants.JSON_EXTENSION)); pageSpecificDirectory.resolve(pageName + CommonConstants.JSON_EXTENSION));
Map<String, JSONObject> result = DSLTransformerHelper.flatten( Map<String, JSONObject> result = DSLTransformerHelper.flatten(
new JSONObject(applicationGitReference.getPageDsl().get(pageName))); new JSONObject(applicationGitReference.getPageDsl().get(pageName)));
result.forEach((key, jsonObject) -> { result.keySet().parallelStream().forEach(key -> {
JSONObject jsonObject = result.get(key);
String widgetName = key.substring(key.lastIndexOf(CommonConstants.DELIMITER_POINT) + 1); String widgetName = key.substring(key.lastIndexOf(CommonConstants.DELIMITER_POINT) + 1);
String childPath = DSLTransformerHelper.getPathToWidgetFile(key, jsonObject, widgetName); String childPath = DSLTransformerHelper.getPathToWidgetFile(key, jsonObject, widgetName);
@ -291,8 +293,8 @@ public class FileUtilsCEImpl implements FileInterface {
Path jsLibDirectory = baseRepo.resolve(JS_LIB_DIRECTORY); Path jsLibDirectory = baseRepo.resolve(JS_LIB_DIRECTORY);
Set<Map.Entry<String, Object>> jsLibEntries = Set<Map.Entry<String, Object>> jsLibEntries =
applicationGitReference.getJsLibraries().entrySet(); applicationGitReference.getJsLibraries().entrySet();
Set<String> validJsLibs = new HashSet<>(); Set<String> validJsLibs = ConcurrentHashMap.newKeySet();
jsLibEntries.forEach(jsLibEntry -> { jsLibEntries.parallelStream().forEach(jsLibEntry -> {
String uidString = jsLibEntry.getKey(); String uidString = jsLibEntry.getKey();
boolean isResourceUpdated = modifiedResources.isResourceUpdated(CUSTOM_JS_LIB_LIST, uidString); boolean isResourceUpdated = modifiedResources.isResourceUpdated(CUSTOM_JS_LIB_LIST, uidString);
@ -308,18 +310,17 @@ public class FileUtilsCEImpl implements FileInterface {
} }
// Create HashMap for valid actions and actionCollections // Create HashMap for valid actions and actionCollections
HashMap<String, Set<String>> validActionsMap = new HashMap<>(); ConcurrentHashMap<String, Set<String>> validActionsMap = new ConcurrentHashMap<>();
HashMap<String, Set<String>> validActionCollectionsMap = new HashMap<>(); ConcurrentHashMap<String, Set<String>> validActionCollectionsMap = new ConcurrentHashMap<>();
validPages.forEach(validPage -> { validPages.forEach(validPage -> {
validActionsMap.put(validPage, new HashSet<>()); validActionsMap.put(validPage, ConcurrentHashMap.newKeySet());
validActionCollectionsMap.put(validPage, new HashSet<>()); validActionCollectionsMap.put(validPage, ConcurrentHashMap.newKeySet());
}); });
// Save actions // Save actions
for (Map.Entry<String, Object> resource : // queryName_pageName => nomenclature for the keys
applicationGitReference.getActions().entrySet()) { // TODO queryName => for app level queries, this is not implemented yet
// queryName_pageName => nomenclature for the keys applicationGitReference.getActions().entrySet().parallelStream().forEach(resource -> {
// TODO queryName => for app level queries, this is not implemented yet
String[] names = resource.getKey().split(NAME_SEPARATOR); String[] names = resource.getKey().split(NAME_SEPARATOR);
if (names.length > 1 && StringUtils.hasLength(names[1])) { if (names.length > 1 && StringUtils.hasLength(names[1])) {
// For actions, we are referring to validNames to maintain unique file names as just name // For actions, we are referring to validNames to maintain unique file names as just name
@ -349,7 +350,7 @@ public class FileUtilsCEImpl implements FileInterface {
.resolve(queryName + CommonConstants.JSON_EXTENSION)); .resolve(queryName + CommonConstants.JSON_EXTENSION));
} }
} }
} });
validActionsMap.forEach((pageName, validActionNames) -> { validActionsMap.forEach((pageName, validActionNames) -> {
Path pageSpecificDirectory = pageDirectory.resolve(pageName); Path pageSpecificDirectory = pageDirectory.resolve(pageName);
@ -358,35 +359,38 @@ public class FileUtilsCEImpl implements FileInterface {
}); });
// Save JSObjects // Save JSObjects
for (Map.Entry<String, Object> resource : // JSObjectName_pageName => nomenclature for the keys
applicationGitReference.getActionCollections().entrySet()) { // TODO JSObjectName => for app level JSObjects, this is not implemented yet
// JSObjectName_pageName => nomenclature for the keys applicationGitReference.getActionCollections().entrySet().parallelStream()
// TODO JSObjectName => for app level JSObjects, this is not implemented yet .forEach(resource -> {
String[] names = resource.getKey().split(NAME_SEPARATOR); String[] names = resource.getKey().split(NAME_SEPARATOR);
if (names.length > 1 && StringUtils.hasLength(names[1])) { if (names.length > 1 && StringUtils.hasLength(names[1])) {
final String actionCollectionName = names[0]; final String actionCollectionName = names[0];
final String pageName = names[1]; final String pageName = names[1];
Path pageSpecificDirectory = pageDirectory.resolve(pageName); Path pageSpecificDirectory = pageDirectory.resolve(pageName);
Path actionCollectionSpecificDirectory = pageSpecificDirectory.resolve(ACTION_COLLECTION_DIRECTORY); Path actionCollectionSpecificDirectory =
pageSpecificDirectory.resolve(ACTION_COLLECTION_DIRECTORY);
if (!validActionCollectionsMap.containsKey(pageName)) { if (!validActionCollectionsMap.containsKey(pageName)) {
validActionCollectionsMap.put(pageName, new HashSet<>()); validActionCollectionsMap.put(pageName, new HashSet<>());
} }
validActionCollectionsMap.get(pageName).add(actionCollectionName); validActionCollectionsMap.get(pageName).add(actionCollectionName);
boolean isResourceUpdated = modifiedResources != null boolean isResourceUpdated = modifiedResources != null
&& modifiedResources.isResourceUpdated(ACTION_COLLECTION_LIST, resource.getKey()); && modifiedResources.isResourceUpdated(ACTION_COLLECTION_LIST, resource.getKey());
if (Boolean.TRUE.equals(isResourceUpdated)) { if (Boolean.TRUE.equals(isResourceUpdated)) {
saveActionCollection( saveActionCollection(
resource.getValue(), resource.getValue(),
applicationGitReference.getActionCollectionBody().get(resource.getKey()), applicationGitReference
actionCollectionName, .getActionCollectionBody()
actionCollectionSpecificDirectory.resolve(actionCollectionName)); .get(resource.getKey()),
// Delete the resource from the old file structure v2 actionCollectionName,
fileOperations.deleteFile(actionCollectionSpecificDirectory.resolve( actionCollectionSpecificDirectory.resolve(actionCollectionName));
actionCollectionName + CommonConstants.JSON_EXTENSION)); // Delete the resource from the old file structure v2
} fileOperations.deleteFile(actionCollectionSpecificDirectory.resolve(
} actionCollectionName + CommonConstants.JSON_EXTENSION));
} }
}
});
// Verify if the old files are deleted // Verify if the old files are deleted
validActionCollectionsMap.forEach((pageName, validActionCollectionNames) -> { validActionCollectionsMap.forEach((pageName, validActionCollectionNames) -> {