Protei/PromucFlow_constructor
2
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

chore: Added changes for splitting git related files for mi (#30202)

Browse Source
This commit is contained in:
Nidhi 2024-01-10 17:57:54 +05:30 committed by GitHub
parent d12b8ff820
commit 861c6e0fa5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
52 changed files with 3941 additions and 3404 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
app/server/appsmith-git/src/main/java/com/appsmith/git/constants/GitDirectories.java
View File

@ -1,9 +1,5 @@
package com.appsmith.git.constants;
public interface GitDirectories {
String PAGE_DIRECTORY = "pages";
String ACTION_DIRECTORY = "queries";
String ACTION_COLLECTION_DIRECTORY = "jsobjects";
String DATASOURCE_DIRECTORY = "datasources";
String JS_LIB_DIRECTORY = "jslibs";
}
import com.appsmith.git.constants.ce.GitDirectoriesCE;
public interface GitDirectories extends GitDirectoriesCE {}

9
app/server/appsmith-git/src/main/java/com/appsmith/git/constants/ce/GitDirectoriesCE.java Normal file
View File

@ -0,0 +1,9 @@
package com.appsmith.git.constants.ce;
public interface GitDirectoriesCE {
String PAGE_DIRECTORY = "pages";
String ACTION_DIRECTORY = "queries";
String ACTION_COLLECTION_DIRECTORY = "jsobjects";
String DATASOURCE_DIRECTORY = "datasources";
String JS_LIB_DIRECTORY = "jslibs";
}

1134
app/server/appsmith-git/src/main/java/com/appsmith/git/helpers/FileUtilsImpl.java
View File

File diff suppressed because it is too large Load Diff

1154
app/server/appsmith-git/src/main/java/com/appsmith/git/helpers/ce/FileUtilsCEImpl.java Normal file
View File

File diff suppressed because it is too large Load Diff

905
app/server/appsmith-git/src/main/java/com/appsmith/git/service/GitExecutorImpl.java
View File

@ -1,913 +1,16 @@
package com.appsmith.git.service;
import com.appsmith.external.constants.AnalyticsEvents;
import com.appsmith.external.constants.ErrorReferenceDocUrl;
import com.appsmith.external.dtos.GitBranchDTO;
import com.appsmith.external.dtos.GitLogDTO;
import com.appsmith.external.dtos.GitStatusDTO;
import com.appsmith.external.dtos.MergeStatusDTO;
import com.appsmith.external.git.GitExecutor;
import com.appsmith.external.helpers.Stopwatch;
import com.appsmith.git.configurations.GitServiceConfig;
import com.appsmith.git.constants.AppsmithBotAsset;
import com.appsmith.git.constants.CommonConstants;
import com.appsmith.git.constants.Constraint;
import com.appsmith.git.constants.GitDirectories;
import com.appsmith.git.helpers.RepositoryHelper;
import com.appsmith.git.helpers.SshTransportConfigCallback;
import com.appsmith.git.helpers.StopwatchHelpers;
import lombok.RequiredArgsConstructor;
import com.appsmith.git.service.ce.GitExecutorCEImpl;
import lombok.extern.slf4j.Slf4j;
import org.eclipse.jgit.api.CreateBranchCommand;
import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.api.ListBranchCommand;
import org.eclipse.jgit.api.MergeCommand;
import org.eclipse.jgit.api.MergeResult;
import org.eclipse.jgit.api.RebaseCommand;
import org.eclipse.jgit.api.RebaseResult;
import org.eclipse.jgit.api.ResetCommand;
import org.eclipse.jgit.api.Status;
import org.eclipse.jgit.api.TransportConfigCallback;
import org.eclipse.jgit.api.errors.CheckoutConflictException;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.lib.BranchTrackingStatus;
import org.eclipse.jgit.lib.PersonIdent;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.lib.StoredConfig;
import org.eclipse.jgit.merge.MergeStrategy;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.transport.RefSpec;
import org.eclipse.jgit.util.StringUtils;
import org.springframework.stereotype.Component;
import org.springframework.util.FileSystemUtils;
import reactor.core.publisher.Mono;
import reactor.core.scheduler.Scheduler;
import reactor.core.scheduler.Schedulers;
import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.Duration;
import java.time.ZoneId;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import static com.appsmith.git.constants.CommonConstants.FILE_MIGRATION_MESSAGE;
@RequiredArgsConstructor
@Component
@Slf4j
public class GitExecutorImpl implements GitExecutor {
public class GitExecutorImpl extends GitExecutorCEImpl implements GitExecutor {
private final RepositoryHelper repositoryHelper = new RepositoryHelper();
private final GitServiceConfig gitServiceConfig;
public static final DateTimeFormatter ISO_FORMATTER =
DateTimeFormatter.ISO_INSTANT.withZone(ZoneId.from(ZoneOffset.UTC));
private final Scheduler scheduler = Schedulers.boundedElastic();
private static final String SUCCESS_MERGE_STATUS = "This branch has no conflicts with the base branch.";
/**
* This method will handle the git-commit functionality. Under the hood it checks if the repo has already been
* initialised and will be initialised if git repo is not present
* @param path parent path to repo
* @param commitMessage message which will be registered for this commit
* @param authorName author details
* @param authorEmail author details
* @param doAmend To amend with the previous commit
* @return if the commit was successful
*/
@Override
public Mono<String> commitApplication(
Path path,
String commitMessage,
String authorName,
String authorEmail,
boolean isSuffixedPath,
boolean doAmend) {
final String finalAuthorName =
StringUtils.isEmptyOrNull(authorName) ? AppsmithBotAsset.APPSMITH_BOT_USERNAME : authorName;
final String finalAuthorEmail =
StringUtils.isEmptyOrNull(authorEmail) ? AppsmithBotAsset.APPSMITH_BOT_EMAIL : authorEmail;
return Mono.fromCallable(() -> {
log.debug("Trying to commit to local repo path, {}", path);
Path repoPath = path;
if (Boolean.TRUE.equals(isSuffixedPath)) {
repoPath = createRepoPath(repoPath);
}
Stopwatch processStopwatch =
StopwatchHelpers.startStopwatch(repoPath, AnalyticsEvents.GIT_COMMIT.getEventName());
// Just need to open a repository here and make a commit
try (Git git = Git.open(repoPath.toFile())) {
// Stage all the files added and modified
git.add().addFilepattern(".").call();
// Stage modified and deleted files
git.add().setUpdate(true).addFilepattern(".").call();
// Commit the changes
git.commit()
.setMessage(commitMessage)
// Only make a commit if there are any updates
.setAllowEmpty(false)
.setAuthor(finalAuthorName, finalAuthorEmail)
.setCommitter(finalAuthorName, finalAuthorEmail)
.setAmend(doAmend)
.call();
processStopwatch.stopAndLogTimeInMillis();
return "Committed successfully!";
}
})
.timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS))
.subscribeOn(scheduler);
}
/**
* Method to create a new repository to provided path
* @param repoPath path where new repo needs to be created
* @return if the operation was successful
*/
@Override
public boolean createNewRepository(Path repoPath) throws GitAPIException {
// create new repo to the mentioned path
log.debug("Trying to create new repository: {}", repoPath);
Git.init().setDirectory(repoPath.toFile()).call();
return true;
}
/**
* Method to get the commit history
* @param repoSuffix Path used to generate the repo url specific to the application for which the commit history is requested
* @return list of git commits
*/
@Override
public Mono<List<GitLogDTO>> getCommitHistory(Path repoSuffix) {
return Mono.fromCallable(() -> {
log.debug(Thread.currentThread().getName() + ": get commit history for " + repoSuffix);
List<GitLogDTO> commitLogs = new ArrayList<>();
Path repoPath = createRepoPath(repoSuffix);
Stopwatch processStopwatch = StopwatchHelpers.startStopwatch(
repoPath, AnalyticsEvents.GIT_COMMIT_HISTORY.getEventName());
try (Git git = Git.open(repoPath.toFile())) {
Iterable<RevCommit> gitLogs = git.log()
.setMaxCount(Constraint.MAX_COMMIT_LOGS)
.call();
gitLogs.forEach(revCommit -> {
PersonIdent author = revCommit.getAuthorIdent();
GitLogDTO gitLog = new GitLogDTO(
revCommit.getName(),
author.getName(),
author.getEmailAddress(),
revCommit.getFullMessage(),
ISO_FORMATTER.format(new Date(revCommit.getCommitTime() * 1000L).toInstant()));
processStopwatch.stopAndLogTimeInMillis();
commitLogs.add(gitLog);
});
return commitLogs;
}
})
.timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS))
.subscribeOn(scheduler);
}
@Override
public Path createRepoPath(Path suffix) {
return Paths.get(gitServiceConfig.getGitRootPath()).resolve(suffix);
}
/**
* Method to push changes to remote repo
* @param repoSuffix Path used to generate the repo url specific to the application which needs to be pushed to remote
* @param remoteUrl remote repo url
* @param publicKey
* @param privateKey
* @return Success message
*/
@Override
public Mono<String> pushApplication(
Path repoSuffix, String remoteUrl, String publicKey, String privateKey, String branchName) {
// We can safely assume that repo has been already initialised either in commit or clone flow and can directly
// open the repo
return Mono.fromCallable(() -> {
log.debug(Thread.currentThread().getName() + ": pushing changes to remote " + remoteUrl);
// open the repo
Path baseRepoPath = createRepoPath(repoSuffix);
Stopwatch processStopwatch =
StopwatchHelpers.startStopwatch(baseRepoPath, AnalyticsEvents.GIT_PUSH.getEventName());
try (Git git = Git.open(baseRepoPath.toFile())) {
TransportConfigCallback transportConfigCallback =
new SshTransportConfigCallback(privateKey, publicKey);
StringBuilder result = new StringBuilder("Pushed successfully with status : ");
git.push()
.setTransportConfigCallback(transportConfigCallback)
.setRemote(remoteUrl)
.call()
.forEach(pushResult -> pushResult
.getRemoteUpdates()
.forEach(remoteRefUpdate -> {
result.append(remoteRefUpdate.getStatus())
.append(",");
if (!StringUtils.isEmptyOrNull(remoteRefUpdate.getMessage())) {
result.append(remoteRefUpdate.getMessage())
.append(",");
}
}));
// We can support username and password in future if needed
// pushCommand.setCredentialsProvider(new UsernamePasswordCredentialsProvider("username",
// "password"));
processStopwatch.stopAndLogTimeInMillis();
return result.substring(0, result.length() - 1);
}
})
.timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS))
.subscribeOn(scheduler);
}
/** Clone the repo to the file path : container-volume/orgId/defaultAppId/repo/applicationData
*
* @param repoSuffix combination of orgId, defaultId and repoName
* @param remoteUrl ssh url of the git repo(we support cloning via ssh url only with deploy key)
* @param privateKey generated by us and specific to the defaultApplication
* @param publicKey generated by us and specific to the defaultApplication
* @return defaultBranchName of the repo
* */
@Override
public Mono<String> cloneApplication(Path repoSuffix, String remoteUrl, String privateKey, String publicKey) {
Stopwatch processStopwatch =
StopwatchHelpers.startStopwatch(repoSuffix, AnalyticsEvents.GIT_CLONE.getEventName());
return Mono.fromCallable(() -> {
log.debug(Thread.currentThread().getName() + ": Cloning the repo from the remote " + remoteUrl);
final TransportConfigCallback transportConfigCallback =
new SshTransportConfigCallback(privateKey, publicKey);
File file = Paths.get(gitServiceConfig.getGitRootPath())
.resolve(repoSuffix)
.toFile();
while (file.exists()) {
FileSystemUtils.deleteRecursively(file);
}
Git git = Git.cloneRepository()
.setURI(remoteUrl)
.setTransportConfigCallback(transportConfigCallback)
.setDirectory(file)
.call();
String branchName = git.getRepository().getBranch();
repositoryHelper.updateRemoteBranchTrackingConfig(branchName, git);
git.close();
processStopwatch.stopAndLogTimeInMillis();
return branchName;
})
.timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS))
.subscribeOn(scheduler);
}
@Override
public Mono<String> createAndCheckoutToBranch(Path repoSuffix, String branchName) {
// We can safely assume that repo has been already initialised either in commit or clone flow and can directly
// open the repo
Stopwatch processStopwatch =
StopwatchHelpers.startStopwatch(repoSuffix, AnalyticsEvents.GIT_CREATE_BRANCH.getEventName());
return Mono.fromCallable(() -> {
log.debug(Thread.currentThread().getName() + ": Creating branch " + branchName + "for the repo "
+ repoSuffix);
// open the repo
Path baseRepoPath = createRepoPath(repoSuffix);
try (Git git = Git.open(baseRepoPath.toFile())) {
// Create and checkout to new branch
git.checkout()
.setCreateBranch(Boolean.TRUE)
.setName(branchName)
.setUpstreamMode(CreateBranchCommand.SetupUpstreamMode.TRACK)
.call();
repositoryHelper.updateRemoteBranchTrackingConfig(branchName, git);
processStopwatch.stopAndLogTimeInMillis();
return git.getRepository().getBranch();
}
})
.timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS))
.subscribeOn(scheduler);
}
@Override
public Mono<Boolean> deleteBranch(Path repoSuffix, String branchName) {
// We can safely assume that repo has been already initialised either in commit or clone flow and can directly
// open the repo
Stopwatch processStopwatch =
StopwatchHelpers.startStopwatch(repoSuffix, AnalyticsEvents.GIT_DELETE_BRANCH.getEventName());
return Mono.fromCallable(() -> {
log.debug(Thread.currentThread().getName() + ": Deleting branch " + branchName + "for the repo "
+ repoSuffix);
// open the repo
Path baseRepoPath = createRepoPath(repoSuffix);
try (Git git = Git.open(baseRepoPath.toFile())) {
// Create and checkout to new branch
List<String> deleteBranchList = git.branchDelete()
.setBranchNames(branchName)
.setForce(Boolean.TRUE)
.call();
processStopwatch.stopAndLogTimeInMillis();
if (deleteBranchList.isEmpty()) {
return Boolean.FALSE;
}
return Boolean.TRUE;
}
})
.timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS))
.subscribeOn(scheduler);
}
@Override
public Mono<Boolean> checkoutToBranch(Path repoSuffix, String branchName) {
Stopwatch processStopwatch =
StopwatchHelpers.startStopwatch(repoSuffix, AnalyticsEvents.GIT_CHECKOUT.getEventName());
return Mono.fromCallable(() -> {
log.debug(Thread.currentThread().getName() + ": Switching to the branch " + branchName);
// We can safely assume that repo has been already initialised either in commit or clone flow and
// can directly
// open the repo
Path baseRepoPath = createRepoPath(repoSuffix);
try (Git git = Git.open(baseRepoPath.toFile())) {
if (StringUtils.equalsIgnoreCase(
branchName, git.getRepository().getBranch())) {
return Boolean.TRUE;
}
// Create and checkout to new branch
String checkedOutBranch = git.checkout()
.setCreateBranch(Boolean.FALSE)
.setName(branchName)
.setUpstreamMode(CreateBranchCommand.SetupUpstreamMode.SET_UPSTREAM)
.call()
.getName();
processStopwatch.stopAndLogTimeInMillis();
return StringUtils.equalsIgnoreCase(checkedOutBranch, "refs/heads/" + branchName);
} catch (Exception e) {
throw new Exception(e);
}
})
.timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS))
.subscribeOn(scheduler);
}
@Override
public Mono<MergeStatusDTO> pullApplication(
Path repoSuffix, String remoteUrl, String branchName, String privateKey, String publicKey)
throws IOException {
Stopwatch processStopwatch =
StopwatchHelpers.startStopwatch(repoSuffix, AnalyticsEvents.GIT_PULL.getEventName());
TransportConfigCallback transportConfigCallback = new SshTransportConfigCallback(privateKey, publicKey);
try (Git git = Git.open(createRepoPath(repoSuffix).toFile())) {
return Mono.fromCallable(() -> {
log.debug(Thread.currentThread().getName() + ": Pull changes from remote " + remoteUrl
+ " for the branch " + branchName);
// checkout the branch on which the merge command is run
git.checkout()
.setName(branchName)
.setCreateBranch(false)
.call();
MergeResult mergeResult = git.pull()
.setRemoteBranchName(branchName)
.setTransportConfigCallback(transportConfigCallback)
.setFastForward(MergeCommand.FastForwardMode.FF)
.call()
.getMergeResult();
MergeStatusDTO mergeStatus = new MergeStatusDTO();
Long count =
Arrays.stream(mergeResult.getMergedCommits()).count();
if (mergeResult.getMergeStatus().isSuccessful()) {
mergeStatus.setMergeAble(true);
mergeStatus.setStatus(count + " commits merged from origin/" + branchName);
} else {
// If there are conflicts add the conflicting file names to the response structure
mergeStatus.setMergeAble(false);
List<String> mergeConflictFiles = new ArrayList<>();
if (!Optional.ofNullable(mergeResult.getConflicts()).isEmpty()) {
mergeConflictFiles.addAll(
mergeResult.getConflicts().keySet());
}
mergeStatus.setConflictingFiles(mergeConflictFiles);
// On merge conflicts abort the merge => git merge --abort
git.getRepository().writeMergeCommitMsg(null);
git.getRepository().writeMergeHeads(null);
processStopwatch.stopAndLogTimeInMillis();
throw new org.eclipse.jgit.errors.CheckoutConflictException(mergeConflictFiles.toString());
}
processStopwatch.stopAndLogTimeInMillis();
return mergeStatus;
})
.onErrorResume(error -> {
try {
return resetToLastCommit(git).flatMap(ignore -> Mono.error(error));
} catch (GitAPIException e) {
log.error("Error for hard resetting to latest commit {0}", e);
return Mono.error(e);
}
})
.timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS))
.subscribeOn(scheduler);
}
}
@Override
public Mono<List<GitBranchDTO>> listBranches(Path repoSuffix) {
Path baseRepoPath = createRepoPath(repoSuffix);
return Mono.fromCallable(() -> {
log.debug(Thread.currentThread().getName() + ": Get branches for the application " + repoSuffix);
Git git = Git.open(baseRepoPath.toFile());
List<Ref> refList = git.branchList()
.setListMode(ListBranchCommand.ListMode.ALL)
.call();
List<GitBranchDTO> branchList = new ArrayList<>();
GitBranchDTO gitBranchDTO = new GitBranchDTO();
if (refList.isEmpty()) {
gitBranchDTO.setBranchName(git.getRepository().getBranch());
branchList.add(gitBranchDTO);
} else {
for (Ref ref : refList) {
// if (!ref.getName().equals(defaultBranch)) {
gitBranchDTO = new GitBranchDTO();
gitBranchDTO.setBranchName(ref.getName()
.replace("refs/", "")
.replace("heads/", "")
.replace("remotes/", ""));
branchList.add(gitBranchDTO);
}
}
git.close();
return branchList;
})
.timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS))
.subscribeOn(scheduler);
}
@Override
public Mono<String> getRemoteDefaultBranch(Path repoSuffix, String remoteUrl, String privateKey, String publicKey) {
Path baseRepoPath = createRepoPath(repoSuffix);
return Mono.fromCallable(() -> {
TransportConfigCallback transportConfigCallback =
new SshTransportConfigCallback(privateKey, publicKey);
Git git = Git.open(baseRepoPath.toFile());
return git.lsRemote()
.setRemote(remoteUrl)
.setTransportConfigCallback(transportConfigCallback)
.callAsMap()
.get("HEAD")
.getTarget()
.getName()
.replace("refs/heads/", "");
})
.timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS))
.subscribeOn(scheduler);
}
/**
* This method will handle the git-status functionality
*
* @param repoPath Path to actual repo
* @param branchName branch name for which the status is required
* @return Map of file names those are modified, conflicted etc.
*/
@Override
public Mono<GitStatusDTO> getStatus(Path repoPath, String branchName) {
Stopwatch processStopwatch =
StopwatchHelpers.startStopwatch(repoPath, AnalyticsEvents.GIT_STATUS.getEventName());
return Mono.fromCallable(() -> {
try (Git git = Git.open(repoPath.toFile())) {
log.debug(Thread.currentThread().getName() + ": Get status for repo " + repoPath + ", branch "
+ branchName);
Status status = git.status().call();
GitStatusDTO response = new GitStatusDTO();
Set<String> modifiedAssets = new HashSet<>();
modifiedAssets.addAll(status.getModified());
modifiedAssets.addAll(status.getAdded());
modifiedAssets.addAll(status.getRemoved());
modifiedAssets.addAll(status.getUncommittedChanges());
modifiedAssets.addAll(status.getUntracked());
response.setAdded(status.getAdded());
response.setRemoved(status.getRemoved());
Set<String> queriesModified = new HashSet<>();
Set<String> jsObjectsModified = new HashSet<>();
Set<String> pagesModified = new HashSet<>();
int modifiedPages = 0;
int modifiedQueries = 0;
int modifiedJSObjects = 0;
int modifiedDatasources = 0;
int modifiedJSLibs = 0;
for (String x : modifiedAssets) {
// begins with pages and filename and parent name should be same or contains widgets
if (x.contains(CommonConstants.WIDGETS)) {
if (!pagesModified.contains(getPageName(x))) {
pagesModified.add(getPageName(x));
modifiedPages++;
}
} else if (!x.contains(CommonConstants.WIDGETS)
&& x.startsWith(GitDirectories.PAGE_DIRECTORY)
&& !x.contains(GitDirectories.ACTION_DIRECTORY)
&& !x.contains(GitDirectories.ACTION_COLLECTION_DIRECTORY)) {
if (!pagesModified.contains(getPageName(x))) {
pagesModified.add(getPageName(x));
modifiedPages++;
}
} else if (x.contains(GitDirectories.ACTION_DIRECTORY + CommonConstants.DELIMITER_PATH)) {
String queryName =
x.split(GitDirectories.ACTION_DIRECTORY + CommonConstants.DELIMITER_PATH)[1];
int position = queryName.indexOf(CommonConstants.DELIMITER_PATH);
if (position != -1) {
queryName = queryName.substring(0, position);
String pageName = x.split(CommonConstants.DELIMITER_PATH)[1];
if (!queriesModified.contains(pageName + queryName)) {
queriesModified.add(pageName + queryName);
modifiedQueries++;
}
}
} else if (x.contains(
GitDirectories.ACTION_COLLECTION_DIRECTORY + CommonConstants.DELIMITER_PATH)
&& !x.endsWith(CommonConstants.JSON_EXTENSION)) {
String queryName = x.substring(x.lastIndexOf(CommonConstants.DELIMITER_PATH) + 1);
String pageName = x.split(CommonConstants.DELIMITER_PATH)[1];
if (!jsObjectsModified.contains(pageName + queryName)) {
jsObjectsModified.add(pageName + queryName);
modifiedJSObjects++;
}
} else if (x.contains(
GitDirectories.DATASOURCE_DIRECTORY + CommonConstants.DELIMITER_PATH)) {
modifiedDatasources++;
} else if (x.contains(GitDirectories.JS_LIB_DIRECTORY + CommonConstants.DELIMITER_PATH)) {
modifiedJSLibs++;
// remove this code in future when all the older format js libs are migrated to new
// format
if (x.contains("js.json")) {
/*
As this updated filename has color(:), it means this is the older format js
lib file that we're going to rename with the format without colon.
Hence, we need to show a message to user saying this might be a system level change.
*/
response.setMigrationMessage(FILE_MIGRATION_MESSAGE);
}
}
}
response.setModified(modifiedAssets);
response.setConflicting(status.getConflicting());
response.setIsClean(status.isClean());
response.setModifiedPages(modifiedPages);
response.setModifiedQueries(modifiedQueries);
response.setModifiedJSObjects(modifiedJSObjects);
response.setModifiedDatasources(modifiedDatasources);
response.setModifiedJSLibs(modifiedJSLibs);
BranchTrackingStatus trackingStatus = BranchTrackingStatus.of(git.getRepository(), branchName);
if (trackingStatus != null) {
response.setAheadCount(trackingStatus.getAheadCount());
response.setBehindCount(trackingStatus.getBehindCount());
response.setRemoteBranch(trackingStatus.getRemoteTrackingBranch());
} else {
log.debug(
"Remote tracking details not present for branch: {}, repo: {}",
branchName,
repoPath);
response.setAheadCount(0);
response.setBehindCount(0);
response.setRemoteBranch("untracked");
}
// Remove modified changes from current branch so that checkout to other branches will be
// possible
if (!status.isClean()) {
return resetToLastCommit(git).map(ref -> {
processStopwatch.stopAndLogTimeInMillis();
return response;
});
}
processStopwatch.stopAndLogTimeInMillis();
return Mono.just(response);
}
})
.timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS))
.flatMap(response -> response)
.subscribeOn(scheduler);
}
private String getPageName(String path) {
String[] pathArray = path.split(CommonConstants.DELIMITER_PATH);
return pathArray[1];
}
@Override
public Mono<String> mergeBranch(Path repoSuffix, String sourceBranch, String destinationBranch) {
return Mono.fromCallable(() -> {
Stopwatch processStopwatch =
StopwatchHelpers.startStopwatch(repoSuffix, AnalyticsEvents.GIT_MERGE.getEventName());
log.debug(Thread.currentThread().getName() + ": Merge branch " + sourceBranch + " on "
+ destinationBranch);
try (Git git = Git.open(createRepoPath(repoSuffix).toFile())) {
try {
// checkout the branch on which the merge command is run
git.checkout()
.setName(destinationBranch)
.setCreateBranch(false)
.call();
MergeResult mergeResult = git.merge()
.include(git.getRepository().findRef(sourceBranch))
.setStrategy(MergeStrategy.RECURSIVE)
.call();
processStopwatch.stopAndLogTimeInMillis();
return mergeResult.getMergeStatus().name();
} catch (GitAPIException e) {
// On merge conflicts abort the merge => git merge --abort
git.getRepository().writeMergeCommitMsg(null);
git.getRepository().writeMergeHeads(null);
processStopwatch.stopAndLogTimeInMillis();
throw new Exception(e);
}
}
})
.onErrorResume(error -> {
try {
return resetToLastCommit(repoSuffix, destinationBranch).thenReturn(error.getMessage());
} catch (GitAPIException | IOException e) {
log.error("Error while hard resetting to latest commit {0}", e);
return Mono.error(e);
}
})
.timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS))
.subscribeOn(scheduler);
}
@Override
public Mono<String> fetchRemote(
Path repoSuffix,
String publicKey,
String privateKey,
boolean isRepoPath,
String branchName,
boolean isFetchAll) {
Stopwatch processStopwatch =
StopwatchHelpers.startStopwatch(repoSuffix, AnalyticsEvents.GIT_FETCH.getEventName());
Path repoPath = Boolean.TRUE.equals(isRepoPath) ? repoSuffix : createRepoPath(repoSuffix);
return Mono.fromCallable(() -> {
TransportConfigCallback config = new SshTransportConfigCallback(privateKey, publicKey);
try (Git git = Git.open(repoPath.toFile())) {
String fetchMessages;
if (Boolean.TRUE.equals(isFetchAll)) {
fetchMessages = git.fetch()
.setRemoveDeletedRefs(true)
.setTransportConfigCallback(config)
.call()
.getMessages();
} else {
RefSpec ref =
new RefSpec("refs/heads/" + branchName + ":refs/remotes/origin/" + branchName);
fetchMessages = git.fetch()
.setRefSpecs(ref)
.setRemoveDeletedRefs(true)
.setTransportConfigCallback(config)
.call()
.getMessages();
}
processStopwatch.stopAndLogTimeInMillis();
return fetchMessages;
}
})
.onErrorResume(error -> {
log.error(error.getMessage());
return Mono.error(error);
})
.timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS))
.subscribeOn(scheduler);
}
@Override
public Mono<MergeStatusDTO> isMergeBranch(Path repoSuffix, String sourceBranch, String destinationBranch) {
Stopwatch processStopwatch =
StopwatchHelpers.startStopwatch(repoSuffix, AnalyticsEvents.GIT_MERGE_CHECK.getEventName());
return Mono.fromCallable(() -> {
log.debug(
Thread.currentThread().getName()
+ ": Check mergeability for repo {} with src: {}, dest: {}",
repoSuffix,
sourceBranch,
destinationBranch);
try (Git git = Git.open(createRepoPath(repoSuffix).toFile())) {
// checkout the branch on which the merge command is run
try {
git.checkout()
.setName(destinationBranch)
.setCreateBranch(false)
.call();
} catch (GitAPIException e) {
if (e instanceof CheckoutConflictException) {
MergeStatusDTO mergeStatus = new MergeStatusDTO();
mergeStatus.setMergeAble(false);
mergeStatus.setConflictingFiles(((CheckoutConflictException) e).getConflictingPaths());
processStopwatch.stopAndLogTimeInMillis();
return mergeStatus;
}
}
MergeResult mergeResult = git.merge()
.include(git.getRepository().findRef(sourceBranch))
.setFastForward(MergeCommand.FastForwardMode.NO_FF)
.setCommit(false)
.call();
MergeStatusDTO mergeStatus = new MergeStatusDTO();
if (mergeResult.getMergeStatus().isSuccessful()) {
mergeStatus.setMergeAble(true);
mergeStatus.setMessage(SUCCESS_MERGE_STATUS);
} else {
// If there aer conflicts add the conflicting file names to the response structure
mergeStatus.setMergeAble(false);
List<String> mergeConflictFiles =
new ArrayList<>(mergeResult.getConflicts().keySet());
mergeStatus.setConflictingFiles(mergeConflictFiles);
StringBuilder errorMessage = new StringBuilder();
if (mergeResult.getMergeStatus().equals(MergeResult.MergeStatus.CONFLICTING)) {
errorMessage.append("Conflicts");
} else {
errorMessage.append(mergeResult.getMergeStatus().toString());
}
errorMessage
.append(" while merging branch: ")
.append(destinationBranch)
.append(" <= ")
.append(sourceBranch);
mergeStatus.setMessage(errorMessage.toString());
mergeStatus.setReferenceDoc(ErrorReferenceDocUrl.GIT_MERGE_CONFLICT.getDocUrl());
}
mergeStatus.setStatus(mergeResult.getMergeStatus().name());
return mergeStatus;
}
})
.flatMap(status -> {
try {
// Revert uncommitted changes if any
return resetToLastCommit(repoSuffix, destinationBranch).map(ignore -> {
processStopwatch.stopAndLogTimeInMillis();
return status;
});
} catch (GitAPIException | IOException e) {
log.error("Error for hard resetting to latest commit {0}", e);
return Mono.error(e);
}
})
.timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS))
.subscribeOn(scheduler);
}
public Mono<String> checkoutRemoteBranch(Path repoSuffix, String branchName) {
// We can safely assume that repo has been already initialised either in commit or clone flow and can directly
// open the repo
return Mono.fromCallable(() -> {
log.debug(Thread.currentThread().getName() + ": Checking out remote branch origin/" + branchName
+ " for the repo " + repoSuffix);
// open the repo
Path baseRepoPath = createRepoPath(repoSuffix);
try (Git git = Git.open(baseRepoPath.toFile())) {
// Create and checkout to new branch
git.checkout()
.setCreateBranch(Boolean.TRUE)
.setName(branchName)
.setUpstreamMode(CreateBranchCommand.SetupUpstreamMode.TRACK)
.setStartPoint("origin/" + branchName)
.call();
StoredConfig config = git.getRepository().getConfig();
config.setString("branch", branchName, "remote", "origin");
config.setString("branch", branchName, "merge", "refs/heads/" + branchName);
config.save();
return git.getRepository().getBranch();
}
})
.timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS))
.subscribeOn(scheduler);
}
@Override
public Mono<Boolean> testConnection(String publicKey, String privateKey, String remoteUrl) {
return Mono.fromCallable(() -> {
TransportConfigCallback transportConfigCallback =
new SshTransportConfigCallback(privateKey, publicKey);
Git.lsRemoteRepository()
.setTransportConfigCallback(transportConfigCallback)
.setRemote(remoteUrl)
.setHeads(true)
.setTags(true)
.call();
return true;
})
.timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS))
.subscribeOn(scheduler);
}
private Mono<Ref> resetToLastCommit(Git git) throws GitAPIException {
Stopwatch processStopwatch = StopwatchHelpers.startStopwatch(
git.getRepository().getDirectory().toPath().getParent(), AnalyticsEvents.GIT_RESET.getEventName());
return Mono.fromCallable(() -> {
// Remove tracked files
Ref ref = git.reset().setMode(ResetCommand.ResetType.HARD).call();
// Remove untracked files
git.clean().setForce(true).setCleanDirectories(true).call();
processStopwatch.stopAndLogTimeInMillis();
return ref;
})
.timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS))
.subscribeOn(scheduler);
}
public Mono<Boolean> resetToLastCommit(Path repoSuffix, String branchName) throws GitAPIException, IOException {
try (Git git = Git.open(createRepoPath(repoSuffix).toFile())) {
return this.resetToLastCommit(git)
.flatMap(ref -> checkoutToBranch(repoSuffix, branchName))
.flatMap(checkedOut -> {
try {
return resetToLastCommit(git).thenReturn(true);
} catch (GitAPIException e) {
log.error(e.getMessage());
return Mono.error(e);
}
});
}
}
public Mono<Boolean> resetHard(Path repoSuffix, String branchName) {
return this.checkoutToBranch(repoSuffix, branchName)
.flatMap(aBoolean -> {
try (Git git = Git.open(createRepoPath(repoSuffix).toFile())) {
Ref ref = git.reset()
.setMode(ResetCommand.ResetType.HARD)
.setRef("HEAD~1")
.call();
return Mono.just(true);
} catch (GitAPIException | IOException e) {
log.error("Error while resetting the commit, {}", e.getMessage());
}
return Mono.just(false);
})
.timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS))
.subscribeOn(scheduler);
}
public Mono<Boolean> rebaseBranch(Path repoSuffix, String branchName) {
return this.checkoutToBranch(repoSuffix, branchName)
.flatMap(isCheckedOut -> {
try (Git git = Git.open(createRepoPath(repoSuffix).toFile())) {
RebaseResult result =
git.rebase().setUpstream("origin/" + branchName).call();
if (result.getStatus().isSuccessful()) {
return Mono.just(true);
} else {
log.error(
"Error while rebasing the branch, {}, {}",
result.getStatus().name(),
result.getConflicts());
git.rebase()
.setUpstream("origin/" + branchName)
.setOperation(RebaseCommand.Operation.ABORT)
.call();
return Mono.error(new Exception("Error while rebasing the branch, "
+ result.getStatus().name()));
}
} catch (GitAPIException | IOException e) {
log.error("Error while rebasing the branch, {}", e.getMessage());
return Mono.error(e);
}
})
.timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS))
.subscribeOn(scheduler);
}
@Override
public Mono<BranchTrackingStatus> getBranchTrackingStatus(Path repoPath, String branchName) {
return Mono.fromCallable(() -> {
try (Git git = Git.open(createRepoPath(repoPath).toFile())) {
return BranchTrackingStatus.of(git.getRepository(), branchName);
}
})
.timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS))
.subscribeOn(scheduler);
public GitExecutorImpl(GitServiceConfig gitServiceConfig) {
super(gitServiceConfig);
}
}

917
app/server/appsmith-git/src/main/java/com/appsmith/git/service/ce/GitExecutorCEImpl.java Normal file
View File

@ -0,0 +1,917 @@
package com.appsmith.git.service.ce;
import com.appsmith.external.constants.AnalyticsEvents;
import com.appsmith.external.constants.ErrorReferenceDocUrl;
import com.appsmith.external.dtos.GitBranchDTO;
import com.appsmith.external.dtos.GitLogDTO;
import com.appsmith.external.dtos.GitStatusDTO;
import com.appsmith.external.dtos.MergeStatusDTO;
import com.appsmith.external.git.GitExecutor;
import com.appsmith.external.helpers.Stopwatch;
import com.appsmith.git.configurations.GitServiceConfig;
import com.appsmith.git.constants.AppsmithBotAsset;
import com.appsmith.git.constants.CommonConstants;
import com.appsmith.git.constants.Constraint;
import com.appsmith.git.constants.GitDirectories;
import com.appsmith.git.helpers.RepositoryHelper;
import com.appsmith.git.helpers.SshTransportConfigCallback;
import com.appsmith.git.helpers.StopwatchHelpers;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.eclipse.jgit.api.CreateBranchCommand;
import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.api.ListBranchCommand;
import org.eclipse.jgit.api.MergeCommand;
import org.eclipse.jgit.api.MergeResult;
import org.eclipse.jgit.api.RebaseCommand;
import org.eclipse.jgit.api.RebaseResult;
import org.eclipse.jgit.api.ResetCommand;
import org.eclipse.jgit.api.Status;
import org.eclipse.jgit.api.TransportConfigCallback;
import org.eclipse.jgit.api.errors.CheckoutConflictException;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.lib.BranchTrackingStatus;
import org.eclipse.jgit.lib.PersonIdent;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.lib.StoredConfig;
import org.eclipse.jgit.merge.MergeStrategy;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.transport.RefSpec;
import org.eclipse.jgit.util.StringUtils;
import org.springframework.stereotype.Component;
import org.springframework.util.FileSystemUtils;
import reactor.core.publisher.Mono;
import reactor.core.scheduler.Scheduler;
import reactor.core.scheduler.Schedulers;
import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.Duration;
import java.time.ZoneId;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import static com.appsmith.git.constants.CommonConstants.FILE_MIGRATION_MESSAGE;
@RequiredArgsConstructor
@Component
@Slf4j
public class GitExecutorCEImpl implements GitExecutor {
private final RepositoryHelper repositoryHelper = new RepositoryHelper();
private final GitServiceConfig gitServiceConfig;
public static final DateTimeFormatter ISO_FORMATTER =
DateTimeFormatter.ISO_INSTANT.withZone(ZoneId.from(ZoneOffset.UTC));
private final Scheduler scheduler = Schedulers.boundedElastic();
private static final String SUCCESS_MERGE_STATUS = "This branch has no conflicts with the base branch.";
/**
* This method will handle the git-commit functionality. Under the hood it checks if the repo has already been
* initialised and will be initialised if git repo is not present
* @param path parent path to repo
* @param commitMessage message which will be registered for this commit
* @param authorName author details
* @param authorEmail author details
* @param doAmend To amend with the previous commit
* @return if the commit was successful
*/
@Override
public Mono<String> commitApplication(
Path path,
String commitMessage,
String authorName,
String authorEmail,
boolean isSuffixedPath,
boolean doAmend) {
final String finalAuthorName =
StringUtils.isEmptyOrNull(authorName) ? AppsmithBotAsset.APPSMITH_BOT_USERNAME : authorName;
final String finalAuthorEmail =
StringUtils.isEmptyOrNull(authorEmail) ? AppsmithBotAsset.APPSMITH_BOT_EMAIL : authorEmail;
return Mono.fromCallable(() -> {
log.debug("Trying to commit to local repo path, {}", path);
Path repoPath = path;
if (Boolean.TRUE.equals(isSuffixedPath)) {
repoPath = createRepoPath(repoPath);
}
Stopwatch processStopwatch =
StopwatchHelpers.startStopwatch(repoPath, AnalyticsEvents.GIT_COMMIT.getEventName());
// Just need to open a repository here and make a commit
try (Git git = Git.open(repoPath.toFile())) {
// Stage all the files added and modified
git.add().addFilepattern(".").call();
// Stage modified and deleted files
git.add().setUpdate(true).addFilepattern(".").call();
// Commit the changes
git.commit()
.setMessage(commitMessage)
// Only make a commit if there are any updates
.setAllowEmpty(false)
.setAuthor(finalAuthorName, finalAuthorEmail)
.setCommitter(finalAuthorName, finalAuthorEmail)
.setAmend(doAmend)
.call();
processStopwatch.stopAndLogTimeInMillis();
return "Committed successfully!";
}
})
.timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS))
.subscribeOn(scheduler);
}
/**
* Method to create a new repository to provided path
* @param repoPath path where new repo needs to be created
* @return if the operation was successful
*/
@Override
public boolean createNewRepository(Path repoPath) throws GitAPIException {
// create new repo to the mentioned path
log.debug("Trying to create new repository: {}", repoPath);
Git.init().setDirectory(repoPath.toFile()).call();
return true;
}
/**
* Method to get the commit history
* @param repoSuffix Path used to generate the repo url specific to the application for which the commit history is requested
* @return list of git commits
*/
@Override
public Mono<List<GitLogDTO>> getCommitHistory(Path repoSuffix) {
return Mono.fromCallable(() -> {
log.debug(Thread.currentThread().getName() + ": get commit history for " + repoSuffix);
List<GitLogDTO> commitLogs = new ArrayList<>();
Path repoPath = createRepoPath(repoSuffix);
Stopwatch processStopwatch = StopwatchHelpers.startStopwatch(
repoPath, AnalyticsEvents.GIT_COMMIT_HISTORY.getEventName());
try (Git git = Git.open(repoPath.toFile())) {
Iterable<RevCommit> gitLogs = git.log()
.setMaxCount(Constraint.MAX_COMMIT_LOGS)
.call();
gitLogs.forEach(revCommit -> {
PersonIdent author = revCommit.getAuthorIdent();
GitLogDTO gitLog = new GitLogDTO(
revCommit.getName(),
author.getName(),
author.getEmailAddress(),
revCommit.getFullMessage(),
ISO_FORMATTER.format(new Date(revCommit.getCommitTime() * 1000L).toInstant()));
processStopwatch.stopAndLogTimeInMillis();
commitLogs.add(gitLog);
});
return commitLogs;
}
})
.timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS))
.subscribeOn(scheduler);
}
@Override
public Path createRepoPath(Path suffix) {
return Paths.get(gitServiceConfig.getGitRootPath()).resolve(suffix);
}
/**
* Method to push changes to remote repo
* @param repoSuffix Path used to generate the repo url specific to the application which needs to be pushed to remote
* @param remoteUrl remote repo url
* @param publicKey
* @param privateKey
* @return Success message
*/
@Override
public Mono<String> pushApplication(
Path repoSuffix, String remoteUrl, String publicKey, String privateKey, String branchName) {
// We can safely assume that repo has been already initialised either in commit or clone flow and can directly
// open the repo
return Mono.fromCallable(() -> {
log.debug(Thread.currentThread().getName() + ": pushing changes to remote " + remoteUrl);
// open the repo
Path baseRepoPath = createRepoPath(repoSuffix);
Stopwatch processStopwatch =
StopwatchHelpers.startStopwatch(baseRepoPath, AnalyticsEvents.GIT_PUSH.getEventName());
try (Git git = Git.open(baseRepoPath.toFile())) {
TransportConfigCallback transportConfigCallback =
new SshTransportConfigCallback(privateKey, publicKey);
StringBuilder result = new StringBuilder("Pushed successfully with status : ");
git.push()
.setTransportConfigCallback(transportConfigCallback)
.setRemote(remoteUrl)
.call()
.forEach(pushResult -> pushResult
.getRemoteUpdates()
.forEach(remoteRefUpdate -> {
result.append(remoteRefUpdate.getStatus())
.append(",");
if (!StringUtils.isEmptyOrNull(remoteRefUpdate.getMessage())) {
result.append(remoteRefUpdate.getMessage())
.append(",");
}
}));
// We can support username and password in future if needed
// pushCommand.setCredentialsProvider(new UsernamePasswordCredentialsProvider("username",
// "password"));
processStopwatch.stopAndLogTimeInMillis();
return result.substring(0, result.length() - 1);
}
})
.timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS))
.subscribeOn(scheduler);
}
/** Clone the repo to the file path : container-volume/orgId/defaultAppId/repo/applicationData
*
* @param repoSuffix combination of orgId, defaultId and repoName
* @param remoteUrl ssh url of the git repo(we support cloning via ssh url only with deploy key)
* @param privateKey generated by us and specific to the defaultApplication
* @param publicKey generated by us and specific to the defaultApplication
* @return defaultBranchName of the repo
* */
@Override
public Mono<String> cloneApplication(Path repoSuffix, String remoteUrl, String privateKey, String publicKey) {
Stopwatch processStopwatch =
StopwatchHelpers.startStopwatch(repoSuffix, AnalyticsEvents.GIT_CLONE.getEventName());
return Mono.fromCallable(() -> {
log.debug(Thread.currentThread().getName() + ": Cloning the repo from the remote " + remoteUrl);
final TransportConfigCallback transportConfigCallback =
new SshTransportConfigCallback(privateKey, publicKey);
File file = Paths.get(gitServiceConfig.getGitRootPath())
.resolve(repoSuffix)
.toFile();
while (file.exists()) {
FileSystemUtils.deleteRecursively(file);
}
Git git = Git.cloneRepository()
.setURI(remoteUrl)
.setTransportConfigCallback(transportConfigCallback)
.setDirectory(file)
.call();
String branchName = git.getRepository().getBranch();
repositoryHelper.updateRemoteBranchTrackingConfig(branchName, git);
git.close();
processStopwatch.stopAndLogTimeInMillis();
return branchName;
})
.timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS))
.subscribeOn(scheduler);
}
@Override
public Mono<String> createAndCheckoutToBranch(Path repoSuffix, String branchName) {
// We can safely assume that repo has been already initialised either in commit or clone flow and can directly
// open the repo
Stopwatch processStopwatch =
StopwatchHelpers.startStopwatch(repoSuffix, AnalyticsEvents.GIT_CREATE_BRANCH.getEventName());
return Mono.fromCallable(() -> {
log.debug(Thread.currentThread().getName() + ": Creating branch " + branchName + "for the repo "
+ repoSuffix);
// open the repo
Path baseRepoPath = createRepoPath(repoSuffix);
try (Git git = Git.open(baseRepoPath.toFile())) {
// Create and checkout to new branch
git.checkout()
.setCreateBranch(Boolean.TRUE)
.setName(branchName)
.setUpstreamMode(CreateBranchCommand.SetupUpstreamMode.TRACK)
.call();
repositoryHelper.updateRemoteBranchTrackingConfig(branchName, git);
processStopwatch.stopAndLogTimeInMillis();
return git.getRepository().getBranch();
}
})
.timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS))
.subscribeOn(scheduler);
}
@Override
public Mono<Boolean> deleteBranch(Path repoSuffix, String branchName) {
// We can safely assume that repo has been already initialised either in commit or clone flow and can directly
// open the repo
Stopwatch processStopwatch =
StopwatchHelpers.startStopwatch(repoSuffix, AnalyticsEvents.GIT_DELETE_BRANCH.getEventName());
return Mono.fromCallable(() -> {
log.debug(Thread.currentThread().getName() + ": Deleting branch " + branchName + "for the repo "
+ repoSuffix);
// open the repo
Path baseRepoPath = createRepoPath(repoSuffix);
try (Git git = Git.open(baseRepoPath.toFile())) {
// Create and checkout to new branch
List<String> deleteBranchList = git.branchDelete()
.setBranchNames(branchName)
.setForce(Boolean.TRUE)
.call();
processStopwatch.stopAndLogTimeInMillis();
if (deleteBranchList.isEmpty()) {
return Boolean.FALSE;
}
return Boolean.TRUE;
}
})
.timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS))
.subscribeOn(scheduler);
}
@Override
public Mono<Boolean> checkoutToBranch(Path repoSuffix, String branchName) {
Stopwatch processStopwatch =
StopwatchHelpers.startStopwatch(repoSuffix, AnalyticsEvents.GIT_CHECKOUT.getEventName());
return Mono.fromCallable(() -> {
log.debug(Thread.currentThread().getName() + ": Switching to the branch " + branchName);
// We can safely assume that repo has been already initialised either in commit or clone flow and
// can directly
// open the repo
Path baseRepoPath = createRepoPath(repoSuffix);
try (Git git = Git.open(baseRepoPath.toFile())) {
if (StringUtils.equalsIgnoreCase(
branchName, git.getRepository().getBranch())) {
return Boolean.TRUE;
}
// Create and checkout to new branch
String checkedOutBranch = git.checkout()
.setCreateBranch(Boolean.FALSE)
.setName(branchName)
.setUpstreamMode(CreateBranchCommand.SetupUpstreamMode.SET_UPSTREAM)
.call()
.getName();
processStopwatch.stopAndLogTimeInMillis();
return StringUtils.equalsIgnoreCase(checkedOutBranch, "refs/heads/" + branchName);
} catch (Exception e) {
throw new Exception(e);
}
})
.timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS))
.subscribeOn(scheduler);
}
@Override
public Mono<MergeStatusDTO> pullApplication(
Path repoSuffix, String remoteUrl, String branchName, String privateKey, String publicKey)
throws IOException {
Stopwatch processStopwatch =
StopwatchHelpers.startStopwatch(repoSuffix, AnalyticsEvents.GIT_PULL.getEventName());
TransportConfigCallback transportConfigCallback = new SshTransportConfigCallback(privateKey, publicKey);
try (Git git = Git.open(createRepoPath(repoSuffix).toFile())) {
return Mono.fromCallable(() -> {
log.debug(Thread.currentThread().getName() + ": Pull changes from remote " + remoteUrl
+ " for the branch " + branchName);
// checkout the branch on which the merge command is run
git.checkout()
.setName(branchName)
.setCreateBranch(false)
.call();
MergeResult mergeResult = git.pull()
.setRemoteBranchName(branchName)
.setTransportConfigCallback(transportConfigCallback)
.setFastForward(MergeCommand.FastForwardMode.FF)
.call()
.getMergeResult();
MergeStatusDTO mergeStatus = new MergeStatusDTO();
Long count =
Arrays.stream(mergeResult.getMergedCommits()).count();
if (mergeResult.getMergeStatus().isSuccessful()) {
mergeStatus.setMergeAble(true);
mergeStatus.setStatus(count + " commits merged from origin/" + branchName);
} else {
// If there are conflicts add the conflicting file names to the response structure
mergeStatus.setMergeAble(false);
List<String> mergeConflictFiles = new ArrayList<>();
if (!Optional.ofNullable(mergeResult.getConflicts()).isEmpty()) {
mergeConflictFiles.addAll(
mergeResult.getConflicts().keySet());
}
mergeStatus.setConflictingFiles(mergeConflictFiles);
// On merge conflicts abort the merge => git merge --abort
git.getRepository().writeMergeCommitMsg(null);
git.getRepository().writeMergeHeads(null);
processStopwatch.stopAndLogTimeInMillis();
throw new org.eclipse.jgit.errors.CheckoutConflictException(mergeConflictFiles.toString());
}
processStopwatch.stopAndLogTimeInMillis();
return mergeStatus;
})
.onErrorResume(error -> {
try {
return resetToLastCommit(git).flatMap(ignore -> Mono.error(error));
} catch (GitAPIException e) {
log.error("Error for hard resetting to latest commit {0}", e);
return Mono.error(e);
}
})
.timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS))
.subscribeOn(scheduler);
}
}
@Override
public Mono<List<GitBranchDTO>> listBranches(Path repoSuffix) {
Path baseRepoPath = createRepoPath(repoSuffix);
return Mono.fromCallable(() -> {
log.debug(Thread.currentThread().getName() + ": Get branches for the application " + repoSuffix);
Git git = Git.open(baseRepoPath.toFile());
List<Ref> refList = git.branchList()
.setListMode(ListBranchCommand.ListMode.ALL)
.call();
List<GitBranchDTO> branchList = new ArrayList<>();
GitBranchDTO gitBranchDTO = new GitBranchDTO();
if (refList.isEmpty()) {
gitBranchDTO.setBranchName(git.getRepository().getBranch());
branchList.add(gitBranchDTO);
} else {
for (Ref ref : refList) {
// if (!ref.getName().equals(defaultBranch)) {
gitBranchDTO = new GitBranchDTO();
gitBranchDTO.setBranchName(ref.getName()
.replace("refs/", "")
.replace("heads/", "")
.replace("remotes/", ""));
branchList.add(gitBranchDTO);
}
}
git.close();
return branchList;
})
.timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS))
.subscribeOn(scheduler);
}
@Override
public Mono<String> getRemoteDefaultBranch(Path repoSuffix, String remoteUrl, String privateKey, String publicKey) {
Path baseRepoPath = createRepoPath(repoSuffix);
return Mono.fromCallable(() -> {
TransportConfigCallback transportConfigCallback =
new SshTransportConfigCallback(privateKey, publicKey);
Git git = Git.open(baseRepoPath.toFile());
return git.lsRemote()
.setRemote(remoteUrl)
.setTransportConfigCallback(transportConfigCallback)
.callAsMap()
.get("HEAD")
.getTarget()
.getName()
.replace("refs/heads/", "");
})
.timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS))
.subscribeOn(scheduler);
}
/**
* This method will handle the git-status functionality
*
* @param repoPath Path to actual repo
* @param branchName branch name for which the status is required
* @return Map of file names those are modified, conflicted etc.
*/
@Override
public Mono<GitStatusDTO> getStatus(Path repoPath, String branchName) {
Stopwatch processStopwatch =
StopwatchHelpers.startStopwatch(repoPath, AnalyticsEvents.GIT_STATUS.getEventName());
return Mono.fromCallable(() -> {
try (Git git = Git.open(repoPath.toFile())) {
log.debug(Thread.currentThread().getName() + ": Get status for repo " + repoPath + ", branch "
+ branchName);
Status status = git.status().call();
GitStatusDTO response = new GitStatusDTO();
Set<String> modifiedAssets = new HashSet<>();
modifiedAssets.addAll(status.getModified());
modifiedAssets.addAll(status.getAdded());
modifiedAssets.addAll(status.getRemoved());
modifiedAssets.addAll(status.getUncommittedChanges());
modifiedAssets.addAll(status.getUntracked());
response.setAdded(status.getAdded());
response.setRemoved(status.getRemoved());
populateModifiedEntities(status, response, modifiedAssets);
BranchTrackingStatus trackingStatus = BranchTrackingStatus.of(git.getRepository(), branchName);
if (trackingStatus != null) {
response.setAheadCount(trackingStatus.getAheadCount());
response.setBehindCount(trackingStatus.getBehindCount());
response.setRemoteBranch(trackingStatus.getRemoteTrackingBranch());
} else {
log.debug(
"Remote tracking details not present for branch: {}, repo: {}",
branchName,
repoPath);
response.setAheadCount(0);
response.setBehindCount(0);
response.setRemoteBranch("untracked");
}
// Remove modified changes from current branch so that checkout to other branches will be
// possible
if (!status.isClean()) {
return resetToLastCommit(git).map(ref -> {
processStopwatch.stopAndLogTimeInMillis();
return response;
});
}
processStopwatch.stopAndLogTimeInMillis();
return Mono.just(response);
}
})
.timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS))
.flatMap(response -> response)
.subscribeOn(scheduler);
}
protected void populateModifiedEntities(Status status, GitStatusDTO response, Set<String> modifiedAssets) {
Set<String> queriesModified = new HashSet<>();
Set<String> jsObjectsModified = new HashSet<>();
Set<String> pagesModified = new HashSet<>();
int modifiedPages = 0;
int modifiedQueries = 0;
int modifiedJSObjects = 0;
int modifiedDatasources = 0;
int modifiedJSLibs = 0;
for (String x : modifiedAssets) {
// begins with pages and filename and parent name should be same or contains widgets
if (x.contains(CommonConstants.WIDGETS)) {
if (!pagesModified.contains(getPageName(x))) {
pagesModified.add(getPageName(x));
modifiedPages++;
}
} else if (isAModifiedPage(x)) {
if (!pagesModified.contains(getPageName(x))) {
pagesModified.add(getPageName(x));
modifiedPages++;
}
} else if (x.contains(GitDirectories.ACTION_DIRECTORY + CommonConstants.DELIMITER_PATH)) {
String queryName = x.split(GitDirectories.ACTION_DIRECTORY + CommonConstants.DELIMITER_PATH)[1];
int position = queryName.indexOf(CommonConstants.DELIMITER_PATH);
if (position != -1) {
queryName = queryName.substring(0, position);
String pageName = x.split(CommonConstants.DELIMITER_PATH)[1];
if (!queriesModified.contains(pageName + queryName)) {
queriesModified.add(pageName + queryName);
modifiedQueries++;
}
}
} else if (x.contains(GitDirectories.ACTION_COLLECTION_DIRECTORY + CommonConstants.DELIMITER_PATH)
&& !x.endsWith(CommonConstants.JSON_EXTENSION)) {
String queryName = x.substring(x.lastIndexOf(CommonConstants.DELIMITER_PATH) + 1);
String pageName = x.split(CommonConstants.DELIMITER_PATH)[1];
if (!jsObjectsModified.contains(pageName + queryName)) {
jsObjectsModified.add(pageName + queryName);
modifiedJSObjects++;
}
} else if (x.contains(GitDirectories.DATASOURCE_DIRECTORY + CommonConstants.DELIMITER_PATH)) {
modifiedDatasources++;
} else if (x.contains(GitDirectories.JS_LIB_DIRECTORY + CommonConstants.DELIMITER_PATH)) {
modifiedJSLibs++;
// remove this code in future when all the older format js libs are migrated to new
// format
if (x.contains("js.json")) {
/*
As this updated filename has color(:), it means this is the older format js
lib file that we're going to rename with the format without colon.
Hence, we need to show a message to user saying this might be a system level change.
*/
response.setMigrationMessage(FILE_MIGRATION_MESSAGE);
}
}
}
response.setModified(modifiedAssets);
response.setConflicting(status.getConflicting());
response.setIsClean(status.isClean());
response.setModifiedPages(modifiedPages);
response.setModifiedQueries(modifiedQueries);
response.setModifiedJSObjects(modifiedJSObjects);
response.setModifiedDatasources(modifiedDatasources);
response.setModifiedJSLibs(modifiedJSLibs);
}
protected boolean isAModifiedPage(String x) {
return !x.contains(CommonConstants.WIDGETS)
&& x.startsWith(GitDirectories.PAGE_DIRECTORY)
&& !x.contains(GitDirectories.ACTION_DIRECTORY)
&& !x.contains(GitDirectories.ACTION_COLLECTION_DIRECTORY);
}
private String getPageName(String path) {
String[] pathArray = path.split(CommonConstants.DELIMITER_PATH);
return pathArray[1];
}
@Override
public Mono<String> mergeBranch(Path repoSuffix, String sourceBranch, String destinationBranch) {
return Mono.fromCallable(() -> {
Stopwatch processStopwatch =
StopwatchHelpers.startStopwatch(repoSuffix, AnalyticsEvents.GIT_MERGE.getEventName());
log.debug(Thread.currentThread().getName() + ": Merge branch " + sourceBranch + " on "
+ destinationBranch);
try (Git git = Git.open(createRepoPath(repoSuffix).toFile())) {
try {
// checkout the branch on which the merge command is run
git.checkout()
.setName(destinationBranch)
.setCreateBranch(false)
.call();
MergeResult mergeResult = git.merge()
.include(git.getRepository().findRef(sourceBranch))
.setStrategy(MergeStrategy.RECURSIVE)
.call();
processStopwatch.stopAndLogTimeInMillis();
return mergeResult.getMergeStatus().name();
} catch (GitAPIException e) {
// On merge conflicts abort the merge => git merge --abort
git.getRepository().writeMergeCommitMsg(null);
git.getRepository().writeMergeHeads(null);
processStopwatch.stopAndLogTimeInMillis();
throw new Exception(e);
}
}
})
.onErrorResume(error -> {
try {
return resetToLastCommit(repoSuffix, destinationBranch).thenReturn(error.getMessage());
} catch (GitAPIException | IOException e) {
log.error("Error while hard resetting to latest commit {0}", e);
return Mono.error(e);
}
})
.timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS))
.subscribeOn(scheduler);
}
@Override
public Mono<String> fetchRemote(
Path repoSuffix,
String publicKey,
String privateKey,
boolean isRepoPath,
String branchName,
boolean isFetchAll) {
Stopwatch processStopwatch =
StopwatchHelpers.startStopwatch(repoSuffix, AnalyticsEvents.GIT_FETCH.getEventName());
Path repoPath = Boolean.TRUE.equals(isRepoPath) ? repoSuffix : createRepoPath(repoSuffix);
return Mono.fromCallable(() -> {
TransportConfigCallback config = new SshTransportConfigCallback(privateKey, publicKey);
try (Git git = Git.open(repoPath.toFile())) {
String fetchMessages;
if (Boolean.TRUE.equals(isFetchAll)) {
fetchMessages = git.fetch()
.setRemoveDeletedRefs(true)
.setTransportConfigCallback(config)
.call()
.getMessages();
} else {
RefSpec ref =
new RefSpec("refs/heads/" + branchName + ":refs/remotes/origin/" + branchName);
fetchMessages = git.fetch()
.setRefSpecs(ref)
.setRemoveDeletedRefs(true)
.setTransportConfigCallback(config)
.call()
.getMessages();
}
processStopwatch.stopAndLogTimeInMillis();
return fetchMessages;
}
})
.onErrorResume(error -> {
log.error(error.getMessage());
return Mono.error(error);
})
.timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS))
.subscribeOn(scheduler);
}
@Override
public Mono<MergeStatusDTO> isMergeBranch(Path repoSuffix, String sourceBranch, String destinationBranch) {
Stopwatch processStopwatch =
StopwatchHelpers.startStopwatch(repoSuffix, AnalyticsEvents.GIT_MERGE_CHECK.getEventName());
return Mono.fromCallable(() -> {
log.debug(
Thread.currentThread().getName()
+ ": Check mergeability for repo {} with src: {}, dest: {}",
repoSuffix,
sourceBranch,
destinationBranch);
try (Git git = Git.open(createRepoPath(repoSuffix).toFile())) {
// checkout the branch on which the merge command is run
try {
git.checkout()
.setName(destinationBranch)
.setCreateBranch(false)
.call();
} catch (GitAPIException e) {
if (e instanceof CheckoutConflictException) {
MergeStatusDTO mergeStatus = new MergeStatusDTO();
mergeStatus.setMergeAble(false);
mergeStatus.setConflictingFiles(((CheckoutConflictException) e).getConflictingPaths());
processStopwatch.stopAndLogTimeInMillis();
return mergeStatus;
}
}
MergeResult mergeResult = git.merge()
.include(git.getRepository().findRef(sourceBranch))
.setFastForward(MergeCommand.FastForwardMode.NO_FF)
.setCommit(false)
.call();
MergeStatusDTO mergeStatus = new MergeStatusDTO();
if (mergeResult.getMergeStatus().isSuccessful()) {
mergeStatus.setMergeAble(true);
mergeStatus.setMessage(SUCCESS_MERGE_STATUS);
} else {
// If there aer conflicts add the conflicting file names to the response structure
mergeStatus.setMergeAble(false);
List<String> mergeConflictFiles =
new ArrayList<>(mergeResult.getConflicts().keySet());
mergeStatus.setConflictingFiles(mergeConflictFiles);
StringBuilder errorMessage = new StringBuilder();
if (mergeResult.getMergeStatus().equals(MergeResult.MergeStatus.CONFLICTING)) {
errorMessage.append("Conflicts");
} else {
errorMessage.append(mergeResult.getMergeStatus().toString());
}
errorMessage
.append(" while merging branch: ")
.append(destinationBranch)
.append(" <= ")
.append(sourceBranch);
mergeStatus.setMessage(errorMessage.toString());
mergeStatus.setReferenceDoc(ErrorReferenceDocUrl.GIT_MERGE_CONFLICT.getDocUrl());
}
mergeStatus.setStatus(mergeResult.getMergeStatus().name());
return mergeStatus;
}
})
.flatMap(status -> {
try {
// Revert uncommitted changes if any
return resetToLastCommit(repoSuffix, destinationBranch).map(ignore -> {
processStopwatch.stopAndLogTimeInMillis();
return status;
});
} catch (GitAPIException | IOException e) {
log.error("Error for hard resetting to latest commit {0}", e);
return Mono.error(e);
}
})
.timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS))
.subscribeOn(scheduler);
}
public Mono<String> checkoutRemoteBranch(Path repoSuffix, String branchName) {
// We can safely assume that repo has been already initialised either in commit or clone flow and can directly
// open the repo
return Mono.fromCallable(() -> {
log.debug(Thread.currentThread().getName() + ": Checking out remote branch origin/" + branchName
+ " for the repo " + repoSuffix);
// open the repo
Path baseRepoPath = createRepoPath(repoSuffix);
try (Git git = Git.open(baseRepoPath.toFile())) {
// Create and checkout to new branch
git.checkout()
.setCreateBranch(Boolean.TRUE)
.setName(branchName)
.setUpstreamMode(CreateBranchCommand.SetupUpstreamMode.TRACK)
.setStartPoint("origin/" + branchName)
.call();
StoredConfig config = git.getRepository().getConfig();
config.setString("branch", branchName, "remote", "origin");
config.setString("branch", branchName, "merge", "refs/heads/" + branchName);
config.save();
return git.getRepository().getBranch();
}
})
.timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS))
.subscribeOn(scheduler);
}
@Override
public Mono<Boolean> testConnection(String publicKey, String privateKey, String remoteUrl) {
return Mono.fromCallable(() -> {
TransportConfigCallback transportConfigCallback =
new SshTransportConfigCallback(privateKey, publicKey);
Git.lsRemoteRepository()
.setTransportConfigCallback(transportConfigCallback)
.setRemote(remoteUrl)
.setHeads(true)
.setTags(true)
.call();
return true;
})
.timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS))
.subscribeOn(scheduler);
}
private Mono<Ref> resetToLastCommit(Git git) throws GitAPIException {
Stopwatch processStopwatch = StopwatchHelpers.startStopwatch(
git.getRepository().getDirectory().toPath().getParent(), AnalyticsEvents.GIT_RESET.getEventName());
return Mono.fromCallable(() -> {
// Remove tracked files
Ref ref = git.reset().setMode(ResetCommand.ResetType.HARD).call();
// Remove untracked files
git.clean().setForce(true).setCleanDirectories(true).call();
processStopwatch.stopAndLogTimeInMillis();
return ref;
})
.timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS))
.subscribeOn(scheduler);
}
public Mono<Boolean> resetToLastCommit(Path repoSuffix, String branchName) throws GitAPIException, IOException {
try (Git git = Git.open(createRepoPath(repoSuffix).toFile())) {
return this.resetToLastCommit(git)
.flatMap(ref -> checkoutToBranch(repoSuffix, branchName))
.flatMap(checkedOut -> {
try {
return resetToLastCommit(git).thenReturn(true);
} catch (GitAPIException e) {
log.error(e.getMessage());
return Mono.error(e);
}
});
}
}
public Mono<Boolean> resetHard(Path repoSuffix, String branchName) {
return this.checkoutToBranch(repoSuffix, branchName)
.flatMap(aBoolean -> {
try (Git git = Git.open(createRepoPath(repoSuffix).toFile())) {
Ref ref = git.reset()
.setMode(ResetCommand.ResetType.HARD)
.setRef("HEAD~1")
.call();
return Mono.just(true);
} catch (GitAPIException | IOException e) {
log.error("Error while resetting the commit, {}", e.getMessage());
}
return Mono.just(false);
})
.timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS))
.subscribeOn(scheduler);
}
public Mono<Boolean> rebaseBranch(Path repoSuffix, String branchName) {
return this.checkoutToBranch(repoSuffix, branchName)
.flatMap(isCheckedOut -> {
try (Git git = Git.open(createRepoPath(repoSuffix).toFile())) {
RebaseResult result =
git.rebase().setUpstream("origin/" + branchName).call();
if (result.getStatus().isSuccessful()) {
return Mono.just(true);
} else {
log.error(
"Error while rebasing the branch, {}, {}",
result.getStatus().name(),
result.getConflicts());
git.rebase()
.setUpstream("origin/" + branchName)
.setOperation(RebaseCommand.Operation.ABORT)
.call();
return Mono.error(new Exception("Error while rebasing the branch, "
+ result.getStatus().name()));
}
} catch (GitAPIException | IOException e) {
log.error("Error while rebasing the branch, {}", e.getMessage());
return Mono.error(e);
}
})
.timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS))
.subscribeOn(scheduler);
}
@Override
public Mono<BranchTrackingStatus> getBranchTrackingStatus(Path repoPath, String branchName) {
return Mono.fromCallable(() -> {
try (Git git = Git.open(repoPath.toFile())) {
return BranchTrackingStatus.of(git.getRepository(), branchName);
}
})
.timeout(Duration.ofMillis(Constraint.TIMEOUT_MILLIS))
.subscribeOn(scheduler);
}
}

23
app/server/appsmith-interfaces/src/main/java/com/appsmith/external/constants/GitConstants.java vendored
View File

@ -1,24 +1,5 @@
package com.appsmith.external.constants;
public class GitConstants {
// This will be used as a key separator for action and jsobjects name
// pageName{{seperator}}entityName this is needed to filter the entities to save in appropriate page directory
public static final String NAME_SEPARATOR = "##ENTITY_SEPARATOR##";
public static final String PAGE_LIST = "pageList";
public static final String CUSTOM_JS_LIB_LIST = "customJSLibList";
public static final String ACTION_LIST = "actionList";
public static final String ACTION_COLLECTION_LIST = "actionCollectionList";
import com.appsmith.external.constants.ce.GitConstantsCE;
public static final String DEFAULT_COMMIT_MESSAGE = "System generated commit, ";
public static final String EMPTY_COMMIT_ERROR_MESSAGE = "On current branch nothing to commit, working tree clean";
public static final String MERGE_CONFLICT_BRANCH_NAME = "_mergeConflict";
public static final String CONFLICTED_SUCCESS_MESSAGE = "branch has been created from conflicted state. Please "
+ "resolve merge conflicts in remote and pull again";
public static final String GIT_CONFIG_ERROR =
"Unable to find the git configuration, please configure your application "
+ "with git to use version control service";
public static final String GIT_PROFILE_ERROR = "Unable to find git author configuration for logged-in user. You can"
+ " set up a git profile from the user profile section.";
}
public class GitConstants extends GitConstantsCE {}

24
app/server/appsmith-interfaces/src/main/java/com/appsmith/external/constants/ce/GitConstantsCE.java vendored Normal file
View File

@ -0,0 +1,24 @@
package com.appsmith.external.constants.ce;
public class GitConstantsCE {
// This will be used as a key separator for action and jsobjects name
// pageName{{seperator}}entityName this is needed to filter the entities to save in appropriate page directory
public static final String NAME_SEPARATOR = "##ENTITY_SEPARATOR##";
public static final String PAGE_LIST = "pageList";
public static final String CUSTOM_JS_LIB_LIST = "customJSLibList";
public static final String ACTION_LIST = "actionList";
public static final String ACTION_COLLECTION_LIST = "actionCollectionList";
public static final String DEFAULT_COMMIT_MESSAGE = "System generated commit, ";
public static final String EMPTY_COMMIT_ERROR_MESSAGE = "On current branch nothing to commit, working tree clean";
public static final String MERGE_CONFLICT_BRANCH_NAME = "_mergeConflict";
public static final String CONFLICTED_SUCCESS_MESSAGE = "branch has been created from conflicted state. Please "
+ "resolve merge conflicts in remote and pull again";
public static final String GIT_CONFIG_ERROR =
"Unable to find the git configuration, please configure your application "
+ "with git to use version control service";
public static final String GIT_PROFILE_ERROR = "Unable to find git author configuration for logged-in user. You can"
+ " set up a git profile from the user profile section.";
}

53
app/server/appsmith-interfaces/src/main/java/com/appsmith/external/dtos/GitStatusDTO.java vendored
View File

@ -1,57 +1,12 @@
package com.appsmith.external.dtos;
import com.appsmith.external.constants.Assets;
import com.appsmith.external.dtos.ce.GitStatusCE_DTO;
import lombok.Data;
import java.util.Set;
import lombok.EqualsAndHashCode;
/**
* DTO to convey the status local git repo
*/
@EqualsAndHashCode(callSuper = true)
@Data
public class GitStatusDTO {
// Name of modified, added and deleted resources in local git repo
Set<String> modified;
// Name of added resources to local git repo
Set<String> added;
// Name of deleted resources from local git repo
Set<String> removed;
// Name of conflicting resources
Set<String> conflicting;
Boolean isClean;
// number of modified custom JS libs
int modifiedJSLibs;
// number of modified pages
int modifiedPages;
// number of modified actions
int modifiedQueries;
// number of modified JSObjects
int modifiedJSObjects;
// number of modified JSObjects
int modifiedDatasources;
// number of local commits which are not present in remote repo
Integer aheadCount;
// number of remote commits which are not present in local repo
Integer behindCount;
// Remote tracking branch name
String remoteBranch;
// Documentation url for discard and pull functionality
String discardDocUrl = Assets.GIT_DISCARD_DOC_URL;
// File Format migration
String migrationMessage = "";
}
public class GitStatusDTO extends GitStatusCE_DTO {}

57
app/server/appsmith-interfaces/src/main/java/com/appsmith/external/dtos/ce/GitStatusCE_DTO.java vendored Normal file
View File

@ -0,0 +1,57 @@
package com.appsmith.external.dtos.ce;
import com.appsmith.external.constants.Assets;
import lombok.Data;
import java.util.Set;
/**
* DTO to convey the status local git repo
*/
@Data
public class GitStatusCE_DTO {
// Name of modified, added and deleted resources in local git repo
Set<String> modified;
// Name of added resources to local git repo
Set<String> added;
// Name of deleted resources from local git repo
Set<String> removed;
// Name of conflicting resources
Set<String> conflicting;
Boolean isClean;
// number of modified custom JS libs
int modifiedJSLibs;
// number of modified pages
int modifiedPages;
// number of modified actions
int modifiedQueries;
// number of modified JSObjects
int modifiedJSObjects;
// number of modified JSObjects
int modifiedDatasources;
// number of local commits which are not present in remote repo
Integer aheadCount;
// number of remote commits which are not present in local repo
Integer behindCount;
// Remote tracking branch name
String remoteBranch;
// Documentation url for discard and pull functionality
String discardDocUrl = Assets.GIT_DISCARD_DOC_URL;
// File Format migration
String migrationMessage = "";
}

29
app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/ApplicationGitReference.java vendored
View File

@ -1,36 +1,15 @@
package com.appsmith.external.models;
import com.appsmith.external.models.ce.ApplicationGitReferenceCE;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.NoArgsConstructor;
import java.util.Map;
import java.util.Set;
/**
* A DTO class to hold complete information about an application, which will then be serialized to a file so as to
* export/save that application into a json files.
*/
@EqualsAndHashCode(callSuper = true)
@Data
@NoArgsConstructor
public class ApplicationGitReference {
Object application;
Object metadata;
Object theme;
Map<String, Object> actions;
Map<String, String> actionBody;
Map<String, Object> actionCollections;
Map<String, String> actionCollectionBody;
Map<String, Object> pages;
Map<String, String> pageDsl;
Map<String, Object> datasources;
Map<String, Object> jsLibraries;
/**
* This field will be used to store map of files to be updated in local file system by comparing the recent
* changes in database and the last local git commit.
* This field can be used while saving resources to local file system and only update the resource files which
* are updated in the database.
*/
Map<String, Set<String>> updatedResources;
}
public class ApplicationGitReference extends ApplicationGitReferenceCE {}

32
app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/DefaultResources.java vendored
View File

@ -1,37 +1,13 @@
package com.appsmith.external.models;
import com.appsmith.external.models.ce.DefaultResourcesCE;
import lombok.Data;
import lombok.EqualsAndHashCode;
/**
* This class will be used for connecting resources across branches for git connected application
* e.g. Page1 in branch1 will have the same defaultResources.pageId as of Page1 of branch2
*/
@EqualsAndHashCode(callSuper = true)
@Data
public class DefaultResources {
/**
* When present, actionId will hold the default action id
*/
String actionId;
/**
* When present, applicationId will hold the default application id
*/
String applicationId;
/**
* When present, pageId will hold the default page id
*/
String pageId;
/**
* When present, collectionId will hold the default collection id
*/
String collectionId;
/**
* When present, branchName will hold the current branch name.
* For example, if we've a page in both main and develop branch, then default resources of those two pages will
* have same applicationId, pageId but branchName will contain the corresponding branch name.
*/
String branchName;
}
public class DefaultResources extends DefaultResourcesCE {}

19
app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/ce/ActionCE_DTO.java vendored
View File

@ -171,8 +171,7 @@ public class ActionCE_DTO implements Identifiable, Executable {
DefaultResources defaultResources;
// This field will be used to store analytics data related to this specific domain object. It's been introduced in
// order to track
// success metrics of modules. Learn more on GitHub issue#24734
// order to track success metrics of modules. Learn more on GitHub issue#24734
@JsonView(Views.Public.class)
AnalyticsInfo eventData;
@ -216,6 +215,7 @@ public class ActionCE_DTO implements Identifiable, Executable {
}
public void sanitiseToExportDBObject() {
this.resetTransientFields();
this.setEventData(null);
this.setDefaultResources(null);
this.setCacheResponse(null);
@ -302,4 +302,19 @@ public class ActionCE_DTO implements Identifiable, Executable {
this.datasource.setIsAutoGenerated(true);
}
}
protected void resetTransientFields() {
this.setId(null);
this.setApplicationId(null);
this.setWorkspaceId(null);
this.setPluginId(null);
this.setPluginName(null);
this.setPluginType(null);
this.setErrorReports(null);
// this.setMessages(null);
this.setTemplateId(null);
this.setProvider(null);
this.setProviderId(null);
this.setDocumentation(null);
}
}

36
app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/ce/ApplicationGitReferenceCE.java vendored Normal file
View File

@ -0,0 +1,36 @@
package com.appsmith.external.models.ce;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.util.Map;
import java.util.Set;
/**
* A DTO class to hold complete information about an application, which will then be serialized to a file so as to
* export/save that application into a json files.
*/
@Data
@NoArgsConstructor
public class ApplicationGitReferenceCE {
Object application;
Object metadata;
Object theme;
Map<String, Object> actions;
Map<String, String> actionBody;
Map<String, Object> actionCollections;
Map<String, String> actionCollectionBody;
Map<String, Object> pages;
Map<String, String> pageDsl;
Map<String, Object> datasources;
Map<String, Object> jsLibraries;
/**
* This field will be used to store map of files to be updated in local file system by comparing the recent
* changes in database and the last local git commit.
* This field can be used while saving resources to local file system and only update the resource files which
* are updated in the database.
*/
Map<String, Set<String>> updatedResources;
}

37
app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/ce/DefaultResourcesCE.java vendored Normal file
View File

@ -0,0 +1,37 @@
package com.appsmith.external.models.ce;
import lombok.Data;
/**
* This class will be used for connecting resources across branches for git connected application
* e.g. Page1 in branch1 will have the same defaultResources.pageId as of Page1 of branch2
*/
@Data
public class DefaultResourcesCE {
/**
* When present, actionId will hold the default action id
*/
String actionId;
/**
* When present, applicationId will hold the default application id
*/
String applicationId;
/**
* When present, pageId will hold the default page id
*/
String pageId;
/**
* When present, collectionId will hold the default collection id
*/
String collectionId;
/**
* When present, branchName will hold the current branch name.
* For example, if we've a page in both main and develop branch, then default resources of those two pages will
* have same applicationId, pageId but branchName will contain the corresponding branch name.
*/
String branchName;
}

22
app/server/appsmith-server/src/main/java/com/appsmith/server/actioncollections/base/ActionCollectionServiceCEImpl.java
View File

@ -9,6 +9,7 @@ import com.appsmith.server.acl.AclPermission;
import com.appsmith.server.acl.PolicyGenerator;
import com.appsmith.server.applications.base.ApplicationService;
import com.appsmith.server.constants.FieldName;
import com.appsmith.server.defaultresources.DefaultResourcesService;
import com.appsmith.server.domains.Action;
import com.appsmith.server.domains.ActionCollection;
import com.appsmith.server.domains.NewAction;
@ -29,7 +30,6 @@ import com.appsmith.server.solutions.ApplicationPermission;
import jakarta.validation.Validator;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import org.bson.types.ObjectId;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Sort;
@ -38,6 +38,7 @@ import org.springframework.data.mongodb.core.convert.MongoConverter;
import org.springframework.util.CollectionUtils;
import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap;
import org.springframework.util.StringUtils;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import reactor.core.scheduler.Scheduler;
@ -66,6 +67,7 @@ public class ActionCollectionServiceCEImpl extends BaseService<ActionCollectionR
private final ResponseUtils responseUtils;
private final ApplicationPermission applicationPermission;
private final ActionPermission actionPermission;
private final DefaultResourcesService<ActionCollection> defaultResourcesService;
@Autowired
public ActionCollectionServiceCEImpl(
@ -80,7 +82,8 @@ public class ActionCollectionServiceCEImpl extends BaseService<ActionCollectionR
ApplicationService applicationService,
ResponseUtils responseUtils,
ApplicationPermission applicationPermission,
ActionPermission actionPermission) {
ActionPermission actionPermission,
DefaultResourcesService<ActionCollection> defaultResourcesService) {
super(scheduler, validator, mongoConverter, reactiveMongoTemplate, repository, analyticsService);
this.newActionService = newActionService;
@ -89,6 +92,7 @@ public class ActionCollectionServiceCEImpl extends BaseService<ActionCollectionR
this.responseUtils = responseUtils;
this.applicationPermission = applicationPermission;
this.actionPermission = actionPermission;
this.defaultResourcesService = defaultResourcesService;
}
@Override
@ -333,9 +337,10 @@ public class ActionCollectionServiceCEImpl extends BaseService<ActionCollectionR
copyNewFieldValuesIntoOldObject(actionCollectionDTO, dbActionCollection.getUnpublishedCollection());
// No need to save defaultPageId at actionCollection level as this will be stored inside the
// actionCollectionDTO
DefaultResourcesUtils.createDefaultIdsOrUpdateWithGivenResourceIds(
defaultResourcesService.initialize(
dbActionCollection,
dbActionCollection.getDefaultResources().getBranchName());
dbActionCollection.getDefaultResources().getBranchName(),
false);
return dbActionCollection;
})
.flatMap(actionCollection -> this.update(id, actionCollection))
@ -492,7 +497,14 @@ public class ActionCollectionServiceCEImpl extends BaseService<ActionCollectionR
@Override
public Flux<ActionCollection> findByPageIdsForExport(List<String> pageIds, Optional<AclPermission> permission) {
return repository.findByPageIds(pageIds, permission);
return repository.findByPageIds(pageIds, permission).doOnNext(actionCollection -> {
actionCollection.getUnpublishedCollection().populateTransientFields(actionCollection);
if (actionCollection.getPublishedCollection() != null
&& StringUtils.hasText(
actionCollection.getPublishedCollection().getName())) {
actionCollection.getPublishedCollection().populateTransientFields(actionCollection);
}
});
}
@Override

8
app/server/appsmith-server/src/main/java/com/appsmith/server/actioncollections/base/ActionCollectionServiceImpl.java
View File

@ -2,6 +2,8 @@ package com.appsmith.server.actioncollections.base;
import com.appsmith.server.acl.PolicyGenerator;
import com.appsmith.server.applications.base.ApplicationService;
import com.appsmith.server.defaultresources.DefaultResourcesService;
import com.appsmith.server.domains.ActionCollection;
import com.appsmith.server.helpers.ResponseUtils;
import com.appsmith.server.newactions.base.NewActionService;
import com.appsmith.server.repositories.ActionCollectionRepository;
@ -31,7 +33,8 @@ public class ActionCollectionServiceImpl extends ActionCollectionServiceCEImpl i
ApplicationService applicationService,
ResponseUtils responseUtils,
ApplicationPermission applicationPermission,
ActionPermission actionPermission) {
ActionPermission actionPermission,
DefaultResourcesService<ActionCollection> defaultResourcesService) {
super(
scheduler,
validator,
@ -44,6 +47,7 @@ public class ActionCollectionServiceImpl extends ActionCollectionServiceCEImpl i
applicationService,
responseUtils,
applicationPermission,
actionPermission);
actionPermission,
defaultResourcesService);
}
}

44
app/server/appsmith-server/src/main/java/com/appsmith/server/actioncollections/defaultresources/ActionCollectionDTODefaultResourcesServiceCEImpl.java Normal file
View File

@ -0,0 +1,44 @@
package com.appsmith.server.actioncollections.defaultresources;
import com.appsmith.external.models.DefaultResources;
import com.appsmith.server.defaultresources.DefaultResourcesServiceCE;
import com.appsmith.server.dtos.ActionCollectionDTO;
import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils;
@Service
public class ActionCollectionDTODefaultResourcesServiceCEImpl
implements DefaultResourcesServiceCE<ActionCollectionDTO> {
@Override
public ActionCollectionDTO initialize(
ActionCollectionDTO domainObject, String branchName, boolean resetExistingValues) {
DefaultResources existingDefaultResources = domainObject.getDefaultResources();
DefaultResources defaultResources = new DefaultResources();
String defaultPageId = domainObject.getPageId();
if (existingDefaultResources != null && !resetExistingValues) {
// Check if there are properties to be copied over from existing
if (StringUtils.hasText(existingDefaultResources.getPageId())) {
defaultPageId = existingDefaultResources.getPageId();
}
}
defaultResources.setPageId(defaultPageId);
domainObject.setDefaultResources(defaultResources);
return domainObject;
}
@Override
public ActionCollectionDTO setFromOtherBranch(
ActionCollectionDTO domainObject, ActionCollectionDTO defaultDomainObject, String branchName) {
DefaultResources defaultResources = new DefaultResources();
defaultResources.setPageId(defaultDomainObject.getDefaultResources().getPageId());
domainObject.setDefaultResources(defaultResources);
return domainObject;
}
}

9
app/server/appsmith-server/src/main/java/com/appsmith/server/actioncollections/defaultresources/ActionCollectionDTODefaultResourcesServiceImpl.java Normal file
View File

@ -0,0 +1,9 @@
package com.appsmith.server.actioncollections.defaultresources;
import com.appsmith.server.defaultresources.DefaultResourcesService;
import com.appsmith.server.dtos.ActionCollectionDTO;
import org.springframework.stereotype.Service;
@Service
public class ActionCollectionDTODefaultResourcesServiceImpl extends ActionCollectionDTODefaultResourcesServiceCEImpl
implements DefaultResourcesService<ActionCollectionDTO> {}

56
app/server/appsmith-server/src/main/java/com/appsmith/server/actioncollections/defaultresources/ActionCollectionDefaultResourcesServiceCEImpl.java Normal file
View File

@ -0,0 +1,56 @@
package com.appsmith.server.actioncollections.defaultresources;
import com.appsmith.external.models.DefaultResources;
import com.appsmith.server.defaultresources.DefaultResourcesServiceCE;
import com.appsmith.server.domains.ActionCollection;
import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils;
@Service
public class ActionCollectionDefaultResourcesServiceCEImpl implements DefaultResourcesServiceCE<ActionCollection> {
@Override
public ActionCollection initialize(ActionCollection domainObject, String branchName, boolean resetExistingValues) {
DefaultResources existingDefaultResources = domainObject.getDefaultResources();
DefaultResources defaultResources = new DefaultResources();
String defaultApplicationId = domainObject.getApplicationId();
String defaultCollectionId = domainObject.getId();
if (existingDefaultResources != null && !resetExistingValues) {
// Check if there are properties to be copied over from existing
if (StringUtils.hasText(existingDefaultResources.getApplicationId())) {
defaultApplicationId = existingDefaultResources.getApplicationId();
}
if (StringUtils.hasText(existingDefaultResources.getCollectionId())) {
defaultCollectionId = existingDefaultResources.getCollectionId();
}
}
defaultResources.setCollectionId(defaultCollectionId);
defaultResources.setApplicationId(defaultApplicationId);
defaultResources.setBranchName(branchName);
domainObject.setDefaultResources(defaultResources);
return domainObject;
}
@Override
public ActionCollection setFromOtherBranch(
ActionCollection domainObject, ActionCollection branchedDomainObject, String branchName) {
DefaultResources defaultResources = domainObject.getDefaultResources();
if (defaultResources == null) {
defaultResources = new DefaultResources();
}
DefaultResources otherDefaultResources = branchedDomainObject.getDefaultResources();
defaultResources.setCollectionId(otherDefaultResources.getCollectionId());
defaultResources.setApplicationId(otherDefaultResources.getApplicationId());
defaultResources.setBranchName(branchName);
domainObject.setDefaultResources(defaultResources);
return domainObject;
}
}

9
app/server/appsmith-server/src/main/java/com/appsmith/server/actioncollections/defaultresources/ActionCollectionDefaultResourcesServiceImpl.java Normal file
View File

@ -0,0 +1,9 @@
package com.appsmith.server.actioncollections.defaultresources;
import com.appsmith.server.defaultresources.DefaultResourcesService;
import com.appsmith.server.domains.ActionCollection;
import org.springframework.stereotype.Service;
@Service
public class ActionCollectionDefaultResourcesServiceImpl extends ActionCollectionDefaultResourcesServiceCEImpl
implements DefaultResourcesService<ActionCollection> {}

131
app/server/appsmith-server/src/main/java/com/appsmith/server/actioncollections/imports/ActionCollectionImportableServiceCEImpl.java
View File

@ -4,6 +4,7 @@ import com.appsmith.external.models.DefaultResources;
import com.appsmith.external.models.Policy;
import com.appsmith.server.actioncollections.base.ActionCollectionService;
import com.appsmith.server.constants.FieldName;
import com.appsmith.server.defaultresources.DefaultResourcesService;
import com.appsmith.server.domains.ActionCollection;
import com.appsmith.server.domains.Application;
import com.appsmith.server.domains.NewPage;
@ -23,6 +24,7 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.bson.types.ObjectId;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import java.util.ArrayList;
@ -39,6 +41,8 @@ import static com.appsmith.external.helpers.AppsmithBeanUtils.copyNestedNonNullP
public class ActionCollectionImportableServiceCEImpl implements ImportableServiceCE<ActionCollection> {
private final ActionCollectionService actionCollectionService;
private final ActionCollectionRepository repository;
private final DefaultResourcesService<ActionCollection> defaultResourcesService;
private final DefaultResourcesService<ActionCollectionDTO> dtoDefaultResourcesService;
// Requires pageNameMap, pageNameToOldNameMap, pluginMap and actionResultDTO to be present in importable resources.
// Updates actionCollectionResultDTO in importable resources.
@ -135,17 +139,16 @@ public class ActionCollectionImportableServiceCEImpl implements ImportableServic
final String workspaceId = importedApplication.getWorkspaceId();
// Map of gitSyncId to actionCollection of the existing records in DB
Mono<Map<String, ActionCollection>> actionCollectionsInCurrentAppMono = repository
.findByApplicationId(importedApplication.getId())
.filter(collection -> collection.getGitSyncId() != null)
.collectMap(ActionCollection::getGitSyncId);
Mono<Map<String, ActionCollection>> actionCollectionsInCurrentAppMono =
getCollectionsInCurrentAppFlux(importedApplication)
.filter(collection -> collection.getGitSyncId() != null)
.collectMap(ActionCollection::getGitSyncId);
Mono<Map<String, ActionCollection>> actionCollectionsInBranchesMono;
if (importedApplication.getGitApplicationMetadata() != null) {
final String defaultApplicationId =
importedApplication.getGitApplicationMetadata().getDefaultApplicationId();
actionCollectionsInBranchesMono = repository
.findByDefaultApplicationId(defaultApplicationId, Optional.empty())
actionCollectionsInBranchesMono = getCollectionsInOtherBranchesFlux(defaultApplicationId)
.filter(actionCollection -> actionCollection.getGitSyncId() != null)
.collectMap(ActionCollection::getGitSyncId);
} else {
@ -215,6 +218,37 @@ public class ActionCollectionImportableServiceCEImpl implements ImportableServic
actionCollection.setWorkspaceId(workspaceId);
actionCollection.setApplicationId(importedApplication.getId());
if (importedApplication.getGitApplicationMetadata() != null) {
final String defaultApplicationId = importedApplication
.getGitApplicationMetadata()
.getDefaultApplicationId();
if (actionsCollectionsInBranches.containsKey(actionCollection.getGitSyncId())) {
ActionCollection branchedActionCollection =
getExistingCollectionForImportedCollection(
mappedImportableResourcesDTO,
actionsCollectionsInBranches,
actionCollection);
defaultResourcesService.setFromOtherBranch(
actionCollection,
branchedActionCollection,
importingMetaDTO.getBranchName());
dtoDefaultResourcesService.setFromOtherBranch(
actionCollection.getUnpublishedCollection(),
branchedActionCollection.getUnpublishedCollection(),
importingMetaDTO.getBranchName());
} else {
defaultResourcesService.initialize(
actionCollection, importingMetaDTO.getBranchName(), false);
actionCollection
.getDefaultResources()
.setApplicationId(defaultApplicationId);
dtoDefaultResourcesService.initialize(
actionCollection.getUnpublishedCollection(),
importingMetaDTO.getBranchName(),
false);
}
}
// Check if the action has gitSyncId and if it's already in DB
if (existingAppContainsCollection(
actionsCollectionsInCurrentApp, actionCollection)) {
@ -226,26 +260,12 @@ public class ActionCollectionImportableServiceCEImpl implements ImportableServic
actionsCollectionsInCurrentApp,
actionCollection);
Set<Policy> existingPolicy = existingActionCollection.getPolicies();
copyNestedNonNullProperties(actionCollection, existingActionCollection);
updateExistingCollection(
importingMetaDTO,
mappedImportableResourcesDTO,
actionCollection,
existingActionCollection);
populateDomainMappedReferences(
mappedImportableResourcesDTO, existingActionCollection);
// Update branchName
existingActionCollection
.getDefaultResources()
.setBranchName(importingMetaDTO.getBranchName());
// Recover the deleted state present in DB from imported actionCollection
existingActionCollection
.getUnpublishedCollection()
.setDeletedAt(actionCollection
.getUnpublishedCollection()
.getDeletedAt());
existingActionCollection.setDeletedAt(actionCollection.getDeletedAt());
existingActionCollection.setPolicies(existingPolicy);
existingActionCollection.updateForBulkWriteOperation();
existingActionCollections.add(existingActionCollection);
resultDTO.getSavedActionCollectionIds().add(existingActionCollection.getId());
resultDTO
@ -261,29 +281,6 @@ public class ActionCollectionImportableServiceCEImpl implements ImportableServic
parentPage.getId());
}
if (importedApplication.getGitApplicationMetadata() != null) {
final String defaultApplicationId = importedApplication
.getGitApplicationMetadata()
.getDefaultApplicationId();
if (actionsCollectionsInBranches.containsKey(
actionCollection.getGitSyncId())) {
ActionCollection branchedActionCollection =
getExistingCollectionForImportedCollection(
mappedImportableResourcesDTO,
actionsCollectionsInBranches,
actionCollection);
actionCollectionService.populateDefaultResources(
actionCollection,
branchedActionCollection,
importingMetaDTO.getBranchName());
} else {
DefaultResources defaultResources = new DefaultResources();
defaultResources.setApplicationId(defaultApplicationId);
defaultResources.setBranchName(importingMetaDTO.getBranchName());
actionCollection.setDefaultResources(defaultResources);
}
}
// this will generate the id and other auto generated fields e.g. createdAt
actionCollection.updateForBulkWriteOperation();
actionCollectionService.generateAndSetPolicies(parentPage, actionCollection);
@ -323,6 +320,44 @@ public class ActionCollectionImportableServiceCEImpl implements ImportableServic
});
}
protected Flux<ActionCollection> getCollectionsInCurrentAppFlux(Application importedApplication) {
return repository.findByApplicationId(importedApplication.getId());
}
protected Flux<ActionCollection> getCollectionsInOtherBranchesFlux(String defaultApplicationId) {
return repository.findByDefaultApplicationId(defaultApplicationId, Optional.empty());
}
private void updateExistingCollection(
ImportingMetaDTO importingMetaDTO,
MappedImportableResourcesDTO mappedImportableResourcesDTO,
ActionCollection actionCollection,
ActionCollection existingActionCollection) {
Set<Policy> existingPolicy = existingActionCollection.getPolicies();
updateImportableCollectionFromExistingCollection(existingActionCollection, actionCollection);
copyNestedNonNullProperties(actionCollection, existingActionCollection);
populateDomainMappedReferences(mappedImportableResourcesDTO, existingActionCollection);
// Update branchName
existingActionCollection.getDefaultResources().setBranchName(importingMetaDTO.getBranchName());
// Recover the deleted state present in DB from imported actionCollection
existingActionCollection
.getUnpublishedCollection()
.setDeletedAt(actionCollection.getUnpublishedCollection().getDeletedAt());
existingActionCollection.setDeletedAt(actionCollection.getDeletedAt());
existingActionCollection.setPolicies(existingPolicy);
existingActionCollection.updateForBulkWriteOperation();
}
protected void updateImportableCollectionFromExistingCollection(
ActionCollection existingActionCollection, ActionCollection actionCollection) {
// Nothing to update from the existing action collection
}
protected ActionCollection getExistingCollectionForImportedCollection(
MappedImportableResourcesDTO mappedImportableResourcesDTO,
Map<String, ActionCollection> actionsCollectionsInCurrentApp,

9
app/server/appsmith-server/src/main/java/com/appsmith/server/actioncollections/imports/ActionCollectionImportableServiceImpl.java
View File

@ -1,7 +1,9 @@
package com.appsmith.server.actioncollections.imports;
import com.appsmith.server.actioncollections.base.ActionCollectionService;
import com.appsmith.server.defaultresources.DefaultResourcesService;
import com.appsmith.server.domains.ActionCollection;
import com.appsmith.server.dtos.ActionCollectionDTO;
import com.appsmith.server.imports.importable.ImportableService;
import com.appsmith.server.repositories.ActionCollectionRepository;
import org.springframework.stereotype.Service;
@ -10,7 +12,10 @@ import org.springframework.stereotype.Service;
public class ActionCollectionImportableServiceImpl extends ActionCollectionImportableServiceCEImpl
implements ImportableService<ActionCollection> {
public ActionCollectionImportableServiceImpl(
ActionCollectionService actionCollectionService, ActionCollectionRepository repository) {
super(actionCollectionService, repository);
ActionCollectionService actionCollectionService,
ActionCollectionRepository repository,
DefaultResourcesService<ActionCollection> defaultResourcesService,
DefaultResourcesService<ActionCollectionDTO> dtoDefaultResourcesService) {
super(actionCollectionService, repository, defaultResourcesService, dtoDefaultResourcesService);
}
}

3
app/server/appsmith-server/src/main/java/com/appsmith/server/defaultresources/DefaultResourcesService.java Normal file
View File

@ -0,0 +1,3 @@
package com.appsmith.server.defaultresources;
public interface DefaultResourcesService<T> extends DefaultResourcesServiceCE<T> {}

8
app/server/appsmith-server/src/main/java/com/appsmith/server/defaultresources/DefaultResourcesServiceCE.java Normal file
View File

@ -0,0 +1,8 @@
package com.appsmith.server.defaultresources;
public interface DefaultResourcesServiceCE<T> {
T initialize(T domainObject, String branchName, boolean resetExistingValues);
T setFromOtherBranch(T domainObject, T defaultDomainObject, String branchName);
}

20
app/server/appsmith-server/src/main/java/com/appsmith/server/dtos/ce/ActionCollectionCE_DTO.java
View File

@ -139,14 +139,32 @@ public class ActionCollectionCE_DTO {
this.setApplicationId(actionCollection.getApplicationId());
this.setWorkspaceId(actionCollection.getWorkspaceId());
this.setUserPermissions(actionCollection.userPermissions);
copyNewFieldValuesIntoOldObject(actionCollection.getDefaultResources(), this.getDefaultResources());
if (this.getDefaultResources() == null) {
this.setDefaultResources(actionCollection.getDefaultResources());
} else {
copyNewFieldValuesIntoOldObject(actionCollection.getDefaultResources(), this.getDefaultResources());
}
}
public void sanitiseForExport() {
this.resetTransientFields();
this.setDefaultResources(null);
this.setDefaultToBranchedActionIdsMap(null);
this.setDefaultToBranchedArchivedActionIdsMap(null);
this.setActionIds(null);
this.setArchivedActionIds(null);
}
public String getUserExecutableName() {
return this.getName();
}
protected void resetTransientFields() {
this.setId(null);
this.setWorkspaceId(null);
this.setApplicationId(null);
this.setErrorReports(null);
this.setActions(List.of());
this.setArchivedActions(List.of());
}
}

649
app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/GitFileUtils.java
View File

@ -1,659 +1,28 @@
package com.appsmith.server.helpers;
import com.appsmith.external.constants.AnalyticsEvents;
import com.appsmith.external.git.FileInterface;
import com.appsmith.external.helpers.Stopwatch;
import com.appsmith.external.models.ActionDTO;
import com.appsmith.external.models.ApplicationGitReference;
import com.appsmith.external.models.BaseDomain;
import com.appsmith.external.models.DatasourceStorage;
import com.appsmith.external.models.PluginType;
import com.appsmith.git.helpers.FileUtilsImpl;
import com.appsmith.server.constants.FieldName;
import com.appsmith.server.domains.ActionCollection;
import com.appsmith.server.domains.Application;
import com.appsmith.server.domains.ApplicationPage;
import com.appsmith.server.domains.CustomJSLib;
import com.appsmith.server.domains.NewAction;
import com.appsmith.server.domains.NewPage;
import com.appsmith.server.domains.Theme;
import com.appsmith.server.dtos.ActionCollectionDTO;
import com.appsmith.server.dtos.ApplicationJson;
import com.appsmith.server.dtos.PageDTO;
import com.appsmith.server.exceptions.AppsmithError;
import com.appsmith.server.exceptions.AppsmithException;
import com.appsmith.server.helpers.ce.GitFileUtilsCE;
import com.appsmith.server.services.AnalyticsService;
import com.appsmith.server.services.SessionUserService;
import com.google.gson.Gson;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import net.minidev.json.JSONObject;
import net.minidev.json.parser.JSONParser;
import net.minidev.json.parser.ParseException;
import org.apache.commons.collections.PredicateUtils;
import org.apache.commons.lang3.StringUtils;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Import;
import org.springframework.stereotype.Component;
import reactor.core.Exceptions;
import reactor.core.publisher.Mono;
import java.io.IOException;
import java.lang.reflect.Field;
import java.lang.reflect.Type;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import static com.appsmith.external.constants.GitConstants.NAME_SEPARATOR;
import static com.appsmith.external.helpers.AppsmithBeanUtils.copyNestedNonNullProperties;
import static com.appsmith.external.helpers.AppsmithBeanUtils.copyProperties;
import static com.appsmith.server.constants.FieldName.ACTION_COLLECTION_LIST;
import static com.appsmith.server.constants.FieldName.ACTION_LIST;
import static com.appsmith.server.constants.FieldName.CUSTOM_JS_LIB_LIST;
import static com.appsmith.server.constants.FieldName.DATASOURCE_LIST;
import static com.appsmith.server.constants.FieldName.DECRYPTED_FIELDS;
import static com.appsmith.server.constants.FieldName.EDIT_MODE_THEME;
import static com.appsmith.server.constants.FieldName.EXPORTED_APPLICATION;
import static com.appsmith.server.constants.FieldName.PAGE_LIST;
@Slf4j
@RequiredArgsConstructor
@Component
@Import({FileUtilsImpl.class})
public class GitFileUtils {
private final FileInterface fileUtils;
private final AnalyticsService analyticsService;
private final SessionUserService sessionUserService;
public class GitFileUtils extends GitFileUtilsCE {
private final Gson gson;
// Number of seconds after lock file is stale
@Value("${appsmith.index.lock.file.time}")
public final int INDEX_LOCK_FILE_STALE_TIME = 300;
// Only include the application helper fields in metadata object
private static final Set<String> blockedMetadataFields = Set.of(
EXPORTED_APPLICATION,
DATASOURCE_LIST,
PAGE_LIST,
ACTION_LIST,
ACTION_COLLECTION_LIST,
DECRYPTED_FIELDS,
EDIT_MODE_THEME,
CUSTOM_JS_LIB_LIST);
/**
* This method will save the complete application in the local repo directory.
* Path to repo will be : ./container-volumes/git-repo/workspaceId/defaultApplicationId/repoName/{application_data}
*
* @param baseRepoSuffix path suffix used to create a local repo path
* @param applicationJson application reference object from which entire application can be rehydrated
* @param branchName name of the branch for the current application
* @return repo path where the application is stored
*/
public Mono<Path> saveApplicationToLocalRepo(
Path baseRepoSuffix, ApplicationJson applicationJson, String branchName)
throws IOException, GitAPIException {
/*
1. Checkout to branch
2. Create application reference for appsmith-git module
3. Save application to git repo
*/
ApplicationGitReference applicationReference = createApplicationReference(applicationJson);
// Save application to git repo
try {
return fileUtils.saveApplicationToGitRepo(baseRepoSuffix, applicationReference, branchName);
} catch (IOException | GitAPIException e) {
log.error("Error occurred while saving files to local git repo: ", e);
throw Exceptions.propagate(e);
}
}
public Mono<Path> saveApplicationToLocalRepoWithAnalytics(
Path baseRepoSuffix, ApplicationJson applicationJson, String branchName)
throws IOException, GitAPIException {
/*
1. Checkout to branch
2. Create application reference for appsmith-git module
3. Save application to git repo
*/
Stopwatch stopwatch = new Stopwatch(AnalyticsEvents.GIT_SERIALIZE_APP_RESOURCES_TO_LOCAL_FILE.getEventName());
// Save application to git repo
try {
Mono<Path> repoPathMono = saveApplicationToLocalRepo(baseRepoSuffix, applicationJson, branchName);
return Mono.zip(repoPathMono, sessionUserService.getCurrentUser()).flatMap(tuple -> {
stopwatch.stopTimer();
Path repoPath = tuple.getT1();
// Path to repo will be : ./container-volumes/git-repo/workspaceId/defaultApplicationId/repoName/
final Map<String, Object> data = Map.of(
FieldName.APPLICATION_ID,
repoPath.getParent().getFileName().toString(),
FieldName.ORGANIZATION_ID,
repoPath.getParent().getParent().getFileName().toString(),
FieldName.FLOW_NAME,
stopwatch.getFlow(),
"executionTime",
stopwatch.getExecutionTime());
return analyticsService
.sendEvent(
AnalyticsEvents.UNIT_EXECUTION_TIME.getEventName(),
tuple.getT2().getUsername(),
data)
.thenReturn(repoPath);
});
} catch (IOException | GitAPIException e) {
log.error("Error occurred while saving files to local git repo: ", e);
throw Exceptions.propagate(e);
}
}
public Mono<Path> saveApplicationToLocalRepo(
String workspaceId,
String defaultApplicationId,
String repoName,
ApplicationJson applicationJson,
String branchName)
throws GitAPIException, IOException {
Path baseRepoSuffix = Paths.get(workspaceId, defaultApplicationId, repoName);
return saveApplicationToLocalRepo(baseRepoSuffix, applicationJson, branchName);
}
/**
* Method to convert application resources to the structure which can be serialised by appsmith-git module for
* serialisation
*
* @param applicationJson application resource including actions, jsobjects, pages
* @return resource which can be saved to file system
*/
public ApplicationGitReference createApplicationReference(ApplicationJson applicationJson) {
ApplicationGitReference applicationReference = new ApplicationGitReference();
Application application = applicationJson.getExportedApplication();
removeUnwantedFieldsFromApplication(application);
// Pass application reference
applicationReference.setApplication(applicationJson.getExportedApplication());
// No need to commit publish mode theme as it leads to conflict resolution at both the places if any
applicationJson.setPublishedTheme(null);
// Pass metadata
Iterable<String> keys = getAllFields(applicationJson)
.map(Field::getName)
.filter(name -> !blockedMetadataFields.contains(name))
.collect(Collectors.toList());
ApplicationJson applicationMetadata = new ApplicationJson();
Map<String, Set<String>> updatedResources = applicationJson.getUpdatedResources();
applicationJson.setUpdatedResources(null);
copyProperties(applicationJson, applicationMetadata, keys);
applicationReference.setMetadata(applicationMetadata);
// Remove internal fields from the themes
removeUnwantedFieldsFromBaseDomain(applicationJson.getEditModeTheme());
applicationReference.setTheme(applicationJson.getEditModeTheme());
// Insert only active pages which will then be committed to repo as individual file
Map<String, Object> resourceMap = new HashMap<>();
Map<String, String> resourceMapBody = new HashMap<>();
Map<String, String> dslBody = new HashMap<>();
applicationJson.getPageList().stream()
// As we are expecting the commit will happen only after the application is published, so we can safely
// assume if the unpublished version is deleted entity should not be committed to git
.filter(newPage -> newPage.getUnpublishedPage() != null
&& newPage.getUnpublishedPage().getDeletedAt() == null)
.forEach(newPage -> {
String pageName = newPage.getUnpublishedPage() != null
? newPage.getUnpublishedPage().getName()
: newPage.getPublishedPage().getName();
removeUnwantedFieldsFromPage(newPage);
JSONObject dsl =
newPage.getUnpublishedPage().getLayouts().get(0).getDsl();
// Get MainContainer widget data, remove the children and club with Canvas.json file
JSONObject mainContainer = new JSONObject(dsl);
mainContainer.remove("children");
newPage.getUnpublishedPage().getLayouts().get(0).setDsl(mainContainer);
// pageName will be used for naming the json file
dslBody.put(pageName, dsl.toString());
resourceMap.put(pageName, newPage);
});
applicationReference.setPages(new HashMap<>(resourceMap));
applicationReference.setPageDsl(new HashMap<>(dslBody));
resourceMap.clear();
resourceMapBody.clear();
// Insert active actions and also assign the keys which later will be used for saving the resource in actual
// filepath
// For actions, we are referring to validNames to maintain unique file names as just name
// field don't guarantee unique constraint for actions within JSObject
// queryValidName_pageName => nomenclature for the keys
applicationJson.getActionList().stream()
// As we are expecting the commit will happen only after the application is published, so we can safely
// assume if the unpublished version is deleted entity should not be committed to git
.filter(newAction -> newAction.getUnpublishedAction() != null
&& newAction.getUnpublishedAction().getDeletedAt() == null)
.forEach(newAction -> {
String prefix = newAction.getUnpublishedAction() != null
? newAction.getUnpublishedAction().getValidName()
+ NAME_SEPARATOR
+ newAction.getUnpublishedAction().getPageId()
: newAction.getPublishedAction().getValidName()
+ NAME_SEPARATOR
+ newAction.getPublishedAction().getPageId();
removeUnwantedFieldFromAction(newAction);
String body = newAction
.getUnpublishedAction()
.getActionConfiguration()
.getBody()
!= null
? newAction
.getUnpublishedAction()
.getActionConfiguration()
.getBody()
: "";
// This is a special case where we are handling REMOTE type plugins based actions such as Twilio
// The user configured values are stored in a attribute called formData which is a map unlike the
// body
if (newAction.getPluginType().toString().equals("REMOTE")
&& newAction
.getUnpublishedAction()
.getActionConfiguration()
.getFormData()
!= null) {
body = new Gson()
.toJson(
newAction
.getUnpublishedAction()
.getActionConfiguration()
.getFormData(),
Map.class);
newAction
.getUnpublishedAction()
.getActionConfiguration()
.setFormData(null);
}
// This is a special case where we are handling JS actions as we don't want to commit the body of JS
// actions
if (newAction.getPluginType().equals(PluginType.JS)) {
newAction
.getUnpublishedAction()
.getActionConfiguration()
.setBody(null);
newAction.getUnpublishedAction().setJsonPathKeys(null);
} else {
// For the regular actions we save the body field to git repo
resourceMapBody.put(prefix, body);
}
resourceMap.put(prefix, newAction);
});
applicationReference.setActions(new HashMap<>(resourceMap));
applicationReference.setActionBody(new HashMap<>(resourceMapBody));
resourceMap.clear();
resourceMapBody.clear();
// Insert JSOObjects and also assign the keys which later will be used for saving the resource in actual
// filepath
// JSObjectName_pageName => nomenclature for the keys
Map<String, String> resourceMapActionCollectionBody = new HashMap<>();
applicationJson.getActionCollectionList().stream()
// As we are expecting the commit will happen only after the application is published, so we can safely
// assume if the unpublished version is deleted entity should not be committed to git
.filter(collection -> collection.getUnpublishedCollection() != null
&& collection.getUnpublishedCollection().getDeletedAt() == null)
.forEach(actionCollection -> {
String prefix = actionCollection.getUnpublishedCollection() != null
? actionCollection.getUnpublishedCollection().getName()
+ NAME_SEPARATOR
+ actionCollection
.getUnpublishedCollection()
.getPageId()
: actionCollection.getPublishedCollection().getName()
+ NAME_SEPARATOR
+ actionCollection.getPublishedCollection().getPageId();
removeUnwantedFieldFromActionCollection(actionCollection);
String body = actionCollection.getUnpublishedCollection().getBody() != null
? actionCollection.getUnpublishedCollection().getBody()
: "";
actionCollection.getUnpublishedCollection().setBody(null);
resourceMapActionCollectionBody.put(prefix, body);
resourceMap.put(prefix, actionCollection);
});
applicationReference.setActionCollections(new HashMap<>(resourceMap));
applicationReference.setActionCollectionBody(new HashMap<>(resourceMapActionCollectionBody));
applicationReference.setUpdatedResources(updatedResources);
resourceMap.clear();
resourceMapActionCollectionBody.clear();
// Send datasources
applicationJson.getDatasourceList().forEach(datasource -> {
removeUnwantedFieldsFromDatasource(datasource);
resourceMap.put(datasource.getName(), datasource);
});
applicationReference.setDatasources(new HashMap<>(resourceMap));
resourceMap.clear();
applicationJson.getCustomJSLibList().forEach(jsLib -> {
removeUnwantedFieldsFromBaseDomain(jsLib);
resourceMap.put(jsLib.getUidString(), jsLib);
});
applicationReference.setJsLibraries(new HashMap<>(resourceMap));
resourceMap.clear();
return applicationReference;
}
protected Stream<Field> getAllFields(ApplicationJson applicationJson) {
Class<?> currentType = applicationJson.getClass();
Set<Class<?>> classes = new HashSet<>();
while (currentType != null) {
classes.add(currentType);
currentType = currentType.getSuperclass();
}
return classes.stream().flatMap(currentClass -> Arrays.stream(currentClass.getDeclaredFields()));
}
/**
* Method to reconstruct the application from the local git repo
*
* @param workspaceId To which workspace application needs to be rehydrated
* @param defaultApplicationId Root application for the current branched application
* @param branchName for which branch the application needs to rehydrate
* @return application reference from which entire application can be rehydrated
*/
public Mono<ApplicationJson> reconstructApplicationJsonFromGitRepoWithAnalytics(
String workspaceId, String defaultApplicationId, String repoName, String branchName) {
Stopwatch stopwatch = new Stopwatch(AnalyticsEvents.GIT_DESERIALIZE_APP_RESOURCES_FROM_FILE.getEventName());
return Mono.zip(
reconstructApplicationJsonFromGitRepo(workspaceId, defaultApplicationId, repoName, branchName),
sessionUserService.getCurrentUser())
.flatMap(tuple -> {
stopwatch.stopTimer();
final Map<String, Object> data = Map.of(
FieldName.APPLICATION_ID,
defaultApplicationId,
FieldName.ORGANIZATION_ID,
workspaceId,
FieldName.FLOW_NAME,
stopwatch.getFlow(),
"executionTime",
stopwatch.getExecutionTime());
return analyticsService
.sendEvent(
AnalyticsEvents.UNIT_EXECUTION_TIME.getEventName(),
tuple.getT2().getUsername(),
data)
.thenReturn(tuple.getT1());
});
}
public Mono<ApplicationJson> reconstructApplicationJsonFromGitRepo(
String workspaceId, String defaultApplicationId, String repoName, String branchName) {
Mono<ApplicationGitReference> appReferenceMono = fileUtils.reconstructApplicationReferenceFromGitRepo(
workspaceId, defaultApplicationId, repoName, branchName);
return appReferenceMono.map(applicationReference -> {
// Extract application metadata from the json
ApplicationJson metadata =
getApplicationResource(applicationReference.getMetadata(), ApplicationJson.class);
ApplicationJson applicationJson = getApplicationJsonFromGitReference(applicationReference);
copyNestedNonNullProperties(metadata, applicationJson);
return applicationJson;
});
}
private <T> List<T> getApplicationResource(Map<String, Object> resources, Type type) {
List<T> deserializedResources = new ArrayList<>();
if (!CollectionUtils.isNullOrEmpty(resources)) {
for (Map.Entry<String, Object> resource : resources.entrySet()) {
deserializedResources.add(getApplicationResource(resource.getValue(), type));
}
}
return deserializedResources;
}
public <T> T getApplicationResource(Object resource, Type type) {
if (resource == null) {
return null;
}
return gson.fromJson(gson.toJson(resource), type);
}
/**
* Once the user connects the existing application to a remote repo, we will initialize the repo with Readme.md -
* Url to the deployed app(view and edit mode)
* Link to discord channel for support
* Link to appsmith documentation for Git related operations
* Welcome message
*
* @param baseRepoSuffix path suffix used to create a branch repo path as per worktree implementation
* @param viewModeUrl URL to deployed version of the application view only mode
* @param editModeUrl URL to deployed version of the application edit mode
* @return Path where the Application is stored
*/
public Mono<Path> initializeReadme(Path baseRepoSuffix, String viewModeUrl, String editModeUrl) throws IOException {
return fileUtils
.initializeReadme(baseRepoSuffix, viewModeUrl, editModeUrl)
.onErrorResume(e -> Mono.error(new AppsmithException(AppsmithError.GIT_FILE_SYSTEM_ERROR, e)));
}
/**
* When the user clicks on detach remote, we need to remove the repo from the file system
*
* @param baseRepoSuffix path suffix used to create a branch repo path as per worktree implementation
* @return success on remove of file system
*/
public Mono<Boolean> deleteLocalRepo(Path baseRepoSuffix) {
return fileUtils.deleteLocalRepo(baseRepoSuffix);
}
public Mono<Boolean> checkIfDirectoryIsEmpty(Path baseRepoSuffix) throws IOException {
return fileUtils
.checkIfDirectoryIsEmpty(baseRepoSuffix)
.onErrorResume(e -> Mono.error(new AppsmithException(AppsmithError.GIT_FILE_SYSTEM_ERROR, e)));
}
private void removeUnwantedFieldsFromBaseDomain(BaseDomain baseDomain) {
baseDomain.setPolicies(null);
baseDomain.setUserPermissions(null);
}
private void removeUnwantedFieldsFromDatasource(DatasourceStorage datasource) {
datasource.setInvalids(null);
removeUnwantedFieldsFromBaseDomain(datasource);
}
private void removeUnwantedFieldsFromPage(NewPage page) {
// As we are publishing the app and then committing to git we expect the published and unpublished PageDTO will
// be same, so we only commit unpublished PageDTO.
page.setPublishedPage(null);
removeUnwantedFieldsFromBaseDomain(page);
}
private void removeUnwantedFieldsFromApplication(Application application) {
// Don't commit application name as while importing we are using the repoName as application name
application.setName(null);
application.setPublishedPages(null);
application.setIsPublic(null);
application.setSlug(null);
application.setPublishedApplicationDetail(null);
removeUnwantedFieldsFromBaseDomain(application);
// we can call the sanitiseToExportDBObject() from BaseDomain as well here
}
private void removeUnwantedFieldFromAction(NewAction action) {
// As we are publishing the app and then committing to git we expect the published and unpublished ActionDTO
// will be same, so we only commit unpublished ActionDTO.
action.setPublishedAction(null);
removeUnwantedFieldsFromBaseDomain(action);
}
private void removeUnwantedFieldFromActionCollection(ActionCollection actionCollection) {
// As we are publishing the app and then committing to git we expect the published and unpublished
// ActionCollectionDTO will be same, so we only commit unpublished ActionCollectionDTO.
actionCollection.setPublishedCollection(null);
removeUnwantedFieldsFromBaseDomain(actionCollection);
}
private ApplicationJson getApplicationJsonFromGitReference(ApplicationGitReference applicationReference) {
ApplicationJson applicationJson = new ApplicationJson();
// Extract application data from the json
Application application = getApplicationResource(applicationReference.getApplication(), Application.class);
applicationJson.setExportedApplication(application);
applicationJson.setEditModeTheme(getApplicationResource(applicationReference.getTheme(), Theme.class));
// Clone the edit mode theme to published theme as both should be same for git connected application because we
// do deploy and push as a single operation
applicationJson.setPublishedTheme(applicationJson.getEditModeTheme());
if (application != null && !CollectionUtils.isNullOrEmpty(application.getPages())) {
// Remove null values
org.apache.commons.collections.CollectionUtils.filter(
application.getPages(), PredicateUtils.notNullPredicate());
// Create a deep clone of application pages to update independently
application.setViewMode(false);
final List<ApplicationPage> applicationPages =
new ArrayList<>(application.getPages().size());
application
.getPages()
.forEach(applicationPage ->
applicationPages.add(gson.fromJson(gson.toJson(applicationPage), ApplicationPage.class)));
application.setPublishedPages(applicationPages);
}
List<CustomJSLib> customJSLibList =
getApplicationResource(applicationReference.getJsLibraries(), CustomJSLib.class);
// remove the duplicate js libraries if there is any
List<CustomJSLib> customJSLibListWithoutDuplicates = new ArrayList<>(new HashSet<>(customJSLibList));
applicationJson.setCustomJSLibList(customJSLibListWithoutDuplicates);
// Extract pages
List<NewPage> pages = getApplicationResource(applicationReference.getPages(), NewPage.class);
// Remove null values
org.apache.commons.collections.CollectionUtils.filter(pages, PredicateUtils.notNullPredicate());
// Set the DSL to page object before saving
Map<String, String> pageDsl = applicationReference.getPageDsl();
pages.forEach(page -> {
JSONParser jsonParser = new JSONParser();
try {
if (pageDsl != null && pageDsl.get(page.getUnpublishedPage().getName()) != null) {
page.getUnpublishedPage().getLayouts().get(0).setDsl((JSONObject) jsonParser.parse(
pageDsl.get(page.getUnpublishedPage().getName())));
}
} catch (ParseException e) {
log.error(
"Error parsing the page dsl for page: {}",
page.getUnpublishedPage().getName(),
e);
throw new AppsmithException(
AppsmithError.JSON_PROCESSING_ERROR,
page.getUnpublishedPage().getName());
}
});
pages.forEach(newPage -> {
// As we are publishing the app and then committing to git we expect the published and unpublished PageDTO
// will be same, so we create a deep copy for the published version for page from the unpublishedPageDTO
newPage.setPublishedPage(gson.fromJson(gson.toJson(newPage.getUnpublishedPage()), PageDTO.class));
});
applicationJson.setPageList(pages);
// Extract actions
if (CollectionUtils.isNullOrEmpty(applicationReference.getActions())) {
applicationJson.setActionList(new ArrayList<>());
} else {
Map<String, String> actionBody = applicationReference.getActionBody();
List<NewAction> actions = getApplicationResource(applicationReference.getActions(), NewAction.class);
// Remove null values if present
org.apache.commons.collections.CollectionUtils.filter(actions, PredicateUtils.notNullPredicate());
actions.forEach(newAction -> {
// With the file version v4 we have split the actions and metadata separately into two files
// So we need to set the body to the unpublished action
String keyName = newAction.getUnpublishedAction().getName()
+ newAction.getUnpublishedAction().getPageId();
if (actionBody != null
&& (actionBody.containsKey(keyName))
&& !StringUtils.isEmpty(actionBody.get(keyName))) {
// For REMOTE plugin like Twilio the user actions are stored in key value pairs and hence they need
// to be
// deserialized separately unlike the body which is stored as string in the db.
if (newAction.getPluginType().toString().equals("REMOTE")) {
Map<String, Object> formData = gson.fromJson(actionBody.get(keyName), Map.class);
newAction
.getUnpublishedAction()
.getActionConfiguration()
.setFormData(formData);
} else {
newAction
.getUnpublishedAction()
.getActionConfiguration()
.setBody(actionBody.get(keyName));
}
}
// As we are publishing the app and then committing to git we expect the published and unpublished
// actionDTO will be same, so we create a deep copy for the published version for action from
// unpublishedActionDTO
newAction.setPublishedAction(
gson.fromJson(gson.toJson(newAction.getUnpublishedAction()), ActionDTO.class));
});
applicationJson.setActionList(actions);
}
// Extract actionCollection
if (CollectionUtils.isNullOrEmpty(applicationReference.getActionCollections())) {
applicationJson.setActionCollectionList(new ArrayList<>());
} else {
Map<String, String> actionCollectionBody = applicationReference.getActionCollectionBody();
List<ActionCollection> actionCollections =
getApplicationResource(applicationReference.getActionCollections(), ActionCollection.class);
// Remove null values if present
org.apache.commons.collections.CollectionUtils.filter(actionCollections, PredicateUtils.notNullPredicate());
actionCollections.forEach(actionCollection -> {
// Set the js object body to the unpublished collection
// Since file version v3 we are splitting the js object code and metadata separately
String keyName = actionCollection.getUnpublishedCollection().getName()
+ actionCollection.getUnpublishedCollection().getPageId();
if (actionCollectionBody != null && actionCollectionBody.containsKey(keyName)) {
actionCollection.getUnpublishedCollection().setBody(actionCollectionBody.get(keyName));
}
// As we are publishing the app and then committing to git we expect the published and unpublished
// actionCollectionDTO will be same, so we create a deep copy for the published version for
// actionCollection from unpublishedActionCollectionDTO
actionCollection.setPublishedCollection(gson.fromJson(
gson.toJson(actionCollection.getUnpublishedCollection()), ActionCollectionDTO.class));
});
applicationJson.setActionCollectionList(actionCollections);
}
// Extract datasources
applicationJson.setDatasourceList(
getApplicationResource(applicationReference.getDatasources(), DatasourceStorage.class));
return applicationJson;
}
public Mono<Long> deleteIndexLockFile(Path path) {
return fileUtils.deleteIndexLockFile(path, INDEX_LOCK_FILE_STALE_TIME);
public GitFileUtils(
FileInterface fileUtils,
AnalyticsService analyticsService,
SessionUserService sessionUserService,
Gson gson) {
super(fileUtils, analyticsService, sessionUserService, gson);
this.gson = gson;
}
}

340
app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/ResponseUtils.java
View File

@ -1,345 +1,9 @@
package com.appsmith.server.helpers;
import com.appsmith.external.models.ActionDTO;
import com.appsmith.external.models.DefaultResources;
import com.appsmith.server.domains.ActionCollection;
import com.appsmith.server.domains.Application;
import com.appsmith.server.domains.Layout;
import com.appsmith.server.domains.NewAction;
import com.appsmith.server.domains.NewPage;
import com.appsmith.server.dtos.ActionCollectionDTO;
import com.appsmith.server.dtos.ActionCollectionViewDTO;
import com.appsmith.server.dtos.ActionViewDTO;
import com.appsmith.server.dtos.ApplicationPagesDTO;
import com.appsmith.server.dtos.LayoutDTO;
import com.appsmith.server.dtos.PageDTO;
import com.appsmith.server.dtos.PageNameIdDTO;
import com.appsmith.server.exceptions.AppsmithError;
import com.appsmith.server.exceptions.AppsmithException;
import com.appsmith.server.migrations.JsonSchemaVersions;
import com.appsmith.server.helpers.ce.ResponseUtilsCE;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang.StringUtils;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.List;
@Slf4j
@Component
public class ResponseUtils {
public PageDTO updatePageDTOWithDefaultResources(PageDTO page) {
DefaultResources defaultResourceIds = page.getDefaultResources();
if (defaultResourceIds == null
|| StringUtils.isEmpty(defaultResourceIds.getApplicationId())
|| StringUtils.isEmpty(defaultResourceIds.getPageId())) {
if (defaultResourceIds == null) {
return page;
}
if (StringUtils.isEmpty(defaultResourceIds.getApplicationId())) {
defaultResourceIds.setApplicationId(page.getApplicationId());
}
if (StringUtils.isEmpty(defaultResourceIds.getPageId())) {
defaultResourceIds.setPageId(page.getId());
}
}
page.setApplicationId(defaultResourceIds.getApplicationId());
page.setId(defaultResourceIds.getPageId());
page.getLayouts().stream()
.filter(layout -> !CollectionUtils.isEmpty(layout.getLayoutOnLoadActions()))
.forEach(layout -> this.updateLayoutWithDefaultResources(layout));
return page;
}
public NewPage updateNewPageWithDefaultResources(NewPage newPage) {
DefaultResources defaultResourceIds = newPage.getDefaultResources();
if (defaultResourceIds == null
|| StringUtils.isEmpty(defaultResourceIds.getApplicationId())
|| StringUtils.isEmpty(defaultResourceIds.getPageId())) {
log.error(
"Unable to find default ids for page: {}",
newPage.getId(),
new AppsmithException(AppsmithError.DEFAULT_RESOURCES_UNAVAILABLE, "page", newPage.getId()));
if (defaultResourceIds == null) {
return newPage;
}
if (StringUtils.isEmpty(defaultResourceIds.getApplicationId())) {
defaultResourceIds.setApplicationId(newPage.getApplicationId());
}
if (StringUtils.isEmpty(defaultResourceIds.getPageId())) {
defaultResourceIds.setPageId(newPage.getId());
}
}
newPage.setId(defaultResourceIds.getPageId());
newPage.setApplicationId(defaultResourceIds.getApplicationId());
if (newPage.getUnpublishedPage() != null) {
newPage.setUnpublishedPage(this.updatePageDTOWithDefaultResources(newPage.getUnpublishedPage()));
}
if (newPage.getPublishedPage() != null) {
newPage.setPublishedPage(this.updatePageDTOWithDefaultResources(newPage.getPublishedPage()));
}
return newPage;
}
public ApplicationPagesDTO updateApplicationPagesDTOWithDefaultResources(ApplicationPagesDTO applicationPages) {
List<PageNameIdDTO> pageNameIdList = applicationPages.getPages();
for (PageNameIdDTO page : pageNameIdList) {
if (StringUtils.isEmpty(page.getDefaultPageId())) {
log.error(
"Unable to find default pageId for applicationPage: {}",
page.getId(),
new AppsmithException(
AppsmithError.DEFAULT_RESOURCES_UNAVAILABLE, "applicationPage", page.getId()));
continue;
}
page.setId(page.getDefaultPageId());
}
// need to update the application also if it's present
if (applicationPages.getApplication() != null) {
applicationPages.setApplication(updateApplicationWithDefaultResources(applicationPages.getApplication()));
}
return applicationPages;
}
public ActionDTO updateActionDTOWithDefaultResources(ActionDTO action) {
DefaultResources defaultResourceIds = action.getDefaultResources();
if (defaultResourceIds == null
|| StringUtils.isEmpty(defaultResourceIds.getApplicationId())
|| StringUtils.isEmpty(defaultResourceIds.getPageId())
|| StringUtils.isEmpty(defaultResourceIds.getActionId())) {
if (defaultResourceIds == null) {
return action;
}
if (StringUtils.isEmpty(defaultResourceIds.getApplicationId())) {
defaultResourceIds.setApplicationId(action.getApplicationId());
}
if (StringUtils.isEmpty(defaultResourceIds.getPageId())) {
defaultResourceIds.setPageId(action.getPageId());
}
if (StringUtils.isEmpty(defaultResourceIds.getActionId())) {
defaultResourceIds.setActionId(action.getId());
}
}
action.setApplicationId(defaultResourceIds.getApplicationId());
action.setPageId(defaultResourceIds.getPageId());
action.setId(defaultResourceIds.getActionId());
if (!StringUtils.isEmpty(defaultResourceIds.getCollectionId())) {
action.setCollectionId(defaultResourceIds.getCollectionId());
}
return action;
}
public LayoutDTO updateLayoutDTOWithDefaultResources(LayoutDTO layout) {
if (!CollectionUtils.isEmpty(layout.getActionUpdates())) {
layout.getActionUpdates()
.forEach(updateLayoutAction -> updateLayoutAction.setId(updateLayoutAction.getDefaultActionId()));
}
if (!CollectionUtils.isEmpty(layout.getLayoutOnLoadActions())) {
layout.getLayoutOnLoadActions()
.forEach(layoutOnLoadAction -> layoutOnLoadAction.forEach(onLoadAction -> {
if (!StringUtils.isEmpty(onLoadAction.getDefaultActionId())) {
onLoadAction.setId(onLoadAction.getDefaultActionId());
}
if (!StringUtils.isEmpty(onLoadAction.getDefaultCollectionId())) {
onLoadAction.setCollectionId(onLoadAction.getDefaultCollectionId());
}
}));
}
return layout;
}
public Layout updateLayoutWithDefaultResources(Layout layout) {
if (!CollectionUtils.isEmpty(layout.getLayoutOnLoadActions())) {
layout.getLayoutOnLoadActions()
.forEach(layoutOnLoadAction -> layoutOnLoadAction.forEach(onLoadAction -> {
if (!StringUtils.isEmpty(onLoadAction.getDefaultActionId())) {
onLoadAction.setId(onLoadAction.getDefaultActionId());
}
if (!StringUtils.isEmpty(onLoadAction.getDefaultCollectionId())) {
onLoadAction.setCollectionId(onLoadAction.getDefaultCollectionId());
}
}));
}
return layout;
}
public ActionViewDTO updateActionViewDTOWithDefaultResources(ActionViewDTO viewDTO) {
DefaultResources defaultResourceIds = viewDTO.getDefaultResources();
if (defaultResourceIds == null
|| StringUtils.isEmpty(defaultResourceIds.getPageId())
|| StringUtils.isEmpty(defaultResourceIds.getActionId())) {
if (defaultResourceIds == null) {
return viewDTO;
}
if (StringUtils.isEmpty(defaultResourceIds.getPageId())) {
defaultResourceIds.setPageId(viewDTO.getPageId());
}
if (StringUtils.isEmpty(defaultResourceIds.getActionId())) {
defaultResourceIds.setActionId(viewDTO.getId());
}
}
viewDTO.setId(defaultResourceIds.getActionId());
viewDTO.setPageId(defaultResourceIds.getPageId());
return viewDTO;
}
public NewAction updateNewActionWithDefaultResources(NewAction newAction) {
DefaultResources defaultResourceIds = newAction.getDefaultResources();
if (defaultResourceIds == null
|| StringUtils.isEmpty(defaultResourceIds.getApplicationId())
|| StringUtils.isEmpty(defaultResourceIds.getActionId())) {
log.error(
"Unable to find default ids for newAction: {}",
newAction.getId(),
new AppsmithException(AppsmithError.DEFAULT_RESOURCES_UNAVAILABLE, "newAction", newAction.getId()));
if (defaultResourceIds == null) {
return newAction;
}
if (StringUtils.isEmpty(defaultResourceIds.getApplicationId())) {
defaultResourceIds.setApplicationId(newAction.getApplicationId());
}
if (StringUtils.isEmpty(defaultResourceIds.getActionId())) {
defaultResourceIds.setActionId(newAction.getId());
}
}
newAction.setId(defaultResourceIds.getActionId());
newAction.setApplicationId(defaultResourceIds.getApplicationId());
if (newAction.getUnpublishedAction() != null) {
newAction.setUnpublishedAction(this.updateActionDTOWithDefaultResources(newAction.getUnpublishedAction()));
}
if (newAction.getPublishedAction() != null) {
newAction.setPublishedAction(this.updateActionDTOWithDefaultResources(newAction.getPublishedAction()));
}
return newAction;
}
public ActionCollection updateActionCollectionWithDefaultResources(ActionCollection actionCollection) {
DefaultResources defaultResourceIds = actionCollection.getDefaultResources();
if (defaultResourceIds == null
|| StringUtils.isEmpty(defaultResourceIds.getApplicationId())
|| StringUtils.isEmpty(defaultResourceIds.getCollectionId())) {
log.error(
"Unable to find default ids for actionCollection: {}",
actionCollection.getId(),
new AppsmithException(
AppsmithError.DEFAULT_RESOURCES_UNAVAILABLE, "actionCollection", actionCollection.getId()));
if (defaultResourceIds == null) {
return actionCollection;
}
if (StringUtils.isEmpty(defaultResourceIds.getApplicationId())) {
defaultResourceIds.setApplicationId(actionCollection.getApplicationId());
}
if (StringUtils.isEmpty(defaultResourceIds.getCollectionId())) {
defaultResourceIds.setCollectionId(actionCollection.getId());
}
}
actionCollection.setId(defaultResourceIds.getCollectionId());
actionCollection.setApplicationId(defaultResourceIds.getApplicationId());
if (actionCollection.getUnpublishedCollection() != null) {
actionCollection.setUnpublishedCollection(
this.updateCollectionDTOWithDefaultResources(actionCollection.getUnpublishedCollection()));
}
if (actionCollection.getPublishedCollection() != null) {
actionCollection.setPublishedCollection(
this.updateCollectionDTOWithDefaultResources(actionCollection.getPublishedCollection()));
}
return actionCollection;
}
public ActionCollectionDTO updateCollectionDTOWithDefaultResources(ActionCollectionDTO collection) {
DefaultResources defaultResourceIds = collection.getDefaultResources();
if (defaultResourceIds == null
|| StringUtils.isEmpty(defaultResourceIds.getApplicationId())
|| StringUtils.isEmpty(defaultResourceIds.getPageId())
|| StringUtils.isEmpty(defaultResourceIds.getCollectionId())) {
if (defaultResourceIds == null) {
return collection;
}
if (StringUtils.isEmpty(defaultResourceIds.getApplicationId())) {
defaultResourceIds.setApplicationId(collection.getApplicationId());
}
if (StringUtils.isEmpty(defaultResourceIds.getPageId())) {
defaultResourceIds.setPageId(collection.getPageId());
}
if (StringUtils.isEmpty(defaultResourceIds.getCollectionId())) {
defaultResourceIds.setCollectionId(collection.getId());
}
}
collection.setApplicationId(defaultResourceIds.getApplicationId());
collection.setPageId(defaultResourceIds.getPageId());
collection.setId(defaultResourceIds.getCollectionId());
// Update actions within the collection
collection.getActions().forEach(this::updateActionDTOWithDefaultResources);
collection.getArchivedActions().forEach(this::updateActionDTOWithDefaultResources);
return collection;
}
public ActionCollectionViewDTO updateActionCollectionViewDTOWithDefaultResources(ActionCollectionViewDTO viewDTO) {
DefaultResources defaultResourceIds = viewDTO.getDefaultResources();
if (defaultResourceIds == null
|| StringUtils.isEmpty(defaultResourceIds.getPageId())
|| StringUtils.isEmpty(defaultResourceIds.getApplicationId())
|| StringUtils.isEmpty(defaultResourceIds.getCollectionId())) {
if (defaultResourceIds == null) {
return viewDTO;
}
if (StringUtils.isEmpty(defaultResourceIds.getApplicationId())) {
defaultResourceIds.setApplicationId(viewDTO.getApplicationId());
}
if (StringUtils.isEmpty(defaultResourceIds.getPageId())) {
defaultResourceIds.setPageId(viewDTO.getPageId());
}
if (StringUtils.isEmpty(defaultResourceIds.getCollectionId())) {
defaultResourceIds.setCollectionId(viewDTO.getId());
}
}
viewDTO.setId(defaultResourceIds.getCollectionId());
viewDTO.setApplicationId(defaultResourceIds.getApplicationId());
viewDTO.setPageId(defaultResourceIds.getPageId());
viewDTO.getActions().forEach(this::updateActionDTOWithDefaultResources);
return viewDTO;
}
public Application updateApplicationWithDefaultResources(Application application) {
if (application.getGitApplicationMetadata() != null
&& !StringUtils.isEmpty(application.getGitApplicationMetadata().getDefaultApplicationId())) {
application.setId(application.getGitApplicationMetadata().getDefaultApplicationId());
}
if (!CollectionUtils.isEmpty(application.getPages())) {
application.getPages().forEach(page -> {
if (!StringUtils.isEmpty(page.getDefaultPageId())) {
page.setId(page.getDefaultPageId());
}
});
}
if (!CollectionUtils.isEmpty(application.getPublishedPages())) {
application.getPublishedPages().forEach(page -> {
if (!StringUtils.isEmpty(page.getDefaultPageId())) {
page.setId(page.getDefaultPageId());
}
});
}
if (application.getClientSchemaVersion() == null
|| application.getServerSchemaVersion() == null
|| (JsonSchemaVersions.clientVersion.equals(application.getClientSchemaVersion())
&& JsonSchemaVersions.serverVersion.equals(application.getServerSchemaVersion()))) {
application.setIsAutoUpdate(false);
} else {
application.setIsAutoUpdate(true);
}
return application;
}
}
public class ResponseUtils extends ResponseUtilsCE {}

737
app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/ce/GitFileUtilsCE.java Normal file
View File

@ -0,0 +1,737 @@
package com.appsmith.server.helpers.ce;
import com.appsmith.external.constants.AnalyticsEvents;
import com.appsmith.external.git.FileInterface;
import com.appsmith.external.helpers.Stopwatch;
import com.appsmith.external.models.ActionDTO;
import com.appsmith.external.models.ApplicationGitReference;
import com.appsmith.external.models.BaseDomain;
import com.appsmith.external.models.DatasourceStorage;
import com.appsmith.external.models.PluginType;
import com.appsmith.git.helpers.FileUtilsImpl;
import com.appsmith.server.constants.FieldName;
import com.appsmith.server.domains.ActionCollection;
import com.appsmith.server.domains.Application;
import com.appsmith.server.domains.ApplicationPage;
import com.appsmith.server.domains.CustomJSLib;
import com.appsmith.server.domains.NewAction;
import com.appsmith.server.domains.NewPage;
import com.appsmith.server.domains.Theme;
import com.appsmith.server.dtos.ActionCollectionDTO;
import com.appsmith.server.dtos.ApplicationJson;
import com.appsmith.server.dtos.PageDTO;
import com.appsmith.server.exceptions.AppsmithError;
import com.appsmith.server.exceptions.AppsmithException;
import com.appsmith.server.helpers.CollectionUtils;
import com.appsmith.server.services.AnalyticsService;
import com.appsmith.server.services.SessionUserService;
import com.google.gson.Gson;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import net.minidev.json.JSONObject;
import net.minidev.json.parser.JSONParser;
import net.minidev.json.parser.ParseException;
import org.apache.commons.collections.PredicateUtils;
import org.apache.commons.lang3.StringUtils;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Import;
import org.springframework.stereotype.Component;
import reactor.core.Exceptions;
import reactor.core.publisher.Mono;
import java.io.IOException;
import java.lang.reflect.Field;
import java.lang.reflect.Type;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import static com.appsmith.external.constants.GitConstants.NAME_SEPARATOR;
import static com.appsmith.external.helpers.AppsmithBeanUtils.copyNestedNonNullProperties;
import static com.appsmith.external.helpers.AppsmithBeanUtils.copyProperties;
import static com.appsmith.server.constants.FieldName.ACTION_COLLECTION_LIST;
import static com.appsmith.server.constants.FieldName.ACTION_LIST;
import static com.appsmith.server.constants.FieldName.CUSTOM_JS_LIB_LIST;
import static com.appsmith.server.constants.FieldName.DATASOURCE_LIST;
import static com.appsmith.server.constants.FieldName.DECRYPTED_FIELDS;
import static com.appsmith.server.constants.FieldName.EDIT_MODE_THEME;
import static com.appsmith.server.constants.FieldName.EXPORTED_APPLICATION;
import static com.appsmith.server.constants.FieldName.PAGE_LIST;
@Slf4j
@RequiredArgsConstructor
@Component
@Import({FileUtilsImpl.class})
public class GitFileUtilsCE {
private final FileInterface fileUtils;
private final AnalyticsService analyticsService;
private final SessionUserService sessionUserService;
private final Gson gson;
// Number of seconds after lock file is stale
@Value("${appsmith.index.lock.file.time}")
public final int INDEX_LOCK_FILE_STALE_TIME = 300;
// Only include the application helper fields in metadata object
protected Set<String> getBlockedMetadataFields() {
return Set.of(
EXPORTED_APPLICATION,
DATASOURCE_LIST,
PAGE_LIST,
ACTION_LIST,
ACTION_COLLECTION_LIST,
DECRYPTED_FIELDS,
EDIT_MODE_THEME,
CUSTOM_JS_LIB_LIST);
}
/**
* This method will save the complete application in the local repo directory.
* Path to repo will be : ./container-volumes/git-repo/workspaceId/defaultApplicationId/repoName/{application_data}
*
* @param baseRepoSuffix path suffix used to create a local repo path
* @param applicationJson application reference object from which entire application can be rehydrated
* @param branchName name of the branch for the current application
* @return repo path where the application is stored
*/
public Mono<Path> saveApplicationToLocalRepo(
Path baseRepoSuffix, ApplicationJson applicationJson, String branchName)
throws IOException, GitAPIException {
/*
1. Checkout to branch
2. Create application reference for appsmith-git module
3. Save application to git repo
*/
ApplicationGitReference applicationReference = createApplicationReference(applicationJson);
// Save application to git repo
try {
return fileUtils.saveApplicationToGitRepo(baseRepoSuffix, applicationReference, branchName);
} catch (IOException | GitAPIException e) {
log.error("Error occurred while saving files to local git repo: ", e);
throw Exceptions.propagate(e);
}
}
public Mono<Path> saveApplicationToLocalRepoWithAnalytics(
Path baseRepoSuffix, ApplicationJson applicationJson, String branchName)
throws IOException, GitAPIException {
/*
1. Checkout to branch
2. Create application reference for appsmith-git module
3. Save application to git repo
*/
Stopwatch stopwatch = new Stopwatch(AnalyticsEvents.GIT_SERIALIZE_APP_RESOURCES_TO_LOCAL_FILE.getEventName());
// Save application to git repo
try {
Mono<Path> repoPathMono = saveApplicationToLocalRepo(baseRepoSuffix, applicationJson, branchName);
return Mono.zip(repoPathMono, sessionUserService.getCurrentUser()).flatMap(tuple -> {
stopwatch.stopTimer();
Path repoPath = tuple.getT1();
// Path to repo will be : ./container-volumes/git-repo/workspaceId/defaultApplicationId/repoName/
final Map<String, Object> data = Map.of(
FieldName.APPLICATION_ID,
repoPath.getParent().getFileName().toString(),
FieldName.ORGANIZATION_ID,
repoPath.getParent().getParent().getFileName().toString(),
FieldName.FLOW_NAME,
stopwatch.getFlow(),
"executionTime",
stopwatch.getExecutionTime());
return analyticsService
.sendEvent(
AnalyticsEvents.UNIT_EXECUTION_TIME.getEventName(),
tuple.getT2().getUsername(),
data)
.thenReturn(repoPath);
});
} catch (IOException | GitAPIException e) {
log.error("Error occurred while saving files to local git repo: ", e);
throw Exceptions.propagate(e);
}
}
public Mono<Path> saveApplicationToLocalRepo(
String workspaceId,
String defaultApplicationId,
String repoName,
ApplicationJson applicationJson,
String branchName)
throws GitAPIException, IOException {
Path baseRepoSuffix = Paths.get(workspaceId, defaultApplicationId, repoName);
return saveApplicationToLocalRepo(baseRepoSuffix, applicationJson, branchName);
}
/**
* Method to convert application resources to the structure which can be serialised by appsmith-git module for
* serialisation
*
* @param applicationJson application resource including actions, jsobjects, pages
* @return resource which can be saved to file system
*/
public ApplicationGitReference createApplicationReference(ApplicationJson applicationJson) {
ApplicationGitReference applicationReference = new ApplicationGitReference();
applicationReference.setUpdatedResources(applicationJson.getUpdatedResources());
setApplicationInApplicationReference(applicationJson, applicationReference);
setThemesInApplicationReference(applicationJson, applicationReference);
setApplicationMetadataInApplicationReference(applicationJson, applicationReference);
setNewPagesInApplicationReference(applicationJson, applicationReference);
setNewActionsInApplicationReference(applicationJson, applicationReference);
setActionCollectionsInApplicationReference(applicationJson, applicationReference);
setDatasourcesInApplicationReference(applicationJson, applicationReference);
setCustomJSLibsInApplicationReference(applicationJson, applicationReference);
return applicationReference;
}
private void setApplicationInApplicationReference(
ApplicationJson applicationJson, ApplicationGitReference applicationReference) {
Application application = applicationJson.getExportedApplication();
removeUnwantedFieldsFromApplication(application);
// Pass application reference
applicationReference.setApplication(applicationJson.getExportedApplication());
}
private void setApplicationMetadataInApplicationReference(
ApplicationJson applicationJson, ApplicationGitReference applicationReference) {
// Pass metadata
Iterable<String> keys = getAllFields(applicationJson)
.map(Field::getName)
.filter(name -> !getBlockedMetadataFields().contains(name))
.collect(Collectors.toList());
ApplicationJson applicationMetadata = new ApplicationJson();
applicationJson.setUpdatedResources(null);
copyProperties(applicationJson, applicationMetadata, keys);
applicationReference.setMetadata(applicationMetadata);
}
private void setThemesInApplicationReference(
ApplicationJson applicationJson, ApplicationGitReference applicationReference) {
// No need to commit publish mode theme as it leads to conflict resolution at both the places if any
applicationJson.setPublishedTheme(null);
// Remove internal fields from the themes
removeUnwantedFieldsFromBaseDomain(applicationJson.getEditModeTheme());
applicationReference.setTheme(applicationJson.getEditModeTheme());
}
private void setCustomJSLibsInApplicationReference(
ApplicationJson applicationJson, ApplicationGitReference applicationReference) {
Map<String, Object> resourceMap = new HashMap<>();
applicationJson.getCustomJSLibList().forEach(jsLib -> {
removeUnwantedFieldsFromBaseDomain(jsLib);
resourceMap.put(jsLib.getUidString(), jsLib);
});
applicationReference.setJsLibraries(resourceMap);
}
private void setDatasourcesInApplicationReference(
ApplicationJson applicationJson, ApplicationGitReference applicationReference) {
Map<String, Object> resourceMap = new HashMap<>();
// Send datasources
applicationJson.getDatasourceList().forEach(datasource -> {
removeUnwantedFieldsFromDatasource(datasource);
resourceMap.put(datasource.getName(), datasource);
});
applicationReference.setDatasources(resourceMap);
}
private void setActionCollectionsInApplicationReference(
ApplicationJson applicationJson, ApplicationGitReference applicationReference) {
Map<String, Object> resourceMap = new HashMap<>();
Map<String, String> resourceMapActionCollectionBody = new HashMap<>();
// Insert JSOObjects and also assign the keys which later will be used for saving the resource in actual
// filepath
// JSObjectName_pageName => nomenclature for the keys
applicationJson.getActionCollectionList().stream()
// As we are expecting the commit will happen only after the application is published, so we can safely
// assume if the unpublished version is deleted entity should not be committed to git
.filter(collection -> collection.getUnpublishedCollection() != null
&& collection.getUnpublishedCollection().getDeletedAt() == null)
.forEach(actionCollection -> {
String prefix = actionCollection.getUnpublishedCollection().getUserExecutableName()
+ NAME_SEPARATOR
+ actionCollection.getUnpublishedCollection().getPageId();
removeUnwantedFieldFromActionCollection(actionCollection);
String body = actionCollection.getUnpublishedCollection().getBody() != null
? actionCollection.getUnpublishedCollection().getBody()
: "";
actionCollection.getUnpublishedCollection().setBody(null);
resourceMapActionCollectionBody.put(prefix, body);
resourceMap.put(prefix, actionCollection);
});
applicationReference.setActionCollections(resourceMap);
applicationReference.setActionCollectionBody(resourceMapActionCollectionBody);
}
private void setNewPagesInApplicationReference(
ApplicationJson applicationJson, ApplicationGitReference applicationReference) {
// Insert only active pages which will then be committed to repo as individual file
Map<String, Object> resourceMap = new HashMap<>();
Map<String, String> dslBody = new HashMap<>();
applicationJson.getPageList().stream()
// As we are expecting the commit will happen only after the application is published, so we can safely
// assume if the unpublished version is deleted entity should not be committed to git
.filter(newPage -> newPage.getUnpublishedPage() != null
&& newPage.getUnpublishedPage().getDeletedAt() == null)
.forEach(newPage -> {
String pageName = newPage.getUnpublishedPage() != null
? newPage.getUnpublishedPage().getName()
: newPage.getPublishedPage().getName();
removeUnwantedFieldsFromPage(newPage);
JSONObject dsl =
newPage.getUnpublishedPage().getLayouts().get(0).getDsl();
// Get MainContainer widget data, remove the children and club with Canvas.json file
JSONObject mainContainer = new JSONObject(dsl);
mainContainer.remove("children");
newPage.getUnpublishedPage().getLayouts().get(0).setDsl(mainContainer);
// pageName will be used for naming the json file
dslBody.put(pageName, dsl.toString());
resourceMap.put(pageName, newPage);
});
applicationReference.setPages(resourceMap);
applicationReference.setPageDsl(dslBody);
}
private void setNewActionsInApplicationReference(
ApplicationJson applicationJson, ApplicationGitReference applicationReference) {
Map<String, Object> resourceMap = new HashMap<>();
Map<String, String> resourceMapBody = new HashMap<>();
// Insert active actions and also assign the keys which later will be used for saving the resource in actual
// filepath
// For actions, we are referring to validNames to maintain unique file names as just name
// field don't guarantee unique constraint for actions within JSObject
// queryValidName_pageName => nomenclature for the keys
applicationJson.getActionList().stream()
// As we are expecting the commit will happen only after the application is published, so we can safely
// assume if the unpublished version is deleted entity should not be committed to git
.filter(newAction -> newAction.getUnpublishedAction() != null
&& newAction.getUnpublishedAction().getDeletedAt() == null)
.forEach(newAction -> {
String prefix;
if (newAction.getUnpublishedAction() != null) {
prefix = newAction.getUnpublishedAction().getUserExecutableName()
+ NAME_SEPARATOR
+ newAction.getUnpublishedAction().getPageId();
} else {
prefix = newAction.getPublishedAction().getUserExecutableName()
+ NAME_SEPARATOR
+ newAction.getPublishedAction().getPageId();
}
removeUnwantedFieldFromAction(newAction);
String body = newAction.getUnpublishedAction().getActionConfiguration() != null
&& newAction
.getUnpublishedAction()
.getActionConfiguration()
.getBody()
!= null
? newAction
.getUnpublishedAction()
.getActionConfiguration()
.getBody()
: "";
// This is a special case where we are handling REMOTE type plugins based actions such as Twilio
// The user configured values are stored in a attribute called formData which is a map unlike the
// body
if (PluginType.REMOTE.equals(newAction.getPluginType())
&& newAction.getUnpublishedAction().getActionConfiguration() != null
&& newAction
.getUnpublishedAction()
.getActionConfiguration()
.getFormData()
!= null) {
body = new Gson()
.toJson(
newAction
.getUnpublishedAction()
.getActionConfiguration()
.getFormData(),
Map.class);
newAction
.getUnpublishedAction()
.getActionConfiguration()
.setFormData(null);
}
// This is a special case where we are handling JS actions as we don't want to commit the body of JS
// actions
if (PluginType.JS.equals(newAction.getPluginType())) {
if (newAction.getUnpublishedAction().getActionConfiguration() != null) {
newAction
.getUnpublishedAction()
.getActionConfiguration()
.setBody(null);
newAction.getUnpublishedAction().setJsonPathKeys(null);
}
} else {
// For the regular actions we save the body field to git repo
resourceMapBody.put(prefix, body);
}
resourceMap.put(prefix, newAction);
});
applicationReference.setActions(resourceMap);
applicationReference.setActionBody(resourceMapBody);
}
protected Stream<Field> getAllFields(ApplicationJson applicationJson) {
Class<?> currentType = applicationJson.getClass();
Set<Class<?>> classes = new HashSet<>();
while (currentType != null) {
classes.add(currentType);
currentType = currentType.getSuperclass();
}
return classes.stream().flatMap(currentClass -> Arrays.stream(currentClass.getDeclaredFields()));
}
/**
* Method to reconstruct the application from the local git repo
*
* @param workspaceId To which workspace application needs to be rehydrated
* @param defaultApplicationId Root application for the current branched application
* @param branchName for which branch the application needs to rehydrate
* @return application reference from which entire application can be rehydrated
*/
public Mono<ApplicationJson> reconstructApplicationJsonFromGitRepoWithAnalytics(
String workspaceId, String defaultApplicationId, String repoName, String branchName) {
Stopwatch stopwatch = new Stopwatch(AnalyticsEvents.GIT_DESERIALIZE_APP_RESOURCES_FROM_FILE.getEventName());
return Mono.zip(
reconstructApplicationJsonFromGitRepo(workspaceId, defaultApplicationId, repoName, branchName),
sessionUserService.getCurrentUser())
.flatMap(tuple -> {
stopwatch.stopTimer();
final Map<String, Object> data = Map.of(
FieldName.APPLICATION_ID,
defaultApplicationId,
FieldName.ORGANIZATION_ID,
workspaceId,
FieldName.FLOW_NAME,
stopwatch.getFlow(),
"executionTime",
stopwatch.getExecutionTime());
return analyticsService
.sendEvent(
AnalyticsEvents.UNIT_EXECUTION_TIME.getEventName(),
tuple.getT2().getUsername(),
data)
.thenReturn(tuple.getT1());
});
}
public Mono<ApplicationJson> reconstructApplicationJsonFromGitRepo(
String workspaceId, String defaultApplicationId, String repoName, String branchName) {
Mono<ApplicationGitReference> appReferenceMono = fileUtils.reconstructApplicationReferenceFromGitRepo(
workspaceId, defaultApplicationId, repoName, branchName);
return appReferenceMono.map(applicationReference -> {
// Extract application metadata from the json
ApplicationJson metadata =
getApplicationResource(applicationReference.getMetadata(), ApplicationJson.class);
ApplicationJson applicationJson = getApplicationJsonFromGitReference(applicationReference);
copyNestedNonNullProperties(metadata, applicationJson);
return applicationJson;
});
}
protected <T> List<T> getApplicationResource(Map<String, Object> resources, Type type) {
List<T> deserializedResources = new ArrayList<>();
if (!CollectionUtils.isNullOrEmpty(resources)) {
for (Map.Entry<String, Object> resource : resources.entrySet()) {
deserializedResources.add(getApplicationResource(resource.getValue(), type));
}
}
return deserializedResources;
}
public <T> T getApplicationResource(Object resource, Type type) {
if (resource == null) {
return null;
}
return gson.fromJson(gson.toJson(resource), type);
}
/**
* Once the user connects the existing application to a remote repo, we will initialize the repo with Readme.md -
* Url to the deployed app(view and edit mode)
* Link to discord channel for support
* Link to appsmith documentation for Git related operations
* Welcome message
*
* @param baseRepoSuffix path suffix used to create a branch repo path as per worktree implementation
* @param viewModeUrl URL to deployed version of the application view only mode
* @param editModeUrl URL to deployed version of the application edit mode
* @return Path where the Application is stored
*/
public Mono<Path> initializeReadme(Path baseRepoSuffix, String viewModeUrl, String editModeUrl) throws IOException {
return fileUtils
.initializeReadme(baseRepoSuffix, viewModeUrl, editModeUrl)
.onErrorResume(e -> Mono.error(new AppsmithException(AppsmithError.GIT_FILE_SYSTEM_ERROR, e)));
}
/**
* When the user clicks on detach remote, we need to remove the repo from the file system
*
* @param baseRepoSuffix path suffix used to create a branch repo path as per worktree implementation
* @return success on remove of file system
*/
public Mono<Boolean> deleteLocalRepo(Path baseRepoSuffix) {
return fileUtils.deleteLocalRepo(baseRepoSuffix);
}
public Mono<Boolean> checkIfDirectoryIsEmpty(Path baseRepoSuffix) throws IOException {
return fileUtils
.checkIfDirectoryIsEmpty(baseRepoSuffix)
.onErrorResume(e -> Mono.error(new AppsmithException(AppsmithError.GIT_FILE_SYSTEM_ERROR, e)));
}
protected void removeUnwantedFieldsFromBaseDomain(BaseDomain baseDomain) {
baseDomain.setPolicies(null);
baseDomain.setUserPermissions(null);
}
private void removeUnwantedFieldsFromDatasource(DatasourceStorage datasource) {
datasource.setInvalids(null);
removeUnwantedFieldsFromBaseDomain(datasource);
}
private void removeUnwantedFieldsFromPage(NewPage page) {
// As we are publishing the app and then committing to git we expect the published and unpublished PageDTO will
// be same, so we only commit unpublished PageDTO.
page.setPublishedPage(null);
removeUnwantedFieldsFromBaseDomain(page);
}
private void removeUnwantedFieldsFromApplication(Application application) {
// Don't commit application name as while importing we are using the repoName as application name
application.setName(null);
application.setPublishedPages(null);
application.setIsPublic(null);
application.setSlug(null);
application.setPublishedApplicationDetail(null);
removeUnwantedFieldsFromBaseDomain(application);
// we can call the sanitiseToExportDBObject() from BaseDomain as well here
}
private void removeUnwantedFieldFromAction(NewAction action) {
// As we are publishing the app and then committing to git we expect the published and unpublished ActionDTO
// will be same, so we only commit unpublished ActionDTO.
action.setPublishedAction(null);
action.getUnpublishedAction().sanitiseToExportDBObject();
removeUnwantedFieldsFromBaseDomain(action);
}
private void removeUnwantedFieldFromActionCollection(ActionCollection actionCollection) {
// As we are publishing the app and then committing to git we expect the published and unpublished
// ActionCollectionDTO will be same, so we only commit unpublished ActionCollectionDTO.
actionCollection.setPublishedCollection(null);
actionCollection.getUnpublishedCollection().sanitiseForExport();
removeUnwantedFieldsFromBaseDomain(actionCollection);
}
protected ApplicationJson getApplicationJsonFromGitReference(ApplicationGitReference applicationReference) {
ApplicationJson applicationJson = new ApplicationJson();
setApplicationInApplicationJson(applicationReference, applicationJson);
setThemesInApplicationJson(applicationReference, applicationJson);
setCustomJsLibsInApplicationJson(applicationReference, applicationJson);
setNewPagesInApplicationJson(applicationReference, applicationJson);
setNewActionsInApplicationJson(applicationReference, applicationJson);
setActionCollectionsInApplicationJson(applicationReference, applicationJson);
setDatasourcesInApplicationJson(applicationReference, applicationJson);
return applicationJson;
}
private void setApplicationInApplicationJson(
ApplicationGitReference applicationReference, ApplicationJson applicationJson) {
// Extract application data from the json
Application application = getApplicationResource(applicationReference.getApplication(), Application.class);
applicationJson.setExportedApplication(application);
if (application != null && !CollectionUtils.isNullOrEmpty(application.getPages())) {
// Remove null values
org.apache.commons.collections.CollectionUtils.filter(
application.getPages(), PredicateUtils.notNullPredicate());
// Create a deep clone of application pages to update independently
application.setViewMode(false);
final List<ApplicationPage> applicationPages =
new ArrayList<>(application.getPages().size());
application
.getPages()
.forEach(applicationPage ->
applicationPages.add(gson.fromJson(gson.toJson(applicationPage), ApplicationPage.class)));
application.setPublishedPages(applicationPages);
}
}
private void setThemesInApplicationJson(
ApplicationGitReference applicationReference, ApplicationJson applicationJson) {
applicationJson.setEditModeTheme(getApplicationResource(applicationReference.getTheme(), Theme.class));
// Clone the edit mode theme to published theme as both should be same for git connected application because we
// do deploy and push as a single operation
applicationJson.setPublishedTheme(applicationJson.getEditModeTheme());
}
private void setDatasourcesInApplicationJson(
ApplicationGitReference applicationReference, ApplicationJson applicationJson) {
// Extract datasources
applicationJson.setDatasourceList(
getApplicationResource(applicationReference.getDatasources(), DatasourceStorage.class));
}
private void setActionCollectionsInApplicationJson(
ApplicationGitReference applicationReference, ApplicationJson applicationJson) {
// Extract actionCollection
if (CollectionUtils.isNullOrEmpty(applicationReference.getActionCollections())) {
applicationJson.setActionCollectionList(new ArrayList<>());
} else {
Map<String, String> actionCollectionBody = applicationReference.getActionCollectionBody();
List<ActionCollection> actionCollections =
getApplicationResource(applicationReference.getActionCollections(), ActionCollection.class);
// Remove null values if present
org.apache.commons.collections.CollectionUtils.filter(actionCollections, PredicateUtils.notNullPredicate());
actionCollections.forEach(actionCollection -> {
// Set the js object body to the unpublished collection
// Since file version v3 we are splitting the js object code and metadata separately
String keyName = actionCollection.getUnpublishedCollection().getName()
+ actionCollection.getUnpublishedCollection().getPageId();
if (actionCollectionBody != null && actionCollectionBody.containsKey(keyName)) {
actionCollection.getUnpublishedCollection().setBody(actionCollectionBody.get(keyName));
}
// As we are publishing the app and then committing to git we expect the published and unpublished
// actionCollectionDTO will be same, so we create a deep copy for the published version for
// actionCollection from unpublishedActionCollectionDTO
actionCollection.setPublishedCollection(gson.fromJson(
gson.toJson(actionCollection.getUnpublishedCollection()), ActionCollectionDTO.class));
});
applicationJson.setActionCollectionList(actionCollections);
}
}
private void setNewActionsInApplicationJson(
ApplicationGitReference applicationReference, ApplicationJson applicationJson) {
// Extract actions
if (CollectionUtils.isNullOrEmpty(applicationReference.getActions())) {
applicationJson.setActionList(new ArrayList<>());
} else {
Map<String, String> actionBody = applicationReference.getActionBody();
List<NewAction> actions = getApplicationResource(applicationReference.getActions(), NewAction.class);
// Remove null values if present
org.apache.commons.collections.CollectionUtils.filter(actions, PredicateUtils.notNullPredicate());
actions.forEach(newAction -> {
// With the file version v4 we have split the actions and metadata separately into two files
// So we need to set the body to the unpublished action
String keyName = newAction.getUnpublishedAction().getName()
+ newAction.getUnpublishedAction().getPageId();
if (actionBody != null
&& (actionBody.containsKey(keyName))
&& !StringUtils.isEmpty(actionBody.get(keyName))) {
// For REMOTE plugin like Twilio the user actions are stored in key value pairs and hence they need
// to be
// deserialized separately unlike the body which is stored as string in the db.
if (newAction.getPluginType().toString().equals("REMOTE")) {
Map<String, Object> formData = gson.fromJson(actionBody.get(keyName), Map.class);
newAction
.getUnpublishedAction()
.getActionConfiguration()
.setFormData(formData);
} else {
newAction
.getUnpublishedAction()
.getActionConfiguration()
.setBody(actionBody.get(keyName));
}
}
// As we are publishing the app and then committing to git we expect the published and unpublished
// actionDTO will be same, so we create a deep copy for the published version for action from
// unpublishedActionDTO
newAction.setPublishedAction(
gson.fromJson(gson.toJson(newAction.getUnpublishedAction()), ActionDTO.class));
});
applicationJson.setActionList(actions);
}
}
private void setCustomJsLibsInApplicationJson(
ApplicationGitReference applicationReference, ApplicationJson applicationJson) {
List<CustomJSLib> customJSLibList =
getApplicationResource(applicationReference.getJsLibraries(), CustomJSLib.class);
// remove the duplicate js libraries if there is any
List<CustomJSLib> customJSLibListWithoutDuplicates = new ArrayList<>(new HashSet<>(customJSLibList));
applicationJson.setCustomJSLibList(customJSLibListWithoutDuplicates);
}
private void setNewPagesInApplicationJson(
ApplicationGitReference applicationReference, ApplicationJson applicationJson) {
// Extract pages
List<NewPage> pages = getApplicationResource(applicationReference.getPages(), NewPage.class);
// Remove null values
org.apache.commons.collections.CollectionUtils.filter(pages, PredicateUtils.notNullPredicate());
// Set the DSL to page object before saving
Map<String, String> pageDsl = applicationReference.getPageDsl();
pages.forEach(page -> {
JSONParser jsonParser = new JSONParser();
try {
if (pageDsl != null && pageDsl.get(page.getUnpublishedPage().getName()) != null) {
page.getUnpublishedPage().getLayouts().get(0).setDsl((JSONObject) jsonParser.parse(
pageDsl.get(page.getUnpublishedPage().getName())));
}
} catch (ParseException e) {
log.error(
"Error parsing the page dsl for page: {}",
page.getUnpublishedPage().getName(),
e);
throw new AppsmithException(
AppsmithError.JSON_PROCESSING_ERROR,
page.getUnpublishedPage().getName());
}
});
pages.forEach(newPage -> {
// As we are publishing the app and then committing to git we expect the published and unpublished PageDTO
// will be same, so we create a deep copy for the published version for page from the unpublishedPageDTO
newPage.setPublishedPage(gson.fromJson(gson.toJson(newPage.getUnpublishedPage()), PageDTO.class));
});
applicationJson.setPageList(pages);
}
public Mono<Long> deleteIndexLockFile(Path path) {
return fileUtils.deleteIndexLockFile(path, INDEX_LOCK_FILE_STALE_TIME);
}
}

321
app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/ce/ResponseUtilsCE.java Normal file
View File

@ -0,0 +1,321 @@
package com.appsmith.server.helpers.ce;
import com.appsmith.external.models.ActionDTO;
import com.appsmith.external.models.DefaultResources;
import com.appsmith.server.domains.ActionCollection;
import com.appsmith.server.domains.Application;
import com.appsmith.server.domains.Layout;
import com.appsmith.server.domains.NewAction;
import com.appsmith.server.domains.NewPage;
import com.appsmith.server.dtos.ActionCollectionDTO;
import com.appsmith.server.dtos.ActionCollectionViewDTO;
import com.appsmith.server.dtos.ActionViewDTO;
import com.appsmith.server.dtos.ApplicationPagesDTO;
import com.appsmith.server.dtos.LayoutDTO;
import com.appsmith.server.dtos.PageDTO;
import com.appsmith.server.dtos.PageNameIdDTO;
import com.appsmith.server.exceptions.AppsmithError;
import com.appsmith.server.exceptions.AppsmithException;
import com.appsmith.server.migrations.JsonSchemaVersions;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang.StringUtils;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.List;
@Slf4j
@Component
public class ResponseUtilsCE {
public PageDTO updatePageDTOWithDefaultResources(PageDTO page) {
DefaultResources defaultResourceIds = page.getDefaultResources();
if (defaultResourceIds == null
|| StringUtils.isEmpty(defaultResourceIds.getApplicationId())
|| StringUtils.isEmpty(defaultResourceIds.getPageId())) {
if (defaultResourceIds == null) {
return page;
}
if (StringUtils.isEmpty(defaultResourceIds.getApplicationId())) {
defaultResourceIds.setApplicationId(page.getApplicationId());
}
if (StringUtils.isEmpty(defaultResourceIds.getPageId())) {
defaultResourceIds.setPageId(page.getId());
}
}
page.setApplicationId(defaultResourceIds.getApplicationId());
page.setId(defaultResourceIds.getPageId());
page.getLayouts().stream()
.filter(layout -> !CollectionUtils.isEmpty(layout.getLayoutOnLoadActions()))
.forEach(layout -> this.updateLayoutWithDefaultResources(layout));
return page;
}
public NewPage updateNewPageWithDefaultResources(NewPage newPage) {
DefaultResources defaultResourceIds = newPage.getDefaultResources();
if (defaultResourceIds == null
|| StringUtils.isEmpty(defaultResourceIds.getApplicationId())
|| StringUtils.isEmpty(defaultResourceIds.getPageId())) {
log.error(
"Unable to find default ids for page: {}",
newPage.getId(),
new AppsmithException(AppsmithError.DEFAULT_RESOURCES_UNAVAILABLE, "page", newPage.getId()));
if (defaultResourceIds == null) {
return newPage;
}
if (StringUtils.isEmpty(defaultResourceIds.getApplicationId())) {
defaultResourceIds.setApplicationId(newPage.getApplicationId());
}
if (StringUtils.isEmpty(defaultResourceIds.getPageId())) {
defaultResourceIds.setPageId(newPage.getId());
}
}
newPage.setId(defaultResourceIds.getPageId());
newPage.setApplicationId(defaultResourceIds.getApplicationId());
if (newPage.getUnpublishedPage() != null) {
newPage.setUnpublishedPage(this.updatePageDTOWithDefaultResources(newPage.getUnpublishedPage()));
}
if (newPage.getPublishedPage() != null) {
newPage.setPublishedPage(this.updatePageDTOWithDefaultResources(newPage.getPublishedPage()));
}
return newPage;
}
public ApplicationPagesDTO updateApplicationPagesDTOWithDefaultResources(ApplicationPagesDTO applicationPages) {
List<PageNameIdDTO> pageNameIdList = applicationPages.getPages();
for (PageNameIdDTO page : pageNameIdList) {
if (StringUtils.isEmpty(page.getDefaultPageId())) {
log.error(
"Unable to find default pageId for applicationPage: {}",
page.getId(),
new AppsmithException(
AppsmithError.DEFAULT_RESOURCES_UNAVAILABLE, "applicationPage", page.getId()));
continue;
}
page.setId(page.getDefaultPageId());
}
// need to update the application also if it's present
if (applicationPages.getApplication() != null) {
applicationPages.setApplication(updateApplicationWithDefaultResources(applicationPages.getApplication()));
}
return applicationPages;
}
public ActionDTO updateActionDTOWithDefaultResources(ActionDTO action) {
DefaultResources defaultResourceIds = action.getDefaultResources();
if (defaultResourceIds == null) {
return action;
}
if (StringUtils.isEmpty(defaultResourceIds.getApplicationId())) {
defaultResourceIds.setApplicationId(action.getApplicationId());
}
if (StringUtils.isEmpty(defaultResourceIds.getPageId())) {
defaultResourceIds.setPageId(action.getPageId());
}
if (StringUtils.isEmpty(defaultResourceIds.getActionId())) {
defaultResourceIds.setActionId(action.getId());
}
action.setApplicationId(defaultResourceIds.getApplicationId());
action.setPageId(defaultResourceIds.getPageId());
action.setId(defaultResourceIds.getActionId());
if (!StringUtils.isEmpty(defaultResourceIds.getCollectionId())) {
action.setCollectionId(defaultResourceIds.getCollectionId());
}
return action;
}
public LayoutDTO updateLayoutDTOWithDefaultResources(LayoutDTO layout) {
if (!CollectionUtils.isEmpty(layout.getActionUpdates())) {
layout.getActionUpdates()
.forEach(updateLayoutAction -> updateLayoutAction.setId(updateLayoutAction.getDefaultActionId()));
}
if (!CollectionUtils.isEmpty(layout.getLayoutOnLoadActions())) {
layout.getLayoutOnLoadActions()
.forEach(layoutOnLoadAction -> layoutOnLoadAction.forEach(onLoadAction -> {
if (!StringUtils.isEmpty(onLoadAction.getDefaultActionId())) {
onLoadAction.setId(onLoadAction.getDefaultActionId());
}
if (!StringUtils.isEmpty(onLoadAction.getDefaultCollectionId())) {
onLoadAction.setCollectionId(onLoadAction.getDefaultCollectionId());
}
}));
}
return layout;
}
public Layout updateLayoutWithDefaultResources(Layout layout) {
if (!CollectionUtils.isEmpty(layout.getLayoutOnLoadActions())) {
layout.getLayoutOnLoadActions()
.forEach(layoutOnLoadAction -> layoutOnLoadAction.forEach(onLoadAction -> {
if (!StringUtils.isEmpty(onLoadAction.getDefaultActionId())) {
onLoadAction.setId(onLoadAction.getDefaultActionId());
}
if (!StringUtils.isEmpty(onLoadAction.getDefaultCollectionId())) {
onLoadAction.setCollectionId(onLoadAction.getDefaultCollectionId());
}
}));
}
return layout;
}
public ActionViewDTO updateActionViewDTOWithDefaultResources(ActionViewDTO viewDTO) {
DefaultResources defaultResourceIds = viewDTO.getDefaultResources();
if (defaultResourceIds == null) {
return viewDTO;
}
if (StringUtils.isEmpty(defaultResourceIds.getPageId())) {
defaultResourceIds.setPageId(viewDTO.getPageId());
}
if (StringUtils.isEmpty(defaultResourceIds.getActionId())) {
defaultResourceIds.setActionId(viewDTO.getId());
}
viewDTO.setId(defaultResourceIds.getActionId());
viewDTO.setPageId(defaultResourceIds.getPageId());
return viewDTO;
}
public NewAction updateNewActionWithDefaultResources(NewAction newAction) {
DefaultResources defaultResourceIds = newAction.getDefaultResources();
if (defaultResourceIds == null
|| StringUtils.isEmpty(defaultResourceIds.getApplicationId())
|| StringUtils.isEmpty(defaultResourceIds.getActionId())) {
log.error(
"Unable to find default ids for newAction: {}",
newAction.getId(),
new AppsmithException(AppsmithError.DEFAULT_RESOURCES_UNAVAILABLE, "newAction", newAction.getId()));
if (defaultResourceIds == null) {
return newAction;
}
if (StringUtils.isEmpty(defaultResourceIds.getApplicationId())) {
defaultResourceIds.setApplicationId(newAction.getApplicationId());
}
if (StringUtils.isEmpty(defaultResourceIds.getActionId())) {
defaultResourceIds.setActionId(newAction.getId());
}
}
newAction.setId(defaultResourceIds.getActionId());
newAction.setApplicationId(defaultResourceIds.getApplicationId());
if (newAction.getUnpublishedAction() != null) {
newAction.setUnpublishedAction(this.updateActionDTOWithDefaultResources(newAction.getUnpublishedAction()));
}
if (newAction.getPublishedAction() != null) {
newAction.setPublishedAction(this.updateActionDTOWithDefaultResources(newAction.getPublishedAction()));
}
return newAction;
}
public ActionCollection updateActionCollectionWithDefaultResources(ActionCollection actionCollection) {
DefaultResources defaultResourceIds = actionCollection.getDefaultResources();
if (defaultResourceIds == null) {
return actionCollection;
}
if (StringUtils.isEmpty(defaultResourceIds.getApplicationId())) {
defaultResourceIds.setApplicationId(actionCollection.getApplicationId());
}
if (StringUtils.isEmpty(defaultResourceIds.getCollectionId())) {
defaultResourceIds.setCollectionId(actionCollection.getId());
}
actionCollection.setId(defaultResourceIds.getCollectionId());
actionCollection.setApplicationId(defaultResourceIds.getApplicationId());
if (actionCollection.getUnpublishedCollection() != null) {
actionCollection.setUnpublishedCollection(
this.updateCollectionDTOWithDefaultResources(actionCollection.getUnpublishedCollection()));
}
if (actionCollection.getPublishedCollection() != null) {
actionCollection.setPublishedCollection(
this.updateCollectionDTOWithDefaultResources(actionCollection.getPublishedCollection()));
}
return actionCollection;
}
public ActionCollectionDTO updateCollectionDTOWithDefaultResources(ActionCollectionDTO collection) {
DefaultResources defaultResourceIds = collection.getDefaultResources();
if (defaultResourceIds == null) {
return collection;
}
if (StringUtils.isEmpty(defaultResourceIds.getApplicationId())) {
defaultResourceIds.setApplicationId(collection.getApplicationId());
}
if (StringUtils.isEmpty(defaultResourceIds.getPageId())) {
defaultResourceIds.setPageId(collection.getPageId());
}
if (StringUtils.isEmpty(defaultResourceIds.getCollectionId())) {
defaultResourceIds.setCollectionId(collection.getId());
}
collection.setApplicationId(defaultResourceIds.getApplicationId());
collection.setPageId(defaultResourceIds.getPageId());
collection.setId(defaultResourceIds.getCollectionId());
// Update actions within the collection
collection.getActions().forEach(this::updateActionDTOWithDefaultResources);
collection.getArchivedActions().forEach(this::updateActionDTOWithDefaultResources);
return collection;
}
public ActionCollectionViewDTO updateActionCollectionViewDTOWithDefaultResources(ActionCollectionViewDTO viewDTO) {
DefaultResources defaultResourceIds = viewDTO.getDefaultResources();
if (defaultResourceIds == null) {
return viewDTO;
}
if (StringUtils.isEmpty(defaultResourceIds.getApplicationId())) {
defaultResourceIds.setApplicationId(viewDTO.getApplicationId());
}
if (StringUtils.isEmpty(defaultResourceIds.getPageId())) {
defaultResourceIds.setPageId(viewDTO.getPageId());
}
if (StringUtils.isEmpty(defaultResourceIds.getCollectionId())) {
defaultResourceIds.setCollectionId(viewDTO.getId());
}
viewDTO.setId(defaultResourceIds.getCollectionId());
viewDTO.setApplicationId(defaultResourceIds.getApplicationId());
viewDTO.setPageId(defaultResourceIds.getPageId());
viewDTO.getActions().forEach(this::updateActionDTOWithDefaultResources);
return viewDTO;
}
public Application updateApplicationWithDefaultResources(Application application) {
if (application.getGitApplicationMetadata() != null
&& !StringUtils.isEmpty(application.getGitApplicationMetadata().getDefaultApplicationId())) {
application.setId(application.getGitApplicationMetadata().getDefaultApplicationId());
}
if (!CollectionUtils.isEmpty(application.getPages())) {
application.getPages().forEach(page -> {
if (!StringUtils.isEmpty(page.getDefaultPageId())) {
page.setId(page.getDefaultPageId());
}
});
}
if (!CollectionUtils.isEmpty(application.getPublishedPages())) {
application.getPublishedPages().forEach(page -> {
if (!StringUtils.isEmpty(page.getDefaultPageId())) {
page.setId(page.getDefaultPageId());
}
});
}
if (application.getClientSchemaVersion() == null
|| application.getServerSchemaVersion() == null
|| (JsonSchemaVersions.clientVersion.equals(application.getClientSchemaVersion())
&& JsonSchemaVersions.serverVersion.equals(application.getServerSchemaVersion()))) {
application.setIsAutoUpdate(false);
} else {
application.setIsAutoUpdate(true);
}
return application;
}
}

2
app/server/appsmith-server/src/main/java/com/appsmith/server/newactions/base/NewActionServiceCE.java
View File

@ -133,8 +133,6 @@ public interface NewActionServiceCE extends CrudService<NewAction, String> {
Map<String, Object> getAnalyticsProperties(NewAction savedAction);
void populateDefaultResources(NewAction newAction, NewAction branchedAction, String branchName);
Mono<ImportedActionAndCollectionMapsDTO> updateActionsWithImportedCollectionIds(
ImportActionCollectionResultDTO importActionCollectionResultDTO,
ImportActionResultDTO importActionResultDTO);

73
app/server/appsmith-server/src/main/java/com/appsmith/server/newactions/base/NewActionServiceCEImpl.java
View File

@ -23,6 +23,7 @@ import com.appsmith.server.acl.PolicyGenerator;
import com.appsmith.server.applications.base.ApplicationService;
import com.appsmith.server.constants.FieldName;
import com.appsmith.server.datasources.base.DatasourceService;
import com.appsmith.server.defaultresources.DefaultResourcesService;
import com.appsmith.server.domains.Action;
import com.appsmith.server.domains.ActionCollection;
import com.appsmith.server.domains.Application;
@ -136,6 +137,8 @@ public class NewActionServiceCEImpl extends BaseService<NewActionRepository, New
private final ObservationRegistry observationRegistry;
private final Map<String, Plugin> defaultPluginMap = new HashMap<>();
private final AtomicReference<Plugin> jsTypePluginReference = new AtomicReference<>();
private final DefaultResourcesService<NewAction> defaultResourcesService;
private final DefaultResourcesService<ActionDTO> dtoDefaultResourcesService;
public NewActionServiceCEImpl(
Scheduler scheduler,
@ -161,7 +164,9 @@ public class NewActionServiceCEImpl extends BaseService<NewActionRepository, New
PagePermission pagePermission,
ActionPermission actionPermission,
EntityValidationService entityValidationService,
ObservationRegistry observationRegistry) {
ObservationRegistry observationRegistry,
DefaultResourcesService<NewAction> defaultResourcesService,
DefaultResourcesService<ActionDTO> dtoDefaultResourcesService) {
super(scheduler, validator, mongoConverter, reactiveMongoTemplate, repository, analyticsService);
this.repository = repository;
@ -183,6 +188,8 @@ public class NewActionServiceCEImpl extends BaseService<NewActionRepository, New
this.applicationPermission = applicationPermission;
this.pagePermission = pagePermission;
this.actionPermission = actionPermission;
this.defaultResourcesService = defaultResourcesService;
this.dtoDefaultResourcesService = dtoDefaultResourcesService;
}
protected void setCommonFieldsFromNewActionIntoAction(NewAction newAction, ActionDTO action) {
@ -268,6 +275,7 @@ public class NewActionServiceCEImpl extends BaseService<NewActionRepository, New
* Whenever we save an action into the repository using this method, we expect that the action has all its required fields populated,
* and that this is not a partial update. As a result, all validations can be performed, and values can be reset if they do not fit
* our validations.
*
* @param newAction
* @return
*/
@ -294,19 +302,13 @@ public class NewActionServiceCEImpl extends BaseService<NewActionRepository, New
ActionDTO action = newAction.getUnpublishedAction();
defaultResourcesService.initialize(
newAction, newAction.getDefaultResources().getBranchName(), false);
dtoDefaultResourcesService.initialize(
action, newAction.getDefaultResources().getBranchName(), false);
setCommonFieldsFromNewActionIntoAction(newAction, action);
if (action.getDefaultResources() == null) {
return Mono.error(
new AppsmithException(AppsmithError.DEFAULT_RESOURCES_UNAVAILABLE, "action", action.getName()));
}
// Remove default appId, branchName and actionId to avoid duplication these resources will be present in
// NewAction level default resource
action.getDefaultResources().setActionId(null);
action.getDefaultResources().setBranchName(null);
action.getDefaultResources().setApplicationId(null);
// Default the validity to true and invalids to be an empty set.
Set<String> invalids = new HashSet<>();
@ -639,18 +641,15 @@ public class NewActionServiceCEImpl extends BaseService<NewActionRepository, New
/**
* Updates an unpublished action in the database without sending an analytics event.
*
* <p>
* This method performs an update of an unpublished action in the database without triggering an analytics event.
*
* @param id The unique identifier of the unpublished action to be updated.
* @param action The updated action object.
* @param id The unique identifier of the unpublished action to be updated.
* @param action The updated action object.
* @param permission An optional permission parameter for access control.
* @return A Mono emitting a Tuple containing the updated ActionDTO and NewAction after modification.
*
* @throws AppsmithException if the provided ID is invalid or if the action is not found.
*
* @implNote
* This method is used by {#updateUnpublishedAction(String, ActionDTO)}, but it does not send an analytics event. If analytics event tracking is not required for the update, this method can be used to improve performance and reduce overhead.
* @implNote This method is used by {#updateUnpublishedAction(String, ActionDTO)}, but it does not send an analytics event. If analytics event tracking is not required for the update, this method can be used to improve performance and reduce overhead.
*/
@Override
public Mono<Tuple2<ActionDTO, NewAction>> updateUnpublishedActionWithoutAnalytics(
@ -1646,33 +1645,6 @@ public class NewActionServiceCEImpl extends BaseService<NewActionRepository, New
return analyticsProperties;
}
@Override
public void populateDefaultResources(NewAction newAction, NewAction branchedAction, String branchName) {
DefaultResources defaultResources = branchedAction.getDefaultResources();
// Create new action but keep defaultApplicationId and defaultActionId same for both the actions
defaultResources.setBranchName(branchName);
newAction.setDefaultResources(defaultResources);
String defaultPageId = branchedAction.getUnpublishedAction() != null
? branchedAction.getUnpublishedAction().getDefaultResources().getPageId()
: branchedAction.getPublishedAction().getDefaultResources().getPageId();
DefaultResources defaultsDTO = new DefaultResources();
defaultsDTO.setPageId(defaultPageId);
if (newAction.getUnpublishedAction() != null) {
newAction.getUnpublishedAction().setDefaultResources(defaultsDTO);
}
if (newAction.getPublishedAction() != null) {
newAction.getPublishedAction().setDefaultResources(defaultsDTO);
}
newAction
.getUnpublishedAction()
.setDeletedAt(branchedAction.getUnpublishedAction().getDeletedAt());
newAction.setDeletedAt(branchedAction.getDeletedAt());
// Set policies from existing branch object
newAction.setPolicies(branchedAction.getPolicies());
}
@Override
public Mono<ImportedActionAndCollectionMapsDTO> updateActionsWithImportedCollectionIds(
ImportActionCollectionResultDTO importActionCollectionResultDTO,
@ -1722,7 +1694,7 @@ public class NewActionServiceCEImpl extends BaseService<NewActionRepository, New
.get(newAction.getId())
.get(0));
if (unpublishedAction.getDefaultResources() != null
&& org.apache.commons.lang3.StringUtils.isEmpty(
&& !StringUtils.hasText(
unpublishedAction.getDefaultResources().getCollectionId())) {
unpublishedAction
@ -1788,7 +1760,10 @@ public class NewActionServiceCEImpl extends BaseService<NewActionRepository, New
@Override
public Flux<NewAction> findByPageIdsForExport(
List<String> unpublishedPages, Optional<AclPermission> optionalPermission) {
return repository.findByPageIds(unpublishedPages, optionalPermission);
return repository.findByPageIds(unpublishedPages, optionalPermission).doOnNext(newAction -> {
this.setCommonFieldsFromNewActionIntoAction(newAction, newAction.getUnpublishedAction());
this.setCommonFieldsFromNewActionIntoAction(newAction, newAction.getPublishedAction());
});
}
@Override
@ -1857,8 +1832,6 @@ public class NewActionServiceCEImpl extends BaseService<NewActionRepository, New
defaults.setApplicationId(newAction.getApplicationId());
}
newAction.setDefaultResources(defaults);
newAction.setUnpublishedAction(action);
}
@Override

12
app/server/appsmith-server/src/main/java/com/appsmith/server/newactions/base/NewActionServiceImpl.java
View File

@ -1,8 +1,11 @@
package com.appsmith.server.newactions.base;
import com.appsmith.external.models.ActionDTO;
import com.appsmith.server.acl.PolicyGenerator;
import com.appsmith.server.applications.base.ApplicationService;
import com.appsmith.server.datasources.base.DatasourceService;
import com.appsmith.server.defaultresources.DefaultResourcesService;
import com.appsmith.server.domains.NewAction;
import com.appsmith.server.helpers.PluginExecutorHelper;
import com.appsmith.server.helpers.ResponseUtils;
import com.appsmith.server.newactions.helpers.NewActionHelper;
@ -55,8 +58,9 @@ public class NewActionServiceImpl extends NewActionServiceCEImpl implements NewA
PagePermission pagePermission,
ActionPermission actionPermission,
EntityValidationService entityValidationService,
ObservationRegistry observationRegistry) {
ObservationRegistry observationRegistry,
DefaultResourcesService<NewAction> defaultResourcesService,
DefaultResourcesService<ActionDTO> dtoDefaultResourcesService) {
super(
scheduler,
validator,
@ -81,6 +85,8 @@ public class NewActionServiceImpl extends NewActionServiceCEImpl implements NewA
pagePermission,
actionPermission,
entityValidationService,
observationRegistry);
observationRegistry,
defaultResourcesService,
dtoDefaultResourcesService);
}
}

41
app/server/appsmith-server/src/main/java/com/appsmith/server/newactions/defaultresources/ActionDTODefaultResourcesServiceCEImpl.java Normal file
View File

@ -0,0 +1,41 @@
package com.appsmith.server.newactions.defaultresources;
import com.appsmith.external.models.ActionDTO;
import com.appsmith.external.models.DefaultResources;
import com.appsmith.server.defaultresources.DefaultResourcesServiceCE;
import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils;
@Service
public class ActionDTODefaultResourcesServiceCEImpl implements DefaultResourcesServiceCE<ActionDTO> {
@Override
public ActionDTO initialize(ActionDTO domainObject, String branchName, boolean resetExistingValues) {
DefaultResources existingDefaultResources = domainObject.getDefaultResources();
DefaultResources defaultResources = new DefaultResources();
String defaultPageId = domainObject.getPageId();
if (existingDefaultResources != null && !resetExistingValues) {
// Check if there are properties to be copied over from existing
if (StringUtils.hasText(existingDefaultResources.getPageId())) {
defaultPageId = existingDefaultResources.getPageId();
}
}
defaultResources.setPageId(defaultPageId);
domainObject.setDefaultResources(defaultResources);
return domainObject;
}
@Override
public ActionDTO setFromOtherBranch(ActionDTO domainObject, ActionDTO defaultDomainObject, String branchName) {
DefaultResources defaultResources = new DefaultResources();
defaultResources.setPageId(defaultDomainObject.getDefaultResources().getPageId());
domainObject.setDefaultResources(defaultResources);
return domainObject;
}
}

9
app/server/appsmith-server/src/main/java/com/appsmith/server/newactions/defaultresources/ActionDTODefaultResourcesServiceImpl.java Normal file
View File

@ -0,0 +1,9 @@
package com.appsmith.server.newactions.defaultresources;
import com.appsmith.external.models.ActionDTO;
import com.appsmith.server.defaultresources.DefaultResourcesService;
import org.springframework.stereotype.Service;
@Service
public class ActionDTODefaultResourcesServiceImpl extends ActionDTODefaultResourcesServiceCEImpl
implements DefaultResourcesService<ActionDTO> {}

52
app/server/appsmith-server/src/main/java/com/appsmith/server/newactions/defaultresources/NewActionDefaultResourcesServiceCEImpl.java Normal file
View File

@ -0,0 +1,52 @@
package com.appsmith.server.newactions.defaultresources;
import com.appsmith.external.models.DefaultResources;
import com.appsmith.server.defaultresources.DefaultResourcesServiceCE;
import com.appsmith.server.domains.NewAction;
import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils;
@Service
public class NewActionDefaultResourcesServiceCEImpl implements DefaultResourcesServiceCE<NewAction> {
@Override
public NewAction initialize(NewAction domainObject, String branchName, boolean resetExistingValues) {
DefaultResources existingDefaultResources = domainObject.getDefaultResources();
DefaultResources defaultResources = new DefaultResources();
String defaultApplicationId = domainObject.getApplicationId();
String defaultActionId = domainObject.getId();
if (existingDefaultResources != null && !resetExistingValues) {
// Check if there are properties to be copied over from existing
if (StringUtils.hasText(existingDefaultResources.getApplicationId())) {
defaultApplicationId = existingDefaultResources.getApplicationId();
}
if (StringUtils.hasText(existingDefaultResources.getActionId())) {
defaultActionId = existingDefaultResources.getActionId();
}
}
defaultResources.setActionId(defaultActionId);
defaultResources.setApplicationId(defaultApplicationId);
defaultResources.setBranchName(branchName);
domainObject.setDefaultResources(defaultResources);
return domainObject;
}
@Override
public NewAction setFromOtherBranch(NewAction domainObject, NewAction defaultDomainObject, String branchName) {
DefaultResources defaultResources = new DefaultResources();
DefaultResources otherDefaultResources = defaultDomainObject.getDefaultResources();
defaultResources.setActionId(otherDefaultResources.getActionId());
defaultResources.setApplicationId(otherDefaultResources.getApplicationId());
defaultResources.setBranchName(branchName);
domainObject.setDefaultResources(defaultResources);
return domainObject;
}
}

9
app/server/appsmith-server/src/main/java/com/appsmith/server/newactions/defaultresources/NewActionDefaultResourcesServiceImpl.java Normal file
View File

@ -0,0 +1,9 @@
package com.appsmith.server.newactions.defaultresources;
import com.appsmith.server.defaultresources.DefaultResourcesService;
import com.appsmith.server.domains.NewAction;
import org.springframework.stereotype.Service;
@Service
public class NewActionDefaultResourcesServiceImpl extends NewActionDefaultResourcesServiceCEImpl
implements DefaultResourcesService<NewAction> {}

119
app/server/appsmith-server/src/main/java/com/appsmith/server/newactions/imports/NewActionImportableServiceCEImpl.java
View File

@ -5,6 +5,7 @@ import com.appsmith.external.models.DefaultResources;
import com.appsmith.external.models.Policy;
import com.appsmith.server.actioncollections.base.ActionCollectionService;
import com.appsmith.server.constants.FieldName;
import com.appsmith.server.defaultresources.DefaultResourcesService;
import com.appsmith.server.domains.ActionCollection;
import com.appsmith.server.domains.Application;
import com.appsmith.server.domains.NewAction;
@ -49,6 +50,8 @@ public class NewActionImportableServiceCEImpl implements ImportableServiceCE<New
private final NewActionService newActionService;
private final NewActionRepository repository;
private final ActionCollectionService actionCollectionService;
private final DefaultResourcesService<NewAction> defaultResourcesService;
private final DefaultResourcesService<ActionDTO> dtoDefaultResourcesService;
// Requires pageNameMap, pageNameToOldNameMap, pluginMap and datasourceNameToIdMap, to be present in importable
// resources.
@ -229,8 +232,7 @@ public class NewActionImportableServiceCEImpl implements ImportableServiceCE<New
if (importedApplication.getGitApplicationMetadata() != null) {
final String defaultApplicationId =
importedApplication.getGitApplicationMetadata().getDefaultApplicationId();
actionsInOtherBranchesMono = repository
.findByDefaultApplicationId(defaultApplicationId, Optional.empty())
actionsInOtherBranchesMono = getActionInOtherBranchesMono(defaultApplicationId)
.filter(newAction -> newAction.getGitSyncId() != null)
.collectMap(NewAction::getGitSyncId);
} else {
@ -307,6 +309,38 @@ public class NewActionImportableServiceCEImpl implements ImportableServiceCE<New
.get(newAction.getPluginId()));
newActionService.generateAndSetActionPolicies(parentPage, newAction);
if (importedApplication.getGitApplicationMetadata() != null) {
// application is git connected, check if the action is already present in
// any other branch
if (actionsInOtherBranches.containsKey(newAction.getGitSyncId())) {
// action found in other branch, copy the default resources from that
// action
NewAction branchedAction = getExistingActionForImportedAction(
mappedImportableResourcesDTO, actionsInOtherBranches, newAction);
defaultResourcesService.setFromOtherBranch(
newAction, branchedAction, importingMetaDTO.getBranchName());
dtoDefaultResourcesService.setFromOtherBranch(
newAction.getUnpublishedAction(),
branchedAction.getUnpublishedAction(),
importingMetaDTO.getBranchName());
} else {
// This is the first action we are saving with given gitSyncId in this
// instance
DefaultResources defaultResources = new DefaultResources();
defaultResources.setApplicationId(importedApplication
.getGitApplicationMetadata()
.getDefaultApplicationId());
defaultResources.setActionId(newAction.getId());
defaultResources.setBranchName(importingMetaDTO.getBranchName());
newAction.setDefaultResources(defaultResources);
}
} else {
DefaultResources defaultResources = new DefaultResources();
defaultResources.setApplicationId(importedApplication.getId());
defaultResources.setActionId(newAction.getId());
newAction.setDefaultResources(defaultResources);
}
// Check if the action has gitSyncId and if it's already in DB
if (existingAppContainsAction(actionsInCurrentApp, newAction)) {
@ -339,46 +373,13 @@ public class NewActionImportableServiceCEImpl implements ImportableServiceCE<New
parentPage.getId()));
}
populateDomainMappedReferences(mappedImportableResourcesDTO, newAction);
// this will generate the id and other auto generated fields e.g. createdAt
newAction.updateForBulkWriteOperation();
// set gitSyncId, if it doesn't exist
if (newAction.getGitSyncId() == null) {
newAction.setGitSyncId(newAction.getApplicationId() + "_"
+ Instant.now().toString());
}
if (importedApplication.getGitApplicationMetadata() != null) {
// application is git connected, check if the action is already present in
// any other branch
if (actionsInOtherBranches.containsKey(newAction.getGitSyncId())) {
// action found in other branch, copy the default resources from that
// action
NewAction branchedAction = getExistingActionForImportedAction(
mappedImportableResourcesDTO,
actionsInOtherBranches,
newAction);
newActionService.populateDefaultResources(
newAction, branchedAction, importingMetaDTO.getBranchName());
} else {
// This is the first action we are saving with given gitSyncId in this
// instance
DefaultResources defaultResources = new DefaultResources();
defaultResources.setApplicationId(importedApplication
.getGitApplicationMetadata()
.getDefaultApplicationId());
defaultResources.setActionId(newAction.getId());
defaultResources.setBranchName(importingMetaDTO.getBranchName());
newAction.setDefaultResources(defaultResources);
}
} else {
DefaultResources defaultResources = new DefaultResources();
defaultResources.setApplicationId(importedApplication.getId());
defaultResources.setActionId(newAction.getId());
newAction.setDefaultResources(defaultResources);
}
// Add all the properties that will be needed for a new object
populateNewAction(
importingMetaDTO,
mappedImportableResourcesDTO,
importedApplication,
actionsInOtherBranches,
newAction);
// Add it to actions list that'll be inserted or updated in bulk
newNewActionList.add(newAction);
@ -418,6 +419,24 @@ public class NewActionImportableServiceCEImpl implements ImportableServiceCE<New
});
}
private void populateNewAction(
ImportingMetaDTO importingMetaDTO,
MappedImportableResourcesDTO mappedImportableResourcesDTO,
Application importedApplication,
Map<String, NewAction> actionsInOtherBranches,
NewAction newAction) {
// this will generate the id and other auto generated fields e.g. createdAt
newAction.updateForBulkWriteOperation();
populateDomainMappedReferences(mappedImportableResourcesDTO, newAction);
// set gitSyncId, if it doesn't exist
if (newAction.getGitSyncId() == null) {
newAction.setGitSyncId(
newAction.getApplicationId() + "_" + Instant.now().toString());
}
}
protected NewAction getExistingActionForImportedAction(
MappedImportableResourcesDTO mappedImportableResourcesDTO,
Map<String, NewAction> actionsInCurrentApp,
@ -440,6 +459,10 @@ public class NewActionImportableServiceCEImpl implements ImportableServiceCE<New
.filter(newAction -> newAction.getGitSyncId() != null);
}
protected Flux<NewAction> getActionInOtherBranchesMono(String defaultApplicationId) {
return repository.findByDefaultApplicationId(defaultApplicationId, Optional.empty());
}
private NewPage updatePageInAction(
ActionDTO action, Map<String, NewPage> pageNameMap, Map<String, String> actionIdMap) {
NewPage parentPage = pageNameMap.get(action.getPageId());
@ -488,6 +511,13 @@ public class NewActionImportableServiceCEImpl implements ImportableServiceCE<New
private void putActionIdInMap(NewAction newAction, ImportActionResultDTO importActionResultDTO) {
// Populate actionIdsMap to associate the appropriate actions to run on page load
String defaultResourcesActionId = newAction.getDefaultResources().getActionId();
if (defaultResourcesActionId == null) {
defaultResourcesActionId = newAction.getId();
newAction.getDefaultResources().setActionId(newAction.getId());
}
if (newAction.getUnpublishedAction() != null) {
ActionDTO unpublishedAction = newAction.getUnpublishedAction();
importActionResultDTO
@ -505,7 +535,8 @@ public class NewActionImportableServiceCEImpl implements ImportableServiceCE<New
final Map<String, String> actionIds = importActionResultDTO
.getUnpublishedCollectionIdToActionIdsMap()
.get(unpublishedAction.getCollectionId());
actionIds.put(newAction.getDefaultResources().getActionId(), newAction.getId());
actionIds.put(defaultResourcesActionId, newAction.getId());
}
}
if (newAction.getPublishedAction() != null) {
@ -525,7 +556,7 @@ public class NewActionImportableServiceCEImpl implements ImportableServiceCE<New
final Map<String, String> actionIds = importActionResultDTO
.getPublishedCollectionIdToActionIdsMap()
.get(publishedAction.getCollectionId());
actionIds.put(newAction.getDefaultResources().getActionId(), newAction.getId());
actionIds.put(defaultResourcesActionId, newAction.getId());
}
}
}

13
app/server/appsmith-server/src/main/java/com/appsmith/server/newactions/imports/NewActionImportableServiceImpl.java
View File

@ -1,6 +1,8 @@
package com.appsmith.server.newactions.imports;
import com.appsmith.external.models.ActionDTO;
import com.appsmith.server.actioncollections.base.ActionCollectionService;
import com.appsmith.server.defaultresources.DefaultResourcesService;
import com.appsmith.server.domains.NewAction;
import com.appsmith.server.imports.importable.ImportableService;
import com.appsmith.server.newactions.base.NewActionService;
@ -13,7 +15,14 @@ public class NewActionImportableServiceImpl extends NewActionImportableServiceCE
public NewActionImportableServiceImpl(
NewActionService newActionService,
NewActionRepository repository,
ActionCollectionService actionCollectionService) {
super(newActionService, repository, actionCollectionService);
ActionCollectionService actionCollectionService,
DefaultResourcesService<NewAction> defaultResourcesService,
DefaultResourcesService<ActionDTO> dtoDefaultResourcesService) {
super(
newActionService,
repository,
actionCollectionService,
defaultResourcesService,
dtoDefaultResourcesService);
}
}

50
app/server/appsmith-server/src/main/java/com/appsmith/server/newpages/defaultresources/NewPageDefaultResourcesServiceCEImpl.java Normal file
View File

@ -0,0 +1,50 @@
package com.appsmith.server.newpages.defaultresources;
import com.appsmith.external.models.DefaultResources;
import com.appsmith.server.defaultresources.DefaultResourcesServiceCE;
import com.appsmith.server.domains.NewPage;
import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils;
@Service
public class NewPageDefaultResourcesServiceCEImpl implements DefaultResourcesServiceCE<NewPage> {
@Override
public NewPage initialize(NewPage domainObject, String branchName, boolean resetExistingValues) {
DefaultResources existingDefaultResources = domainObject.getDefaultResources();
DefaultResources defaultResources = new DefaultResources();
String defaultApplicationId = domainObject.getApplicationId();
String defaultPageId = domainObject.getId();
if (existingDefaultResources != null && !resetExistingValues) {
// Check if there are properties to be copied over from existing
if (StringUtils.hasText(existingDefaultResources.getApplicationId())) {
defaultApplicationId = existingDefaultResources.getApplicationId();
}
if (StringUtils.hasText(existingDefaultResources.getPageId())) {
defaultPageId = existingDefaultResources.getPageId();
}
}
defaultResources.setPageId(defaultPageId);
defaultResources.setApplicationId(defaultApplicationId);
defaultResources.setBranchName(branchName);
domainObject.setDefaultResources(defaultResources);
return domainObject;
}
@Override
public NewPage setFromOtherBranch(NewPage domainObject, NewPage defaultDomainObject, String branchName) {
DefaultResources defaultResources = new DefaultResources();
defaultResources.setPageId(defaultDomainObject.getId());
defaultResources.setApplicationId(defaultDomainObject.getApplicationId());
defaultResources.setBranchName(branchName);
domainObject.setDefaultResources(defaultResources);
return domainObject;
}
}

9
app/server/appsmith-server/src/main/java/com/appsmith/server/newpages/defaultresources/NewPageDefaultResourcesServiceImpl.java Normal file
View File

@ -0,0 +1,9 @@
package com.appsmith.server.newpages.defaultresources;
import com.appsmith.server.defaultresources.DefaultResourcesService;
import com.appsmith.server.domains.NewPage;
import org.springframework.stereotype.Service;
@Service
public class NewPageDefaultResourcesServiceImpl extends NewPageDefaultResourcesServiceCEImpl
implements DefaultResourcesService<NewPage> {}

3
app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/AppsmithRepository.java
View File

@ -1,6 +1,7 @@
package com.appsmith.server.repositories;
import com.appsmith.server.acl.AclPermission;
import com.mongodb.bulk.BulkWriteResult;
import com.mongodb.client.result.InsertManyResult;
import org.springframework.data.domain.Sort;
import org.springframework.data.mongodb.core.query.Criteria;
@ -48,4 +49,6 @@ public interface AppsmithRepository<T> {
* @return List of actions that were passed in the method
*/
Mono<List<InsertManyResult>> bulkInsert(List<T> domainList);
Mono<List<BulkWriteResult>> bulkUpdate(List<T> domainList);
}

28
app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/ce/BaseAppsmithRepositoryCEImpl.java
View File

@ -11,11 +11,15 @@ import com.appsmith.server.exceptions.AppsmithException;
import com.appsmith.server.repositories.CacheableRepositoryHelper;
import com.mongodb.BasicDBObject;
import com.mongodb.DBObject;
import com.mongodb.bulk.BulkWriteResult;
import com.mongodb.client.model.UpdateOneModel;
import com.mongodb.client.model.WriteModel;
import com.mongodb.client.result.InsertManyResult;
import com.mongodb.client.result.UpdateResult;
import com.querydsl.core.types.Path;
import jakarta.validation.constraints.NotNull;
import org.bson.Document;
import org.bson.types.ObjectId;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.GenericTypeResolver;
import org.springframework.data.domain.Sort;
@ -743,4 +747,28 @@ public abstract class BaseAppsmithRepositoryCEImpl<T extends BaseDomain> {
.flatMapMany(documentMongoCollection -> documentMongoCollection.insertMany(dbObjects))
.collectList();
}
public Mono<List<BulkWriteResult>> bulkUpdate(List<T> domainObjects) {
if (CollectionUtils.isEmpty(domainObjects)) {
return Mono.just(Collections.emptyList());
}
// convert the list of new actions to a list of DBObjects
List<WriteModel<Document>> dbObjects = domainObjects.stream()
.map(actionCollection -> {
assert actionCollection.getId() != null;
Document document = new Document();
mongoOperations.getConverter().write(actionCollection, document);
document.remove("_id");
return (WriteModel<Document>) new UpdateOneModel<Document>(
new Document("_id", new ObjectId(actionCollection.getId())),
new Document("$set", document));
})
.collect(Collectors.toList());
return mongoOperations
.getCollection(mongoOperations.getCollectionName(genericDomain))
.flatMapMany(documentMongoCollection -> documentMongoCollection.bulkWrite(dbObjects))
.collectList();
}
}

3
app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/ce/CustomActionCollectionRepositoryCE.java
View File

@ -4,7 +4,6 @@ import com.appsmith.external.models.CreatorContextType;
import com.appsmith.server.acl.AclPermission;
import com.appsmith.server.domains.ActionCollection;
import com.appsmith.server.repositories.AppsmithRepository;
import com.mongodb.bulk.BulkWriteResult;
import org.springframework.data.domain.Sort;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
@ -49,8 +48,6 @@ public interface CustomActionCollectionRepositoryCE extends AppsmithRepository<A
Flux<ActionCollection> findByPageIds(List<String> pageIds, Optional<AclPermission> permission);
Mono<List<BulkWriteResult>> bulkUpdate(List<ActionCollection> actionCollections);
Flux<ActionCollection> findAllByApplicationIds(List<String> applicationIds, List<String> includeFields);
Flux<ActionCollection> findAllUnpublishedActionCollectionsByContextIdAndContextType(

33
app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/ce/CustomActionCollectionRepositoryCEImpl.java
View File

@ -8,25 +8,17 @@ import com.appsmith.server.domains.ActionCollection;
import com.appsmith.server.domains.QActionCollection;
import com.appsmith.server.repositories.BaseAppsmithRepositoryImpl;
import com.appsmith.server.repositories.CacheableRepositoryHelper;
import com.mongodb.bulk.BulkWriteResult;
import com.mongodb.client.model.UpdateOneModel;
import com.mongodb.client.model.WriteModel;
import org.apache.commons.lang3.StringUtils;
import org.bson.Document;
import org.bson.types.ObjectId;
import org.springframework.data.domain.Sort;
import org.springframework.data.mongodb.core.ReactiveMongoOperations;
import org.springframework.data.mongodb.core.convert.MongoConverter;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.util.CollectionUtils;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
import static org.springframework.data.mongodb.core.query.Criteria.where;
@ -236,31 +228,6 @@ public class CustomActionCollectionRepositoryCEImpl extends BaseAppsmithReposito
return queryAll(List.of(pageIdCriteria), permission);
}
@Override
public Mono<List<BulkWriteResult>> bulkUpdate(List<ActionCollection> actionCollections) {
if (CollectionUtils.isEmpty(actionCollections)) {
return Mono.just(Collections.emptyList());
}
// convert the list of new actions to a list of DBObjects
List<WriteModel<Document>> dbObjects = actionCollections.stream()
.map(actionCollection -> {
assert actionCollection.getId() != null;
Document document = new Document();
mongoOperations.getConverter().write(actionCollection, document);
document.remove("_id");
return (WriteModel<Document>) new UpdateOneModel<Document>(
new Document("_id", new ObjectId(actionCollection.getId())),
new Document("$set", document));
})
.collect(Collectors.toList());
return mongoOperations
.getCollection(mongoOperations.getCollectionName(ActionCollection.class))
.flatMapMany(documentMongoCollection -> documentMongoCollection.bulkWrite(dbObjects))
.collectList();
}
@Override
public Flux<ActionCollection> findAllByApplicationIds(List<String> applicationIds, List<String> includeFields) {
Criteria applicationCriteria = Criteria.where(FieldName.APPLICATION_ID).in(applicationIds);

2
app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/ce/CustomNewActionRepositoryCE.java
View File

@ -73,8 +73,6 @@ public interface CustomNewActionRepositoryCE extends AppsmithRepository<NewActio
Flux<NewAction> findAllNonJsActionsByNameAndPageIdsAndViewMode(
String name, List<String> pageIds, Boolean viewMode, AclPermission aclPermission, Sort sort);
Mono<List<BulkWriteResult>> bulkUpdate(List<NewAction> newActions);
Mono<List<BulkWriteResult>> publishActions(String applicationId, AclPermission permission);
Mono<UpdateResult> archiveDeletedUnpublishedActions(String applicationId, AclPermission permission);

30
app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/ce/CustomNewActionRepositoryCEImpl.java
View File

@ -12,11 +12,8 @@ import com.appsmith.server.dtos.PluginTypeAndCountDTO;
import com.appsmith.server.repositories.BaseAppsmithRepositoryImpl;
import com.appsmith.server.repositories.CacheableRepositoryHelper;
import com.mongodb.bulk.BulkWriteResult;
import com.mongodb.client.model.UpdateOneModel;
import com.mongodb.client.model.WriteModel;
import com.mongodb.client.result.UpdateResult;
import lombok.extern.slf4j.Slf4j;
import org.bson.Document;
import org.bson.types.ObjectId;
import org.springframework.data.domain.Sort;
import org.springframework.data.mongodb.core.MongoTemplate;
@ -31,18 +28,15 @@ import org.springframework.data.mongodb.core.convert.MongoConverter;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.data.mongodb.core.query.Update;
import org.springframework.util.CollectionUtils;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import reactor.core.scheduler.Schedulers;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import static org.springframework.data.mongodb.core.aggregation.Aggregation.group;
import static org.springframework.data.mongodb.core.aggregation.Aggregation.match;
@ -537,30 +531,6 @@ public class CustomNewActionRepositoryCEImpl extends BaseAppsmithRepositoryImpl<
return criteriaList;
}
@Override
public Mono<List<BulkWriteResult>> bulkUpdate(List<NewAction> newActions) {
if (CollectionUtils.isEmpty(newActions)) {
return Mono.just(Collections.emptyList());
}
// convert the list of new actions to a list of DBObjects
List<WriteModel<Document>> dbObjects = newActions.stream()
.map(newAction -> {
assert newAction.getId() != null;
Document document = new Document();
mongoOperations.getConverter().write(newAction, document);
document.remove("_id");
return (WriteModel<Document>) new UpdateOneModel<Document>(
new Document("_id", new ObjectId(newAction.getId())), new Document("$set", document));
})
.collect(Collectors.toList());
return mongoOperations
.getCollection(mongoOperations.getCollectionName(NewAction.class))
.flatMapMany(documentMongoCollection -> documentMongoCollection.bulkWrite(dbObjects))
.collectList();
}
@Override
public Flux<NewAction> findByDefaultApplicationId(String defaultApplicationId, Optional<AclPermission> permission) {
final String defaultResources = fieldName(QBranchAwareDomain.branchAwareDomain.defaultResources);

2
app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/ce/CustomNewPageRepositoryCE.java
View File

@ -45,7 +45,5 @@ public interface CustomNewPageRepositoryCE extends AppsmithRepository<NewPage> {
Mono<List<BulkWriteResult>> publishPages(Collection<String> pageIds, AclPermission permission);
Mono<List<BulkWriteResult>> bulkUpdate(List<NewPage> newPages);
Flux<NewPage> findAllByApplicationIdsWithoutPermission(List<String> applicationIds, List<String> includeFields);
}

31
app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/ce/CustomNewPageRepositoryCEImpl.java
View File

@ -10,11 +10,7 @@ import com.appsmith.server.dtos.PageDTO;
import com.appsmith.server.repositories.BaseAppsmithRepositoryImpl;
import com.appsmith.server.repositories.CacheableRepositoryHelper;
import com.mongodb.bulk.BulkWriteResult;
import com.mongodb.client.model.UpdateOneModel;
import com.mongodb.client.model.WriteModel;
import lombok.extern.slf4j.Slf4j;
import org.bson.Document;
import org.bson.types.ObjectId;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.ReactiveMongoOperations;
import org.springframework.data.mongodb.core.aggregation.Aggregation;
@ -23,18 +19,15 @@ import org.springframework.data.mongodb.core.aggregation.Fields;
import org.springframework.data.mongodb.core.convert.MongoConverter;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.util.CollectionUtils;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import reactor.core.scheduler.Schedulers;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import static org.springframework.data.mongodb.core.query.Criteria.where;
@ -298,30 +291,6 @@ public class CustomNewPageRepositoryCEImpl extends BaseAppsmithRepositoryImpl<Ne
});
}
@Override
public Mono<List<BulkWriteResult>> bulkUpdate(List<NewPage> newPages) {
if (CollectionUtils.isEmpty(newPages)) {
return Mono.just(Collections.emptyList());
}
// convert the list of new pages to a list of DBObjects
List<WriteModel<Document>> dbObjects = newPages.stream()
.map(newPage -> {
assert newPage.getId() != null;
Document document = new Document();
mongoOperations.getConverter().write(newPage, document);
document.remove("_id");
return (WriteModel<Document>) new UpdateOneModel<Document>(
new Document("_id", new ObjectId(newPage.getId())), new Document("$set", document));
})
.collect(Collectors.toList());
return mongoOperations
.getCollection(mongoOperations.getCollectionName(NewPage.class))
.flatMapMany(documentMongoCollection -> documentMongoCollection.bulkWrite(dbObjects))
.collectList();
}
@Override
public Flux<NewPage> findAllByApplicationIdsWithoutPermission(
List<String> applicationIds, List<String> includeFields) {

1
app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/LayoutCollectionServiceCEImpl.java
View File

@ -160,7 +160,6 @@ public class LayoutCollectionServiceCEImpl implements LayoutCollectionServiceCE
defaultResources.setBranchName(branchName);
collectionDTO.setDefaultResources(defaultResources);
actionCollection.setDefaultResources(defaultResources);
actionCollection.setUnpublishedCollection(collectionDTO);
actionCollectionService.generateAndSetPolicies(newPage, actionCollection);
actionCollection.setUnpublishedCollection(collectionDTO);
return Mono.zip(

7
app/server/appsmith-server/src/test/java/com/appsmith/server/services/ActionCollectionServiceImplTest.java
View File

@ -10,6 +10,7 @@ import com.appsmith.server.actioncollections.base.ActionCollectionService;
import com.appsmith.server.actioncollections.base.ActionCollectionServiceImpl;
import com.appsmith.server.applications.base.ApplicationService;
import com.appsmith.server.constants.FieldName;
import com.appsmith.server.defaultresources.DefaultResourcesService;
import com.appsmith.server.domains.ActionCollection;
import com.appsmith.server.domains.Layout;
import com.appsmith.server.domains.NewAction;
@ -124,6 +125,9 @@ public class ActionCollectionServiceImplTest {
@MockBean
private PolicyGenerator policyGenerator;
@MockBean
private DefaultResourcesService<ActionCollection> actionCollectionDefaultResourcesService;
@BeforeEach
public void setUp() {
applicationPermission = new ApplicationPermissionImpl();
@ -141,7 +145,8 @@ public class ActionCollectionServiceImplTest {
applicationService,
responseUtils,
applicationPermission,
actionPermission);
actionPermission,
actionCollectionDefaultResourcesService);
layoutCollectionService = new LayoutCollectionServiceImpl(
newPageService,

11
app/server/appsmith-server/src/test/java/com/appsmith/server/services/ce/NewActionServiceUnitTest.java
View File

@ -6,6 +6,7 @@ import com.appsmith.external.models.PluginType;
import com.appsmith.server.acl.PolicyGenerator;
import com.appsmith.server.applications.base.ApplicationService;
import com.appsmith.server.datasources.base.DatasourceService;
import com.appsmith.server.defaultresources.DefaultResourcesService;
import com.appsmith.server.domains.NewAction;
import com.appsmith.server.domains.Plugin;
import com.appsmith.server.helpers.PluginExecutorHelper;
@ -123,6 +124,12 @@ public class NewActionServiceUnitTest {
@MockBean
ObservationRegistry observationRegistry;
@MockBean
DefaultResourcesService<NewAction> defaultResourcesService;
@MockBean
DefaultResourcesService<ActionDTO> dtoDefaultResourcesService;
@BeforeEach
public void setup() {
newActionService = new NewActionServiceCEImpl(
@ -149,7 +156,9 @@ public class NewActionServiceUnitTest {
pagePermission,
actionPermission,
entityValidationService,
observationRegistry);
observationRegistry,
defaultResourcesService,
dtoDefaultResourcesService);
ObservationRegistry.ObservationConfig mockObservationConfig =
Mockito.mock(ObservationRegistry.ObservationConfig.class);