From dea1c030da221cc4dd614e03615287fb4538184a Mon Sep 17 00:00:00 2001 From: Diljit Date: Fri, 13 Jun 2025 12:16:18 +0530 Subject: [PATCH] chore: in memory git status (#40891) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Description > [!TIP] > _Add a TL;DR when the description is longer than 500 words or extremely technical (helps the content, marketing, and DevRel team)._ > > _Please also include relevant motivation and context. List any dependencies that are required for this change. Add links to Notion, Figma or any other documents that might be relevant to the PR._ Fixes #`Issue Number` _or_ Fixes `Issue URL` > [!WARNING] > _If no issue exists, please create an issue first, and check with the maintainers if the issue is valid._ ## Automation /ok-to-test tags="@tag.Git" ### :mag: Cypress test results > [!TIP] > 🟢 🟢 🟢 All cypress tests have passed! 🎉 🎉 🎉 > Workflow run: > Commit: b50460f31752b3db4a31f035b19b1ea712c7fbde > Cypress dashboard. > Tags: `@tag.Git` > Spec: >
Thu, 12 Jun 2025 11:13:06 UTC ## Communication Should the DevRel and Marketing teams inform users about this change? - [ ] Yes - [ ] No ## Summary by CodeRabbit - **New Features** - Introduced detailed Git status computation showing added, modified, and removed files. - Added branch tracking status retrieval for repositories. - **Improvements** - Enhanced integration of Git status and branch tracking features across interfaces and services. - Improved status reporting by enriching Git status with branch tracking details. --- .../appsmith/git/files/FileUtilsCEImpl.java | 74 ++++++++++++++++++- .../com/appsmith/git/files/FileUtilsImpl.java | 13 +++- .../git/handler/ce/FSGitHandlerCEImpl.java | 3 +- .../git/helpers/FileUtilsImplTest.java | 4 +- .../appsmith/external/git/FileInterface.java | 5 ++ .../external/git/handler/FSGitHandler.java | 2 + .../git/central/CentralGitServiceCEImpl.java | 33 ++++++++- .../git/central/GitHandlingServiceCE.java | 3 + .../server/git/fs/GitFSServiceCEImpl.java | 13 ++++ .../helpers/ce/CommonGitFileUtilsCE.java | 13 ++++ 10 files changed, 152 insertions(+), 11 deletions(-) diff --git a/app/server/appsmith-git/src/main/java/com/appsmith/git/files/FileUtilsCEImpl.java b/app/server/appsmith-git/src/main/java/com/appsmith/git/files/FileUtilsCEImpl.java index 862a8b9596..5cd5eef146 100644 --- a/app/server/appsmith-git/src/main/java/com/appsmith/git/files/FileUtilsCEImpl.java +++ b/app/server/appsmith-git/src/main/java/com/appsmith/git/files/FileUtilsCEImpl.java @@ -1,5 +1,6 @@ package com.appsmith.git.files; +import com.appsmith.external.dtos.GitStatusDTO; import com.appsmith.external.dtos.ModifiedResources; import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginError; import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginException; @@ -19,6 +20,7 @@ import com.appsmith.git.configurations.GitServiceConfig; import com.appsmith.git.constants.CommonConstants; import com.appsmith.git.helpers.DSLTransformerHelper; import com.fasterxml.jackson.databind.ObjectMapper; +import io.micrometer.observation.ObservationRegistry; import io.micrometer.tracing.Span; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -31,6 +33,7 @@ import org.springframework.context.annotation.Import; import org.springframework.stereotype.Component; import org.springframework.util.FileSystemUtils; import org.springframework.util.StringUtils; +import reactor.core.observability.micrometer.Micrometer; import reactor.core.publisher.Mono; import reactor.core.scheduler.Scheduler; import reactor.core.scheduler.Schedulers; @@ -90,7 +93,7 @@ public class FileUtilsCEImpl implements FileInterface { protected final FileOperations fileOperations; private final ObservationHelper observationHelper; protected final ObjectMapper objectMapper; - + private final ObservationRegistry observationRegistry; private static final String EDIT_MODE_URL_TEMPLATE = "{{editModeUrl}}"; private static final String VIEW_MODE_URL_TEMPLATE = "{{viewModeUrl}}"; @@ -108,13 +111,15 @@ public class FileUtilsCEImpl implements FileInterface { GitExecutor gitExecutor, FileOperations fileOperations, ObservationHelper observationHelper, - ObjectMapper objectMapper) { + ObjectMapper objectMapper, + ObservationRegistry observationRegistry) { this.gitServiceConfig = gitServiceConfig; this.fsGitHandler = fsGitHandler; this.gitExecutor = gitExecutor; this.fileOperations = fileOperations; this.observationHelper = observationHelper; this.objectMapper = objectMapper; + this.observationRegistry = observationRegistry; } protected Map getModifiedResourcesTypes() { @@ -290,6 +295,66 @@ public class FileUtilsCEImpl implements FileInterface { .subscribeOn(scheduler); } + public Mono computeGitStatus( + Path baseRepoSuffix, GitResourceMap gitResourceMapFromDB, String branchName, boolean keepWorkingDirChanges) + throws GitAPIException, IOException { + return fsGitHandler + .resetToLastCommit(baseRepoSuffix, branchName, keepWorkingDirChanges) + .flatMap(__ -> constructGitResourceMapFromGitRepo(baseRepoSuffix, branchName)) + .flatMap(gitResourceMapFromFS -> { + Map resourceMapFromDB = gitResourceMapFromDB.getGitResourceMap(); + Map resourceMapFromFS = gitResourceMapFromFS.getGitResourceMap(); + + Map filePathObjectsMapFromFS = resourceMapFromFS.entrySet().parallelStream() + .collect( + Collectors.toMap(entry -> entry.getKey().getFilePath(), entry -> entry.getValue())); + + Map filePathToObjectsFromDB = resourceMapFromDB.entrySet().parallelStream() + .collect( + Collectors.toMap(entry -> entry.getKey().getFilePath(), entry -> entry.getValue())); + + Set filePathsInDb = new HashSet<>(filePathToObjectsFromDB.keySet()); + Set filePathsInFS = new HashSet<>(filePathObjectsMapFromFS.keySet()); + + // added files + Set addedFiles = new HashSet<>(filePathsInDb); + addedFiles.removeAll(filePathsInFS); + + // removed files + Set removedFiles = new HashSet<>(filePathsInFS); + removedFiles.removeAll(filePathsInDb); + removedFiles.remove(README_FILE_NAME); + + // common files + Set commonFiles = new HashSet<>(filePathsInDb); + commonFiles.retainAll(filePathsInFS); + + // modified files + Set modifiedFiles = commonFiles.stream() + .filter(filePath -> { + Object fileInDB = filePathToObjectsFromDB.get(filePath); + Object fileInFS = filePathObjectsMapFromFS.get(filePath); + try { + return fileOperations.hasFileChanged(fileInDB, fileInFS); + } catch (IOException e) { + log.error("Error while checking if file has changed", e); + return false; + } + }) + .collect(Collectors.toSet()); + + GitStatusDTO localRepoStatus = new GitStatusDTO(); + localRepoStatus.setAdded(addedFiles); + localRepoStatus.setModified(modifiedFiles); + localRepoStatus.setRemoved(removedFiles); + boolean isClean = addedFiles.isEmpty() && modifiedFiles.isEmpty() && removedFiles.isEmpty(); + localRepoStatus.setIsClean(isClean); + + fsGitHandler.populateModifiedEntities(localRepoStatus); + return Mono.just(localRepoStatus); + }); + } + protected Set getWhitelistedPaths() { String pages = PAGE_DIRECTORY + DELIMITER_PATH; String datasources = DATASOURCE_DIRECTORY + DELIMITER_PATH; @@ -802,7 +867,10 @@ public class FileUtilsCEImpl implements FileInterface { @Override public Mono constructGitResourceMapFromGitRepo(Path repositorySuffix, String refName) { Path repositoryPath = Paths.get(gitServiceConfig.getGitRootPath()).resolve(repositorySuffix); - return Mono.fromCallable(() -> fetchGitResourceMap(repositoryPath)).subscribeOn(scheduler); + return Mono.fromCallable(() -> fetchGitResourceMap(repositoryPath)) + .subscribeOn(scheduler) + .name("construct-git-resource-map") + .tap(Micrometer.observation(observationRegistry)); } /** diff --git a/app/server/appsmith-git/src/main/java/com/appsmith/git/files/FileUtilsImpl.java b/app/server/appsmith-git/src/main/java/com/appsmith/git/files/FileUtilsImpl.java index 9122b5d7c3..532101f3fe 100644 --- a/app/server/appsmith-git/src/main/java/com/appsmith/git/files/FileUtilsImpl.java +++ b/app/server/appsmith-git/src/main/java/com/appsmith/git/files/FileUtilsImpl.java @@ -7,6 +7,7 @@ import com.appsmith.external.git.operations.FileOperations; import com.appsmith.external.helpers.ObservationHelper; import com.appsmith.git.configurations.GitServiceConfig; import com.fasterxml.jackson.databind.ObjectMapper; +import io.micrometer.observation.ObservationRegistry; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.springframework.context.annotation.Import; @@ -26,7 +27,15 @@ public class FileUtilsImpl extends FileUtilsCEImpl implements FileInterface { GitExecutor gitExecutor, FileOperations fileOperations, ObservationHelper observationHelper, - ObjectMapper objectMapper) { - super(gitServiceConfig, fsGitHandler, gitExecutor, fileOperations, observationHelper, objectMapper); + ObjectMapper objectMapper, + ObservationRegistry observationRegistry) { + super( + gitServiceConfig, + fsGitHandler, + gitExecutor, + fileOperations, + observationHelper, + objectMapper, + observationRegistry); } } diff --git a/app/server/appsmith-git/src/main/java/com/appsmith/git/handler/ce/FSGitHandlerCEImpl.java b/app/server/appsmith-git/src/main/java/com/appsmith/git/handler/ce/FSGitHandlerCEImpl.java index 14f08fb935..8900ff542d 100644 --- a/app/server/appsmith-git/src/main/java/com/appsmith/git/handler/ce/FSGitHandlerCEImpl.java +++ b/app/server/appsmith-git/src/main/java/com/appsmith/git/handler/ce/FSGitHandlerCEImpl.java @@ -935,7 +935,8 @@ public class FSGitHandlerCEImpl implements FSGitHandler { .subscribeOn(scheduler); } - protected void populateModifiedEntities(GitStatusDTO response) { + @Override + public void populateModifiedEntities(GitStatusDTO response) { populatePageChanges(response); populateQueryChanges(response); populateJsObjectChanges(response); diff --git a/app/server/appsmith-git/src/test/java/com/appsmith/git/helpers/FileUtilsImplTest.java b/app/server/appsmith-git/src/test/java/com/appsmith/git/helpers/FileUtilsImplTest.java index c3779473dc..74ea5e1412 100644 --- a/app/server/appsmith-git/src/test/java/com/appsmith/git/helpers/FileUtilsImplTest.java +++ b/app/server/appsmith-git/src/test/java/com/appsmith/git/helpers/FileUtilsImplTest.java @@ -9,6 +9,7 @@ import com.appsmith.git.files.FileUtilsImpl; import com.appsmith.git.files.operations.FileOperationsImpl; import com.appsmith.git.service.GitExecutorImpl; import com.fasterxml.jackson.databind.ObjectMapper; +import io.micrometer.observation.ObservationRegistry; import org.apache.commons.io.FileUtils; import org.eclipse.jgit.api.errors.GitAPIException; import org.junit.jupiter.api.AfterEach; @@ -51,7 +52,8 @@ public class FileUtilsImplTest { gitExecutor, fileOperations, ObservationHelper.NOOP, - new ObjectMapper()); + new ObjectMapper(), + ObservationRegistry.NOOP); } @AfterEach diff --git a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/git/FileInterface.java b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/git/FileInterface.java index 389feea2fc..234f23ddc4 100644 --- a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/git/FileInterface.java +++ b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/git/FileInterface.java @@ -1,5 +1,6 @@ package com.appsmith.external.git; +import com.appsmith.external.dtos.GitStatusDTO; import com.appsmith.external.git.models.GitResourceMap; import com.appsmith.external.models.ApplicationGitReference; import com.appsmith.external.models.ArtifactGitReference; @@ -42,6 +43,10 @@ public interface FileInterface { Path baseRepoSuffix, GitResourceMap gitResourceMap, String branchName, boolean keepWorkingDirChanges) throws GitAPIException, IOException; + Mono computeGitStatus( + Path baseRepoSuffix, GitResourceMap gitResourceMap, String branchName, boolean keepWorkingDirChanges) + throws GitAPIException, IOException; + /** * This method will reconstruct the application from the repo * diff --git a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/git/handler/FSGitHandler.java b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/git/handler/FSGitHandler.java index dd86807c08..ffdcf87b10 100644 --- a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/git/handler/FSGitHandler.java +++ b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/git/handler/FSGitHandler.java @@ -160,6 +160,8 @@ public interface FSGitHandler { */ Mono getStatus(Path repoPath, String branchName, boolean keepWorkingDirChanges); + void populateModifiedEntities(GitStatusDTO response); + /** * This method merges source branch into destination branch for a git repository which is present on the partial * path provided. This assumes that the branch on which the merge will happen is already checked out diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/git/central/CentralGitServiceCEImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/git/central/CentralGitServiceCEImpl.java index e4058d83e5..0b1a009798 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/git/central/CentralGitServiceCEImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/git/central/CentralGitServiceCEImpl.java @@ -1675,8 +1675,10 @@ public class CentralGitServiceCEImpl implements CentralGitServiceCE { Mono lockHandledStatusMono = Mono.usingWhen( exportedArtifactJsonMono, artifactExchangeJson -> { - Mono prepareForStatus = - gitHandlingService.prepareChangesToBeCommitted(jsonTransformationDTO, artifactExchangeJson); + Mono statusMono = gitHandlingService + .computeGitStatus(jsonTransformationDTO, artifactExchangeJson) + .name("in-memory-status-computation") + .tap(Micrometer.observation(observationRegistry)); Mono fetchRemoteMono = Mono.just("ignored"); @@ -1694,8 +1696,31 @@ public class CentralGitServiceCEImpl implements CentralGitServiceCE { error.getMessage())))); } - return Mono.zip(prepareForStatus, fetchRemoteMono) - .then(Mono.defer(() -> gitHandlingService.getStatus(jsonTransformationDTO))) + return Mono.zip(statusMono, fetchRemoteMono) + .flatMap(tuple -> { + return gitHandlingService + .getBranchTrackingStatus(jsonTransformationDTO) + .map(branchTrackingStatus -> { + GitStatusDTO status = tuple.getT1(); + + if (branchTrackingStatus != null) { + status.setAheadCount(branchTrackingStatus.getAheadCount()); + status.setBehindCount(branchTrackingStatus.getBehindCount()); + status.setRemoteBranch(branchTrackingStatus.getRemoteTrackingBranch()); + + } else { + log.debug( + "Remote tracking details not present for branch: {}, repo: {}", + finalBranchName, + repoName); + status.setAheadCount(0); + status.setBehindCount(0); + status.setRemoteBranch("untracked"); + } + + return status; + }); + }) .onErrorResume(throwable -> { /* in case of any error, the global exception handler will release the lock diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/git/central/GitHandlingServiceCE.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/git/central/GitHandlingServiceCE.java index f3f155e400..f7df326090 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/git/central/GitHandlingServiceCE.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/git/central/GitHandlingServiceCE.java @@ -86,6 +86,9 @@ public interface GitHandlingServiceCE { Mono prepareChangesToBeCommitted( ArtifactJsonTransformationDTO jsonTransformationDTO, ArtifactExchangeJson artifactExchangeJson); + Mono computeGitStatus( + ArtifactJsonTransformationDTO jsonTransformationDTO, ArtifactExchangeJson artifactExchangeJson); + Mono> commitArtifact( Artifact branchedArtifact, CommitDTO commitDTO, ArtifactJsonTransformationDTO jsonTransformationDTO); diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/git/fs/GitFSServiceCEImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/git/fs/GitFSServiceCEImpl.java index 4ecd79d268..eccdcf1442 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/git/fs/GitFSServiceCEImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/git/fs/GitFSServiceCEImpl.java @@ -764,6 +764,19 @@ public class GitFSServiceCEImpl implements GitHandlingServiceCE { keepWorkingDirChanges -> fsGitHandler.getStatus(repoPath, refName, keepWorkingDirChanges)); } + public Mono computeGitStatus( + ArtifactJsonTransformationDTO jsonTransformationDTO, ArtifactExchangeJson artifactExchangeJson) { + String workspaceId = jsonTransformationDTO.getWorkspaceId(); + String baseArtifactId = jsonTransformationDTO.getBaseArtifactId(); + String repoName = jsonTransformationDTO.getRepoName(); + String branchName = jsonTransformationDTO.getRefName(); + + ArtifactType artifactType = jsonTransformationDTO.getArtifactType(); + GitArtifactHelper gitArtifactHelper = gitArtifactHelperResolver.getArtifactHelper(artifactType); + Path repoSuffix = gitArtifactHelper.getRepoSuffixPath(workspaceId, baseArtifactId, repoName); + return commonGitFileUtils.computeGitStatus(repoSuffix, artifactExchangeJson, branchName); + } + @Override public Mono createGitReference( ArtifactJsonTransformationDTO baseRefJsonTransformationDTO, diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/ce/CommonGitFileUtilsCE.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/ce/CommonGitFileUtilsCE.java index c339613ccb..e5e3fc29bc 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/ce/CommonGitFileUtilsCE.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/ce/CommonGitFileUtilsCE.java @@ -1,6 +1,7 @@ package com.appsmith.server.helpers.ce; import com.appsmith.external.constants.AnalyticsEvents; +import com.appsmith.external.dtos.GitStatusDTO; import com.appsmith.external.enums.FeatureFlagEnum; import com.appsmith.external.git.FileInterface; import com.appsmith.external.git.models.GitResourceIdentity; @@ -195,6 +196,18 @@ public class CommonGitFileUtilsCE { }); } + public Mono computeGitStatus( + Path baseRepoSuffix, ArtifactExchangeJson artifactExchangeJson, String branchName) { + GitResourceMap gitResourceMapFromDB = createGitResourceMap(artifactExchangeJson); + try { + return fileUtils + .computeGitStatus(baseRepoSuffix, gitResourceMapFromDB, branchName, true) + .subscribeOn(Schedulers.boundedElastic()); + } catch (IOException | GitAPIException exception) { + return Mono.error(exception); + } + } + public Mono saveArtifactToLocalRepoWithAnalytics( Path baseRepoSuffix, ArtifactExchangeJson artifactExchangeJson, String branchName) {