Protei/PromucFlow_constructor
2
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

feat: added commit changes (#37922)

Browse Source 
## Description

Fixes #37437 

## Automation

/ok-to-test tags="@tag.Git"

### 🔍 Cypress test results
<!-- This is an auto-generated comment: Cypress test results  -->
> [!TIP]
> 🟢 🟢 🟢 All cypress tests have passed! 🎉 🎉 🎉
> Workflow run:
<https://github.com/appsmithorg/appsmith/actions/runs/12152404326>
> Commit: 6dc1f5a35764f8dc602cb1b5a7a18c76be534e6e
> <a
href="https://internal.appsmith.com/app/cypress-dashboard/rundetails-65890b3c81d7400d08fa9ee5?branch=master&workflowId=12152404326&attempt=2"
target="_blank">Cypress dashboard</a>.
> Tags: `@tag.Git`
> Spec:
> <hr>Wed, 04 Dec 2024 04:58:02 UTC
<!-- end of auto-generated comment: Cypress test results  -->


## Communication
Should the DevRel and Marketing teams inform users about this change?
- [ ] Yes
- [ ] No


<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

## Release Notes

- **New Features**
- Introduced methods for acquiring and releasing Git locks, enhancing
the locking mechanism.
- Added `commitArtifact` method for committing artifacts in Git
operations.
- New method `publishArtifactPostCommit` for publishing artifacts after
a commit.
  
- **Improvements**
- Enhanced error handling and parameter naming consistency across
various Git-related services.
  
- **Refactor**
- Updated method signatures and added detailed documentation for clarity
and maintainability.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
Manish Kumar 2024-12-04 10:33:25 +05:30 committed by GitHub
parent a2c5caa819
commit 1078a03b23
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
10 changed files with 686 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
app/server/appsmith-server/src/main/java/com/appsmith/server/applications/git/GitApplicationHelperCEImpl.java
View File

@ -317,4 +317,9 @@ public class GitApplicationHelperCEImpl implements GitArtifactHelperCE<Applicati
newApplication.setGitApplicationMetadata(new GitArtifactMetadata());
return newApplication;
}
@Override
public Mono<Application> publishArtifactPostCommit(Artifact committedArtifact) {
return publishArtifact(committedArtifact, true);
}
}

51
app/server/appsmith-server/src/main/java/com/appsmith/server/git/GitRedisUtils.java
View File

@ -23,15 +23,20 @@ public class GitRedisUtils {
private final RedisUtils redisUtils;
private final ObservationRegistry observationRegistry;
public Mono<Boolean> addFileLock(String defaultApplicationId, String commandName, Boolean isRetryAllowed) {
/**
* Adds a baseArtifact id as a key in redis, the presence of this key represents a symbolic lock, essentially meaning that no new operations
* should be performed till this key remains present.
* @param baseArtifactId : base id of the artifact for which the key is getting added.
* @param commandName : Name of the operation which is trying to acquire the lock, this value will be added against the key
* @param isRetryAllowed : Boolean for whether retries for adding the value is allowed
* @return a boolean publisher for the added file locks
*/
public Mono<Boolean> addFileLock(String baseArtifactId, String commandName, Boolean isRetryAllowed) {
long numberOfRetries = Boolean.TRUE.equals(isRetryAllowed) ? MAX_RETRIES : 0L;
log.info(
"Git command {} is trying to acquire the lock for application id {}",
commandName,
defaultApplicationId);
log.info("Git command {} is trying to acquire the lock for application id {}", commandName, baseArtifactId);
return redisUtils
.addFileLock(defaultApplicationId, commandName)
.addFileLock(baseArtifactId, commandName)
.retryWhen(Retry.fixedDelay(numberOfRetries, RETRY_DELAY)
.onRetryExhaustedThrow((retryBackoffSpec, retrySignal) -> {
if (retrySignal.failure() instanceof AppsmithException) {
@ -54,4 +59,38 @@ public class GitRedisUtils {
.name(GitSpan.RELEASE_FILE_LOCK)
.tap(Micrometer.observation(observationRegistry));
}
/**
* This is a wrapper method for acquiring git lock, since multiple ops are used in sequence
* for a complete composite operation not all ops require to acquire the lock hence a dummy flag is sent back for
* operations in that is getting executed in between
* @param baseArtifactId : id of the base artifact for which ops would be locked
* @param isLockRequired : is lock really required or is it a proxy function
* @return : Boolean for whether the lock is acquired
*/
// TODO @Manish add artifactType reference in incoming prs.
public Mono<Boolean> acquireGitLock(String baseArtifactId, String commandName, boolean isLockRequired) {
if (!Boolean.TRUE.equals(isLockRequired)) {
return Mono.just(Boolean.TRUE);
}
return addFileLock(baseArtifactId, commandName);
}
/**
* This is a wrapper method for releasing git lock, since multiple ops are used in sequence
* for a complete composite operation not all ops require to acquire the lock hence a dummy flag is sent back for
* operations in that is getting executed in between
* @param baseArtifactId : id of the base artifact for which ops would be locked
* @param isLockRequired : is lock really required or is it a proxy function
* @return : Boolean for whether the lock is released
*/
// TODO @Manish add artifactType reference in incoming prs
public Mono<Boolean> releaseFileLock(String baseArtifactId, boolean isLockRequired) {
if (!Boolean.TRUE.equals(isLockRequired)) {
return Mono.just(Boolean.TRUE);
}
return releaseFileLock(baseArtifactId);
}
}

4
app/server/appsmith-server/src/main/java/com/appsmith/server/git/central/CentralGitServiceCE.java
View File

@ -1,5 +1,6 @@
package com.appsmith.server.git.central;
import com.appsmith.git.dto.CommitDTO;
import com.appsmith.server.constants.ArtifactType;
import com.appsmith.server.domains.Artifact;
import com.appsmith.server.dtos.ArtifactImportDTO;
@ -17,4 +18,7 @@ public interface CentralGitServiceCE {
String originHeader,
ArtifactType artifactType,
GitType gitType);
Mono<String> commitArtifact(
CommitDTO commitDTO, String branchedArtifactId, ArtifactType artifactType, GitType gitType);
}

10
app/server/appsmith-server/src/main/java/com/appsmith/server/git/central/CentralGitServiceCECompatibleImpl.java
View File

@ -2,6 +2,7 @@ package com.appsmith.server.git.central;
import com.appsmith.server.datasources.base.DatasourceService;
import com.appsmith.server.exports.internal.ExportService;
import com.appsmith.server.git.GitRedisUtils;
import com.appsmith.server.git.resolver.GitArtifactHelperResolver;
import com.appsmith.server.git.resolver.GitHandlingServiceResolver;
import com.appsmith.server.git.utils.GitAnalyticsUtils;
@ -12,6 +13,7 @@ import com.appsmith.server.plugins.base.PluginService;
import com.appsmith.server.services.UserDataService;
import com.appsmith.server.services.WorkspaceService;
import com.appsmith.server.solutions.DatasourcePermission;
import io.micrometer.observation.ObservationRegistry;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
@ -32,7 +34,9 @@ public class CentralGitServiceCECompatibleImpl extends CentralGitServiceCEImpl
WorkspaceService workspaceService,
PluginService pluginService,
ImportService importService,
ExportService exportService) {
ExportService exportService,
GitRedisUtils gitRedisUtils,
ObservationRegistry observationRegistry) {
super(
gitProfileUtils,
gitAnalyticsUtils,
@ -45,6 +49,8 @@ public class CentralGitServiceCECompatibleImpl extends CentralGitServiceCEImpl
workspaceService,
pluginService,
importService,
exportService);
exportService,
gitRedisUtils,
observationRegistry);
}
}

358
app/server/appsmith-server/src/main/java/com/appsmith/server/git/central/CentralGitServiceCEImpl.java
View File

@ -1,6 +1,7 @@
package com.appsmith.server.git.central;
import com.appsmith.external.constants.AnalyticsEvents;
import com.appsmith.external.git.constants.GitConstants;
import com.appsmith.external.models.Datasource;
import com.appsmith.external.models.DatasourceStorage;
import com.appsmith.git.dto.CommitDTO;
@ -24,6 +25,7 @@ import com.appsmith.server.dtos.GitConnectDTO;
import com.appsmith.server.exceptions.AppsmithError;
import com.appsmith.server.exceptions.AppsmithException;
import com.appsmith.server.exports.internal.ExportService;
import com.appsmith.server.git.GitRedisUtils;
import com.appsmith.server.git.dtos.ArtifactJsonTransformationDTO;
import com.appsmith.server.git.resolver.GitArtifactHelperResolver;
import com.appsmith.server.git.resolver.GitHandlingServiceResolver;
@ -36,6 +38,7 @@ import com.appsmith.server.services.GitArtifactHelper;
import com.appsmith.server.services.UserDataService;
import com.appsmith.server.services.WorkspaceService;
import com.appsmith.server.solutions.DatasourcePermission;
import io.micrometer.observation.ObservationRegistry;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.eclipse.jgit.api.errors.InvalidRemoteException;
@ -43,6 +46,7 @@ import org.eclipse.jgit.api.errors.TransportException;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import org.springframework.util.StringUtils;
import reactor.core.observability.micrometer.Micrometer;
import reactor.core.publisher.Mono;
import reactor.util.function.Tuple2;
@ -54,10 +58,15 @@ import java.util.Set;
import java.util.concurrent.TimeoutException;
import static com.appsmith.external.git.constants.ce.GitConstantsCE.DEFAULT_COMMIT_MESSAGE;
import static com.appsmith.external.git.constants.ce.GitConstantsCE.GIT_CONFIG_ERROR;
import static com.appsmith.external.git.constants.ce.GitConstantsCE.GIT_PROFILE_ERROR;
import static com.appsmith.external.git.constants.ce.GitSpanCE.OPS_COMMIT;
import static com.appsmith.external.helpers.AppsmithBeanUtils.copyNestedNonNullProperties;
import static com.appsmith.server.constants.FieldName.DEFAULT;
import static com.appsmith.server.constants.SerialiseArtifactObjective.VERSION_CONTROL;
import static java.lang.Boolean.FALSE;
import static java.lang.Boolean.TRUE;
import static org.springframework.util.StringUtils.hasText;
@Slf4j
@Service
@ -82,6 +91,9 @@ public class CentralGitServiceCEImpl implements CentralGitServiceCE {
private final ImportService importService;
private final ExportService exportService;
private final GitRedisUtils gitRedisUtils;
private final ObservationRegistry observationRegistry;
protected Mono<Boolean> isRepositoryLimitReachedForWorkspace(String workspaceId, Boolean isRepositoryPrivate) {
if (!isRepositoryPrivate) {
return Mono.just(FALSE);
@ -195,7 +207,7 @@ public class CentralGitServiceCEImpl implements CentralGitServiceCE {
ArtifactJsonTransformationDTO jsonMorphDTO = new ArtifactJsonTransformationDTO();
jsonMorphDTO.setWorkspaceId(workspaceId);
jsonMorphDTO.setArtifactId(artifact.getId());
jsonMorphDTO.setBaseArtifactId(artifact.getId());
jsonMorphDTO.setArtifactType(artifactType);
jsonMorphDTO.setRepoName(gitArtifactMetadata.getRepoName());
jsonMorphDTO.setRefType(RefType.BRANCH);
@ -274,7 +286,7 @@ public class CentralGitServiceCEImpl implements CentralGitServiceCE {
return gitHandlingService
.removeRepository(artifactJsonTransformationDTO)
.zipWith(gitArtifactHelper.deleteArtifact(artifactJsonTransformationDTO.getArtifactId()))
.zipWith(gitArtifactHelper.deleteArtifact(artifactJsonTransformationDTO.getBaseArtifactId()))
.map(Tuple2::getT2);
}
@ -443,7 +455,7 @@ public class CentralGitServiceCEImpl implements CentralGitServiceCE {
ArtifactJsonTransformationDTO jsonTransformationDTO =
new ArtifactJsonTransformationDTO();
jsonTransformationDTO.setWorkspaceId(artifact.getWorkspaceId());
jsonTransformationDTO.setArtifactId(artifact.getId());
jsonTransformationDTO.setBaseArtifactId(artifact.getId());
jsonTransformationDTO.setRepoName(repoName);
jsonTransformationDTO.setArtifactType(artifactType);
@ -468,7 +480,7 @@ public class CentralGitServiceCEImpl implements CentralGitServiceCE {
ArtifactJsonTransformationDTO jsonTransformationDTO = new ArtifactJsonTransformationDTO();
jsonTransformationDTO.setWorkspaceId(artifact.getWorkspaceId());
jsonTransformationDTO.setArtifactId(artifact.getId());
jsonTransformationDTO.setBaseArtifactId(artifact.getId());
jsonTransformationDTO.setRepoName(repoName);
jsonTransformationDTO.setArtifactType(artifactType);
@ -524,7 +536,7 @@ public class CentralGitServiceCEImpl implements CentralGitServiceCE {
.flatMap(artifact -> {
ArtifactJsonTransformationDTO jsonTransformationDTO = new ArtifactJsonTransformationDTO();
jsonTransformationDTO.setWorkspaceId(artifact.getWorkspaceId());
jsonTransformationDTO.setArtifactId(artifact.getId());
jsonTransformationDTO.setBaseArtifactId(artifact.getId());
jsonTransformationDTO.setArtifactType(artifactType);
jsonTransformationDTO.setRepoName(repoName);
@ -556,7 +568,7 @@ public class CentralGitServiceCEImpl implements CentralGitServiceCE {
commitDTO.setIsAmendCommit(FALSE);
commitDTO.setMessage(commitMessage);
return this.commitArtifact(baseArtifactId, commitDTO, artifactType, gitType)
return this.commitArtifact(commitDTO, artifact.getId(), artifactType, gitType)
.onErrorResume(error ->
// If the push fails remove all the cloned files from local repo
this.detachRemote(baseArtifactId, artifactType)
@ -592,13 +604,251 @@ public class CentralGitServiceCEImpl implements CentralGitServiceCE {
sink -> connectedArtifactMono.subscribe(sink::success, sink::error, null, sink.currentContext()));
}
/**
* TODO: commit artifact
* @return
*/
public Mono<? extends Artifact> commitArtifact(
String baseArtifactId, CommitDTO commitDTO, ArtifactType artifactType, GitType gitType) {
return null;
@Override
public Mono<String> commitArtifact(
CommitDTO commitDTO, String branchedArtifactId, ArtifactType artifactType, GitType gitType) {
return commitArtifact(commitDTO, branchedArtifactId, artifactType, gitType, TRUE);
}
public Mono<String> commitArtifact(
CommitDTO commitDTO,
String branchedArtifactId,
ArtifactType artifactType,
GitType gitType,
Boolean isFileLock) {
/*
1. Check if application exists and user have sufficient permissions
2. Check if branch name exists in git metadata
3. Save application to the existing local repo
4. Commit application : git add, git commit (Also check if git init required)
*/
String commitMessage = commitDTO.getMessage();
if (commitMessage == null || commitMessage.isEmpty()) {
commitDTO.setMessage(DEFAULT_COMMIT_MESSAGE + GitDefaultCommitMessage.CONNECT_FLOW.getReason());
}
GitArtifactHelper<?> gitArtifactHelper = gitArtifactHelperResolver.getArtifactHelper(artifactType);
AclPermission artifactEditPermission = gitArtifactHelper.getArtifactEditPermission();
Mono<Tuple2<? extends Artifact, ? extends Artifact>> baseAndBranchedArtifactMono = getBaseAndBranchedArtifacts(
branchedArtifactId, artifactType, artifactEditPermission)
.cache();
return baseAndBranchedArtifactMono.flatMap(artifactTuples -> {
Artifact baseArtifact = artifactTuples.getT1();
Artifact branchedArtifact = artifactTuples.getT2();
GitUser author = commitDTO.getAuthor();
Mono<GitUser> gitUserMono = Mono.justOrEmpty(author)
.flatMap(gitUser -> {
if (author == null
|| !StringUtils.hasText(author.getEmail())
|| !StringUtils.hasText(author.getName())) {
return getGitUserForArtifactId(baseArtifact.getId());
}
return Mono.just(gitUser);
})
.switchIfEmpty(getGitUserForArtifactId(baseArtifact.getId()));
return gitUserMono.flatMap(gitUser -> {
commitDTO.setAuthor(gitUser);
commitDTO.setCommitter(gitUser);
return commitArtifact(commitDTO, baseArtifact, branchedArtifact, gitType, isFileLock);
});
});
}
private Mono<String> commitArtifact(
CommitDTO commitDTO,
Artifact baseArtifact,
Artifact branchedArtifact,
GitType gitType,
boolean isFileLock) {
String commitMessage = commitDTO.getMessage();
if (commitMessage == null || commitMessage.isEmpty()) {
commitDTO.setMessage(DEFAULT_COMMIT_MESSAGE + GitDefaultCommitMessage.CONNECT_FLOW.getReason());
}
GitUser author = commitDTO.getAuthor();
if (author == null || !StringUtils.hasText(author.getEmail()) || !StringUtils.hasText(author.getName())) {
String errorMessage = "Unable to find git author configuration for logged-in user. You can set "
+ "up a git profile from the user profile section.";
return gitAnalyticsUtils
.addAnalyticsForGitOperation(
AnalyticsEvents.GIT_COMMIT,
branchedArtifact,
AppsmithError.INVALID_GIT_CONFIGURATION.getErrorType(),
AppsmithError.INVALID_GIT_CONFIGURATION.getMessage(errorMessage),
branchedArtifact.getGitArtifactMetadata().getIsRepoPrivate())
.then(Mono.error(new AppsmithException(AppsmithError.INVALID_GIT_CONFIGURATION, errorMessage)));
}
boolean isSystemGenerated = commitDTO.getMessage().contains(DEFAULT_COMMIT_MESSAGE);
GitArtifactHelper<?> gitArtifactHelper =
gitArtifactHelperResolver.getArtifactHelper(baseArtifact.getArtifactType());
GitHandlingService gitHandlingService = gitHandlingServiceResolver.getGitHandlingService(gitType);
GitArtifactMetadata baseGitMetadata = baseArtifact.getGitArtifactMetadata();
GitArtifactMetadata branchedGitMetadata = branchedArtifact.getGitArtifactMetadata();
if (isBaseGitMetadataInvalid(baseGitMetadata, gitType)) {
return Mono.error(new AppsmithException(AppsmithError.INVALID_GIT_CONFIGURATION, GIT_CONFIG_ERROR));
}
if (branchedGitMetadata == null) {
return Mono.error(new AppsmithException(AppsmithError.INVALID_GIT_CONFIGURATION, GIT_CONFIG_ERROR));
}
final String branchName = branchedGitMetadata.getBranchName();
if (!hasText(branchName)) {
return Mono.error(new AppsmithException(AppsmithError.INVALID_PARAMETER, FieldName.BRANCH_NAME));
}
Mono<Boolean> isBranchProtectedMono = gitPrivateRepoHelper.isBranchProtected(baseGitMetadata, branchName);
Mono<String> commitMono = isBranchProtectedMono
.flatMap(isBranchProtected -> {
if (!TRUE.equals(isBranchProtected)) {
return gitRedisUtils.acquireGitLock(
baseGitMetadata.getDefaultArtifactId(),
GitConstants.GitCommandConstants.COMMIT,
isFileLock);
}
return Mono.error(new AppsmithException(
AppsmithError.GIT_ACTION_FAILED,
"commit",
"Cannot commit to protected branch " + branchName));
})
.flatMap(fileLocked -> {
// Check if the repo is public for current artifact and if the user have changed the access after
// the connection
return gitHandlingService.isRepoPrivate(baseGitMetadata).flatMap(isPrivate -> {
// Check the repo limit if the visibility status is updated, or it is private
// TODO: split both of these conditions @Manish
if (isPrivate.equals(baseGitMetadata.getIsRepoPrivate() && !Boolean.TRUE.equals(isPrivate))) {
return Mono.just(baseArtifact);
}
baseGitMetadata.setIsRepoPrivate(isPrivate);
baseArtifact.setGitArtifactMetadata(baseGitMetadata);
/**
* A separate GitAuth object has been created in which the private key for
* authentication is held. It's done to avoid getting the encrypted value back
* for private key after mongo save.
*
* When an object having an encrypted attribute is saved, the response is still encrypted.
* The value in db would be corrupted if it's saved again,
* as it would encrypt and already encrypted field
* Private key is using encrypted annotation, which means that it's encrypted before
* being persisted in the db. When it's fetched from db, the listener decrypts it.
*/
GitAuth copiedGitAuth = new GitAuth();
copyNestedNonNullProperties(baseGitMetadata.getGitAuth(), copiedGitAuth);
return gitArtifactHelper
.saveArtifact(baseArtifact)
.map(artifact -> {
baseArtifact.getGitArtifactMetadata().setGitAuth(copiedGitAuth);
return artifact;
})
.then(Mono.defer(
() -> gitArtifactHelper.isPrivateRepoLimitReached(baseArtifact, false)));
});
})
.flatMap(artifact -> {
String errorEntity = "";
if (!StringUtils.hasText(branchedGitMetadata.getBranchName())) {
errorEntity = "branch name";
} else if (!StringUtils.hasText(branchedGitMetadata.getDefaultArtifactId())) {
errorEntity = "default artifact";
} else if (!StringUtils.hasText(branchedGitMetadata.getRepoName())) {
errorEntity = "repository name";
}
if (!errorEntity.isEmpty()) {
return Mono.error(new AppsmithException(
AppsmithError.INVALID_GIT_CONFIGURATION, "Unable to find " + errorEntity));
}
return exportService.exportByArtifactId(
branchedArtifact.getId(), VERSION_CONTROL, branchedArtifact.getArtifactType());
})
.flatMap(artifactExchangeJson -> {
ArtifactJsonTransformationDTO jsonTransformationDTO = new ArtifactJsonTransformationDTO();
jsonTransformationDTO.setRefType(RefType.BRANCH);
jsonTransformationDTO.setWorkspaceId(baseArtifact.getWorkspaceId());
jsonTransformationDTO.setBaseArtifactId(baseArtifact.getId());
jsonTransformationDTO.setRepoName(
branchedArtifact.getGitArtifactMetadata().getRepoName());
jsonTransformationDTO.setArtifactType(artifactExchangeJson.getArtifactJsonType());
jsonTransformationDTO.setRefName(
branchedArtifact.getGitArtifactMetadata().getBranchName());
return gitHandlingService
.prepareChangesToBeCommitted(jsonTransformationDTO, artifactExchangeJson)
.then(updateArtifactWithGitMetadataGivenPermission(branchedArtifact, branchedGitMetadata));
})
.flatMap(updatedBranchedArtifact -> {
GitArtifactMetadata gitArtifactMetadata = updatedBranchedArtifact.getGitArtifactMetadata();
ArtifactJsonTransformationDTO jsonTransformationDTO = new ArtifactJsonTransformationDTO();
jsonTransformationDTO.setRefType(RefType.BRANCH);
jsonTransformationDTO.setWorkspaceId(updatedBranchedArtifact.getWorkspaceId());
jsonTransformationDTO.setBaseArtifactId(gitArtifactMetadata.getDefaultArtifactId());
jsonTransformationDTO.setRepoName(gitArtifactMetadata.getRepoName());
jsonTransformationDTO.setArtifactType(branchedArtifact.getArtifactType());
jsonTransformationDTO.setRefName(gitArtifactMetadata.getBranchName());
return gitHandlingService
.commitArtifact(updatedBranchedArtifact, commitDTO, jsonTransformationDTO)
.onErrorResume(error -> {
return gitAnalyticsUtils
.addAnalyticsForGitOperation(
AnalyticsEvents.GIT_COMMIT,
updatedBranchedArtifact,
error.getClass().getName(),
error.getMessage(),
gitArtifactMetadata.getIsRepoPrivate())
.then(Mono.error(new AppsmithException(
AppsmithError.GIT_ACTION_FAILED, "commit", error.getMessage())));
});
})
.flatMap(tuple2 -> {
return Mono.zip(
Mono.just(tuple2.getT2()), gitArtifactHelper.publishArtifactPostCommit(tuple2.getT1()));
})
.flatMap(tuple -> {
String status = tuple.getT1();
Artifact artifactFromBranch = tuple.getT2();
Mono<Boolean> releaseFileLockMono = gitRedisUtils.releaseFileLock(
artifactFromBranch.getGitArtifactMetadata().getDefaultArtifactId(), isFileLock);
Mono<? extends Artifact> updatedArtifactMono =
gitArtifactHelper.updateArtifactWithSchemaVersions(artifactFromBranch);
return Mono.zip(updatedArtifactMono, releaseFileLockMono)
.then(gitAnalyticsUtils.addAnalyticsForGitOperation(
AnalyticsEvents.GIT_COMMIT,
artifactFromBranch,
"",
"",
artifactFromBranch.getGitArtifactMetadata().getIsRepoPrivate(),
isSystemGenerated))
.thenReturn(status)
.name(OPS_COMMIT)
.tap(Micrometer.observation(observationRegistry));
});
return Mono.create(sink -> {
commitMono.subscribe(sink::success, sink::error, null, sink.currentContext());
});
}
/**
@ -618,4 +868,86 @@ public class CentralGitServiceCEImpl implements CentralGitServiceCE {
.getGitHandlingService(gitType)
.isGitAuthInvalid(gitArtifactMetadata.getGitAuth());
}
/**
* Returns baseArtifact and branchedArtifact
* This operation is quite frequently used, hence providing the right set
*
* @param branchedArtifactId : id of the branchedArtifactId
* @param artifactPermission : permission required for getting artifact.
* @return : A tuple of Artifacts
*/
protected Mono<Tuple2<? extends Artifact, ? extends Artifact>> getBaseAndBranchedArtifacts(
String branchedArtifactId, ArtifactType artifactType, AclPermission artifactPermission) {
if (!hasText(branchedArtifactId)) {
return Mono.error(new AppsmithException(AppsmithError.INVALID_PARAMETER, FieldName.ID));
}
GitArtifactHelper<?> artifactGitHelper = gitArtifactHelperResolver.getArtifactHelper(artifactType);
Mono<? extends Artifact> branchedArtifactMono = artifactGitHelper
.getArtifactById(branchedArtifactId, artifactPermission)
.cache();
return branchedArtifactMono.flatMap(branchedArtifact -> {
GitArtifactMetadata branchedMetadata = branchedArtifact.getGitArtifactMetadata();
if (branchedMetadata == null || !hasText(branchedMetadata.getDefaultArtifactId())) {
return Mono.error(new AppsmithException(AppsmithError.INVALID_GIT_CONFIGURATION, GIT_CONFIG_ERROR));
}
String baseArtifactId = branchedMetadata.getDefaultArtifactId();
Mono<? extends Artifact> baseArtifactMono = Mono.just(branchedArtifact);
if (!baseArtifactId.equals(branchedArtifactId)) {
baseArtifactMono = artifactGitHelper.getArtifactById(baseArtifactId, artifactPermission);
}
return baseArtifactMono.zipWith(branchedArtifactMono);
});
}
protected Mono<Tuple2<? extends Artifact, ? extends Artifact>> getBaseAndBranchedArtifacts(
String branchedArtifactId, ArtifactType artifactType) {
GitArtifactHelper<?> gitArtifactHelper = gitArtifactHelperResolver.getArtifactHelper(artifactType);
AclPermission artifactPermission = gitArtifactHelper.getArtifactEditPermission();
return getBaseAndBranchedArtifacts(branchedArtifactId, artifactType, artifactPermission);
}
private Mono<GitUser> getGitUserForArtifactId(String baseArtifactId) {
Mono<UserData> currentUserMono = userDataService
.getForCurrentUser()
.filter(userData -> !CollectionUtils.isEmpty(userData.getGitProfiles()))
.switchIfEmpty(
Mono.error(new AppsmithException(AppsmithError.INVALID_GIT_CONFIGURATION, GIT_PROFILE_ERROR)));
return currentUserMono.map(userData -> {
GitProfile profile = userData.getGitProfileByKey(baseArtifactId);
if (profile == null
|| Boolean.TRUE.equals(profile.getUseGlobalProfile())
|| !StringUtils.hasText(profile.getAuthorName())) {
profile = userData.getGitProfileByKey(DEFAULT);
}
GitUser gitUser = new GitUser();
gitUser.setName(profile.getAuthorName());
gitUser.setEmail(profile.getAuthorEmail());
return gitUser;
});
}
private Mono<? extends Artifact> updateArtifactWithGitMetadataGivenPermission(
Artifact artifact, GitArtifactMetadata gitMetadata) {
if (gitMetadata == null) {
return Mono.error(
new AppsmithException(AppsmithError.INVALID_PARAMETER, "Git metadata values cannot be null"));
}
artifact.setGitArtifactMetadata(gitMetadata);
// For default application we expect a GitAuth to be a part of gitMetadata. We are using save method to leverage
// @Encrypted annotation used for private SSH keys
// applicationService.save sets the transient fields so no need to set it again from this method
return gitArtifactHelperResolver
.getArtifactHelper(artifact.getArtifactType())
.saveArtifact(artifact);
}
}

10
app/server/appsmith-server/src/main/java/com/appsmith/server/git/central/CentralGitServiceImpl.java
View File

@ -2,6 +2,7 @@ package com.appsmith.server.git.central;
import com.appsmith.server.datasources.base.DatasourceService;
import com.appsmith.server.exports.internal.ExportService;
import com.appsmith.server.git.GitRedisUtils;
import com.appsmith.server.git.resolver.GitArtifactHelperResolver;
import com.appsmith.server.git.resolver.GitHandlingServiceResolver;
import com.appsmith.server.git.utils.GitAnalyticsUtils;
@ -12,6 +13,7 @@ import com.appsmith.server.plugins.base.PluginService;
import com.appsmith.server.services.UserDataService;
import com.appsmith.server.services.WorkspaceService;
import com.appsmith.server.solutions.DatasourcePermission;
import io.micrometer.observation.ObservationRegistry;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
@ -31,7 +33,9 @@ public class CentralGitServiceImpl extends CentralGitServiceCECompatibleImpl imp
WorkspaceService workspaceService,
PluginService pluginService,
ImportService importService,
ExportService exportService) {
ExportService exportService,
GitRedisUtils gitRedisUtils,
ObservationRegistry observationRegistry) {
super(
gitProfileUtils,
gitAnalyticsUtils,
@ -44,6 +48,8 @@ public class CentralGitServiceImpl extends CentralGitServiceCECompatibleImpl imp
workspaceService,
pluginService,
importService,
exportService);
exportService,
gitRedisUtils,
observationRegistry);
}
}

9
app/server/appsmith-server/src/main/java/com/appsmith/server/git/central/GitHandlingServiceCE.java
View File

@ -8,6 +8,7 @@ import com.appsmith.server.dtos.ArtifactExchangeJson;
import com.appsmith.server.dtos.GitConnectDTO;
import com.appsmith.server.git.dtos.ArtifactJsonTransformationDTO;
import reactor.core.publisher.Mono;
import reactor.util.function.Tuple2;
import java.util.Set;
@ -19,6 +20,8 @@ public interface GitHandlingServiceCE {
Mono<Boolean> isRepoPrivate(GitConnectDTO gitConnectDTO);
Mono<Boolean> isRepoPrivate(GitArtifactMetadata gitArtifactMetadata);
// TODO: modify git auth class for native implementation
Mono<GitAuth> getGitAuthForUser();
@ -44,4 +47,10 @@ public interface GitHandlingServiceCE {
String originHeader);
Mono<String> createFirstCommit(ArtifactJsonTransformationDTO jsonTransformationDTO, CommitDTO commitDTO);
Mono<Boolean> prepareChangesToBeCommitted(
ArtifactJsonTransformationDTO jsonTransformationDTO, ArtifactExchangeJson artifactExchangeJson);
Mono<Tuple2<? extends Artifact, String>> commitArtifact(
Artifact branchedArtifact, CommitDTO commitDTO, ArtifactJsonTransformationDTO jsonTransformationDTO);
}

2
app/server/appsmith-server/src/main/java/com/appsmith/server/git/dtos/ArtifactJsonTransformationDTO.java
View File

@ -15,7 +15,7 @@ public class ArtifactJsonTransformationDTO {
String workspaceId;
String artifactId;
String baseArtifactId;
String repoName;

265
app/server/appsmith-server/src/main/java/com/appsmith/server/git/fs/GitFSServiceCEImpl.java
View File

@ -1,10 +1,13 @@
package com.appsmith.server.git.fs;
import com.appsmith.external.constants.AnalyticsEvents;
import com.appsmith.external.git.constants.GitConstants;
import com.appsmith.external.git.constants.GitSpan;
import com.appsmith.external.git.handler.FSGitHandler;
import com.appsmith.git.dto.CommitDTO;
import com.appsmith.server.acl.AclPermission;
import com.appsmith.server.configurations.EmailConfig;
import com.appsmith.server.constants.ArtifactType;
import com.appsmith.server.datasources.base.DatasourceService;
import com.appsmith.server.domains.Artifact;
import com.appsmith.server.domains.GitArtifactMetadata;
@ -38,13 +41,16 @@ import com.appsmith.server.solutions.DatasourcePermission;
import io.micrometer.observation.ObservationRegistry;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.eclipse.jgit.api.errors.EmptyCommitException;
import org.eclipse.jgit.api.errors.InvalidRemoteException;
import org.eclipse.jgit.api.errors.TransportException;
import org.eclipse.jgit.errors.RepositoryNotFoundException;
import org.springframework.stereotype.Service;
import org.springframework.transaction.reactive.TransactionalOperator;
import org.springframework.util.StringUtils;
import reactor.core.observability.micrometer.Micrometer;
import reactor.core.publisher.Mono;
import reactor.util.function.Tuple2;
import java.io.IOException;
import java.nio.file.Path;
@ -52,6 +58,9 @@ import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.TimeoutException;
import static com.appsmith.external.git.constants.ce.GitConstantsCE.EMPTY_COMMIT_ERROR_MESSAGE;
import static com.appsmith.external.git.constants.ce.GitConstantsCE.GIT_CONFIG_ERROR;
@Slf4j
@Service
@RequiredArgsConstructor
@ -140,7 +149,16 @@ public class GitFSServiceCEImpl implements GitHandlingServiceCE {
@Override
public Mono<Boolean> isRepoPrivate(GitConnectDTO gitConnectDTO) {
return GitUtils.isRepoPrivate(GitUtils.convertSshUrlToBrowserSupportedUrl(gitConnectDTO.getRemoteUrl()));
return isRepoPrivate(gitConnectDTO.getRemoteUrl());
}
@Override
public Mono<Boolean> isRepoPrivate(GitArtifactMetadata gitArtifactMetadata) {
return isRepoPrivate(gitArtifactMetadata.getRemoteUrl());
}
private Mono<Boolean> isRepoPrivate(String remoteUrl) {
return GitUtils.isRepoPrivate(GitUtils.convertSshUrlToBrowserSupportedUrl(remoteUrl));
}
@Override
@ -213,7 +231,7 @@ public class GitFSServiceCEImpl implements GitHandlingServiceCE {
ArtifactJsonTransformationDTO artifactJsonTransformationDTO) {
return commonGitFileUtils.reconstructArtifactExchangeJsonFromGitRepoWithAnalytics(
artifactJsonTransformationDTO.getWorkspaceId(),
artifactJsonTransformationDTO.getArtifactId(),
artifactJsonTransformationDTO.getBaseArtifactId(),
artifactJsonTransformationDTO.getRepoName(),
artifactJsonTransformationDTO.getRefName(),
artifactJsonTransformationDTO.getArtifactType());
@ -225,7 +243,7 @@ public class GitFSServiceCEImpl implements GitHandlingServiceCE {
gitArtifactHelperResolver.getArtifactHelper(artifactJsonTransformationDTO.getArtifactType());
Path repoSuffix = gitArtifactHelper.getRepoSuffixPath(
artifactJsonTransformationDTO.getWorkspaceId(),
artifactJsonTransformationDTO.getArtifactId(),
artifactJsonTransformationDTO.getBaseArtifactId(),
artifactJsonTransformationDTO.getRepoName());
return commonGitFileUtils.deleteLocalRepo(repoSuffix);
}
@ -236,7 +254,7 @@ public class GitFSServiceCEImpl implements GitHandlingServiceCE {
gitArtifactHelperResolver.getArtifactHelper(artifactJsonTransformationDTO.getArtifactType());
Path repoSuffix = gitArtifactHelper.getRepoSuffixPath(
artifactJsonTransformationDTO.getWorkspaceId(),
artifactJsonTransformationDTO.getArtifactId(),
artifactJsonTransformationDTO.getBaseArtifactId(),
artifactJsonTransformationDTO.getRepoName());
try {
@ -257,7 +275,7 @@ public class GitFSServiceCEImpl implements GitHandlingServiceCE {
gitArtifactHelperResolver.getArtifactHelper(jsonTransformationDTO.getArtifactType());
Path readmePath = gitArtifactHelper.getRepoSuffixPath(
jsonTransformationDTO.getWorkspaceId(),
jsonTransformationDTO.getArtifactId(),
jsonTransformationDTO.getBaseArtifactId(),
jsonTransformationDTO.getRepoName());
try {
return gitArtifactHelper
@ -275,7 +293,7 @@ public class GitFSServiceCEImpl implements GitHandlingServiceCE {
gitArtifactHelperResolver.getArtifactHelper(jsonTransformationDTO.getArtifactType());
Path repoSuffix = gitArtifactHelper.getRepoSuffixPath(
jsonTransformationDTO.getWorkspaceId(),
jsonTransformationDTO.getArtifactId(),
jsonTransformationDTO.getBaseArtifactId(),
jsonTransformationDTO.getRepoName());
return fsGitHandler.commitArtifact(
@ -286,4 +304,239 @@ public class GitFSServiceCEImpl implements GitHandlingServiceCE {
true,
commitDTO.getIsAmendCommit());
}
@Override
public Mono<Boolean> prepareChangesToBeCommitted(
ArtifactJsonTransformationDTO jsonTransformationDTO, ArtifactExchangeJson artifactExchangeJson) {
String workspaceId = jsonTransformationDTO.getWorkspaceId();
String baseArtifactId = jsonTransformationDTO.getBaseArtifactId();
String repoName = jsonTransformationDTO.getRepoName();
String branchName = jsonTransformationDTO.getRefName();
ArtifactType artifactType = jsonTransformationDTO.getArtifactType();
GitArtifactHelper<?> gitArtifactHelper = gitArtifactHelperResolver.getArtifactHelper(artifactType);
Path repoSuffix = gitArtifactHelper.getRepoSuffixPath(workspaceId, baseArtifactId, repoName);
return commonGitFileUtils
.saveArtifactToLocalRepoWithAnalytics(repoSuffix, artifactExchangeJson, branchName)
.map(ignore -> Boolean.TRUE)
.onErrorResume(e -> {
log.error("Error in commit flow: ", e);
if (e instanceof RepositoryNotFoundException) {
return Mono.error(new AppsmithException(AppsmithError.REPOSITORY_NOT_FOUND, baseArtifactId));
} else if (e instanceof AppsmithException) {
return Mono.error(e);
}
return Mono.error(new AppsmithException(AppsmithError.GIT_FILE_SYSTEM_ERROR, e.getMessage()));
});
}
@Override
public Mono<Tuple2<? extends Artifact, String>> commitArtifact(
Artifact branchedArtifact, CommitDTO commitDTO, ArtifactJsonTransformationDTO jsonTransformationDTO) {
String workspaceId = jsonTransformationDTO.getWorkspaceId();
String baseArtifactId = jsonTransformationDTO.getBaseArtifactId();
String repoName = jsonTransformationDTO.getRepoName();
ArtifactType artifactType = jsonTransformationDTO.getArtifactType();
GitArtifactHelper<?> gitArtifactHelper = gitArtifactHelperResolver.getArtifactHelper(artifactType);
Path repoSuffix = gitArtifactHelper.getRepoSuffixPath(workspaceId, baseArtifactId, repoName);
StringBuilder result = new StringBuilder();
result.append("Commit Result : ");
Mono<String> gitCommitMono = fsGitHandler
.commitArtifact(
repoSuffix,
commitDTO.getMessage(),
commitDTO.getAuthor().getName(),
commitDTO.getAuthor().getEmail(),
true,
false)
.onErrorResume(error -> {
if (error instanceof EmptyCommitException) {
return Mono.just(EMPTY_COMMIT_ERROR_MESSAGE);
}
return Mono.error(error);
});
return Mono.zip(gitCommitMono, gitArtifactHelper.getArtifactById(branchedArtifact.getId(), null))
.flatMap(tuple -> {
String commitStatus = tuple.getT1();
result.append(commitStatus);
result.append(".\nPush Result : ");
return Mono.zip(
Mono.just(tuple.getT2()),
pushArtifact(tuple.getT2(), false)
.map(pushResult -> result.append(pushResult).toString()));
});
}
/**
* Used for pushing commits present in the given branched artifact.
* @param branchedArtifactId : id of the branched artifact.
* @param artifactType : type of the artifact
* @return : returns a string which has details of operations
*/
public Mono<String> pushArtifact(String branchedArtifactId, ArtifactType artifactType) {
GitArtifactHelper<?> gitArtifactHelper = gitArtifactHelperResolver.getArtifactHelper(artifactType);
AclPermission artifactEditPermission = gitArtifactHelper.getArtifactEditPermission();
return gitArtifactHelper
.getArtifactById(branchedArtifactId, artifactEditPermission)
.flatMap(branchedArtifact -> pushArtifact(branchedArtifact, true));
}
/**
* Push flow for dehydrated apps
*
* @param branchedArtifact application which needs to be pushed to remote repo
* @return Success message
*/
protected Mono<String> pushArtifact(Artifact branchedArtifact, boolean isFileLock) {
GitArtifactHelper<?> gitArtifactHelper =
gitArtifactHelperResolver.getArtifactHelper(branchedArtifact.getArtifactType());
Mono<GitArtifactMetadata> gitArtifactMetadataMono = Mono.just(branchedArtifact.getGitArtifactMetadata());
if (!branchedArtifact
.getId()
.equals(branchedArtifact.getGitArtifactMetadata().getDefaultArtifactId())) {
gitArtifactMetadataMono = gitArtifactHelper
.getArtifactById(branchedArtifact.getGitArtifactMetadata().getDefaultArtifactId(), null)
.map(baseArtifact -> {
branchedArtifact
.getGitArtifactMetadata()
.setGitAuth(
baseArtifact.getGitArtifactMetadata().getGitAuth());
return branchedArtifact.getGitArtifactMetadata();
});
}
// Make sure that ssh Key is unEncrypted for the use.
Mono<String> gitPushResult = gitArtifactMetadataMono
.flatMap(gitMetadata -> {
return gitRedisUtils
.acquireGitLock(
gitMetadata.getDefaultArtifactId(),
GitConstants.GitCommandConstants.PUSH,
isFileLock)
.thenReturn(branchedArtifact);
})
.flatMap(artifact -> {
GitArtifactMetadata gitData = artifact.getGitArtifactMetadata();
if (gitData == null
|| !StringUtils.hasText(gitData.getBranchName())
|| !StringUtils.hasText(gitData.getDefaultArtifactId())
|| !StringUtils.hasText(gitData.getGitAuth().getPrivateKey())) {
return Mono.error(
new AppsmithException(AppsmithError.INVALID_GIT_CONFIGURATION, GIT_CONFIG_ERROR));
}
Path baseRepoSuffix = gitArtifactHelper.getRepoSuffixPath(
artifact.getWorkspaceId(), gitData.getDefaultArtifactId(), gitData.getRepoName());
GitAuth gitAuth = gitData.getGitAuth();
return fsGitHandler
.checkoutToBranch(
baseRepoSuffix,
artifact.getGitArtifactMetadata().getBranchName())
.then(Mono.defer(() -> fsGitHandler
.pushApplication(
baseRepoSuffix,
gitData.getRemoteUrl(),
gitAuth.getPublicKey(),
gitAuth.getPrivateKey(),
gitData.getBranchName())
.zipWith(Mono.just(artifact))))
.onErrorResume(error -> gitAnalyticsUtils
.addAnalyticsForGitOperation(
AnalyticsEvents.GIT_PUSH,
artifact,
error.getClass().getName(),
error.getMessage(),
artifact.getGitArtifactMetadata().getIsRepoPrivate())
.flatMap(application1 -> {
if (error instanceof TransportException) {
return Mono.error(
new AppsmithException(AppsmithError.INVALID_GIT_SSH_CONFIGURATION));
}
return Mono.error(new AppsmithException(
AppsmithError.GIT_ACTION_FAILED, "push", error.getMessage()));
}));
})
.flatMap(tuple -> {
String pushResult = tuple.getT1();
Artifact artifact = tuple.getT2();
return pushArtifactErrorRecovery(pushResult, artifact).zipWith(Mono.just(artifact));
})
// Add BE analytics
.flatMap(tuple2 -> {
String pushStatus = tuple2.getT1();
Artifact artifact = tuple2.getT2();
Mono<Boolean> fileLockReleasedMono = Mono.just(Boolean.TRUE).flatMap(flag -> {
if (!Boolean.TRUE.equals(isFileLock)) {
return Mono.just(flag);
}
return Mono.defer(() -> releaseFileLock(
artifact.getGitArtifactMetadata().getDefaultArtifactId()));
});
return pushArtifactErrorRecovery(pushStatus, artifact)
.then(fileLockReleasedMono)
.then(gitAnalyticsUtils.addAnalyticsForGitOperation(
AnalyticsEvents.GIT_PUSH,
artifact,
artifact.getGitArtifactMetadata().getIsRepoPrivate()))
.thenReturn(pushStatus);
})
.name(GitSpan.OPS_PUSH)
.tap(Micrometer.observation(observationRegistry));
return Mono.create(sink -> gitPushResult.subscribe(sink::success, sink::error, null, sink.currentContext()));
}
/**
* This method is used to recover from the errors that can occur during the push operation
* Mostly happens when the remote branch is protected or any specific rules in place on the branch.
* Since the users will be in a bad state where the changes are committed locally, but they are
* not able to push them changes or revert the changes either.
* 1. Push rejected due to branch protection rules on remote, reset hard prev commit
*
* @param pushResult status of git push operation
* @param artifact artifact data to be used for analytics
* @return status of the git push flow
*/
private Mono<String> pushArtifactErrorRecovery(String pushResult, Artifact artifact) {
GitArtifactMetadata gitMetadata = artifact.getGitArtifactMetadata();
GitArtifactHelper<?> gitArtifactHelper =
gitArtifactHelperResolver.getArtifactHelper(artifact.getArtifactType());
if (pushResult.contains("REJECTED_NONFASTFORWARD")) {
return gitAnalyticsUtils
.addAnalyticsForGitOperation(
AnalyticsEvents.GIT_PUSH,
artifact,
AppsmithError.GIT_UPSTREAM_CHANGES.getErrorType(),
AppsmithError.GIT_UPSTREAM_CHANGES.getMessage(),
gitMetadata.getIsRepoPrivate())
.flatMap(application1 -> Mono.error(new AppsmithException(AppsmithError.GIT_UPSTREAM_CHANGES)));
} else if (pushResult.contains("REJECTED_OTHERREASON") || pushResult.contains("pre-receive hook declined")) {
Path path = gitArtifactHelper.getRepoSuffixPath(
artifact.getWorkspaceId(), gitMetadata.getDefaultArtifactId(), gitMetadata.getRepoName());
return fsGitHandler
.resetHard(path, gitMetadata.getBranchName())
.then(Mono.error(new AppsmithException(
AppsmithError.GIT_ACTION_FAILED,
"push",
"Unable to push changes as pre-receive hook declined. Please make sure that you don't have any rules enabled on the branch "
+ gitMetadata.getBranchName())));
}
return Mono.just(pushResult);
}
}

2
app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/GitArtifactHelperCE.java
View File

@ -66,4 +66,6 @@ public interface GitArtifactHelperCE<T extends Artifact> {
Boolean isContextInArtifactEmpty(ArtifactExchangeJson artifactExchangeJson);
T getNewArtifact(String workspaceId, String repoName);
Mono<T> publishArtifactPostCommit(Artifact committedArtifact);
}