From 11a5a963d24dc9523301ed8ef64d028419b577df Mon Sep 17 00:00:00 2001 From: Subhrashis Das Date: Mon, 21 Jul 2025 14:11:34 +0530 Subject: [PATCH] feat: add git route aspect for branch handling (#41097) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Description > [!TIP] > _Add a TL;DR when the description is longer than 500 words or extremely technical (helps the content, marketing, and DevRel team)._ > > _Please also include relevant motivation and context. List any dependencies that are required for this change. Add links to Notion, Figma or any other documents that might be relevant to the PR._ Fixes #`Issue Number` _or_ Fixes `Issue URL` > [!WARNING] > _If no issue exists, please create an issue first, and check with the maintainers if the issue is valid._ ## Automation /ok-to-test tags="@tag.All" ### :mag: Cypress test results > [!TIP] > 🟢 🟢 🟢 All cypress tests have passed! 🎉 🎉 🎉 > Workflow run: > Commit: f8257de8135f4243309143396eca2a81bdb6f2a3 > Cypress dashboard. > Tags: `@tag.All` > Spec: >
Thu, 17 Jul 2025 12:14:40 UTC ## Communication Should the DevRel and Marketing teams inform users about this change? - [ ] Yes - [x] No ## Summary by CodeRabbit * **New Features** * Introduced a new annotation to streamline and secure Git-related operations in application APIs. * Added a robust workflow for handling Git operations with enhanced concurrency control and error handling. * Enabled in-memory Git storage mode for improved performance in certain environments. * Added support for executing Git operations via shell scripts, including branch merging and repository management. * **Improvements** * Enhanced configuration flexibility for Git storage and Redis integration. * Improved error reporting with new, descriptive Git-related error messages. * Broadened environment file ignore patterns for better environment management. * **Bug Fixes** * Improved handling of private key formats for Git authentication. * **Documentation** * Added detailed documentation and flow diagrams for new Git operation workflows. * **Chores** * Updated build and test configurations to align with new Git storage paths. * Deprecated and bypassed certain Redis operations when using in-memory Git storage. * **Tests** * Removed several outdated or redundant test cases related to auto-commit and Git serialization. --- Dockerfile | 9 +- app/server/.gitignore | 2 +- app/server/appsmith-git/pom.xml | 9 + .../git/configurations/GitServiceConfig.java | 6 +- .../appsmith/git/dto/BashFunctionResult.java | 14 + .../git/handler/ce/FSGitHandlerCEImpl.java | 11 + .../com/appsmith/git/service/BashService.java | 112 +++++ .../src/main/resources/application.properties | 2 +- .../appsmith-git/src/main/resources/git.sh | 87 ++++ .../external/git/handler/FSGitHandler.java | 10 + app/server/appsmith-server/pom.xml | 13 + .../appsmith/server/annotations/GitRoute.java | 18 + .../artifacts/gitRoute/GitRouteArtifact.java | 12 + .../gitRoute/GitRouteArtifactCE.java | 28 ++ .../server/aspect/GitRouteAspect.java | 393 ++++++++++++++++++ .../appsmith/server/aspect/GitRouteAspect.md | 44 ++ .../server/exceptions/AppsmithError.java | 40 ++ .../server/exceptions/AppsmithErrorCode.java | 8 + .../appsmith/server/git/GitRedisUtils.java | 24 ++ .../GitApplicationControllerCE.java | 22 + .../git/fs/GitFSServiceCECompatibleImpl.java | 7 +- .../server/git/fs/GitFSServiceCEImpl.java | 8 + .../server/git/fs/GitFSServiceImpl.java | 7 +- .../appsmith/server/helpers/RedisUtils.java | 30 ++ .../main/resources/application-ce.properties | 1 + .../git/autocommit/AutoCommitServiceTest.java | 165 -------- .../helpers/GitAutoCommitHelperImplTest.java | 17 - .../ExchangeJsonConversionTests.java | 30 -- .../resources/application-test.properties | 2 +- deploy/docker/fs/opt/appsmith/run-with-env.sh | 5 +- 30 files changed, 912 insertions(+), 224 deletions(-) create mode 100644 app/server/appsmith-git/src/main/java/com/appsmith/git/dto/BashFunctionResult.java create mode 100644 app/server/appsmith-git/src/main/java/com/appsmith/git/service/BashService.java create mode 100644 app/server/appsmith-git/src/main/resources/git.sh create mode 100644 app/server/appsmith-server/src/main/java/com/appsmith/server/annotations/GitRoute.java create mode 100644 app/server/appsmith-server/src/main/java/com/appsmith/server/artifacts/gitRoute/GitRouteArtifact.java create mode 100644 app/server/appsmith-server/src/main/java/com/appsmith/server/artifacts/gitRoute/GitRouteArtifactCE.java create mode 100644 app/server/appsmith-server/src/main/java/com/appsmith/server/aspect/GitRouteAspect.java create mode 100644 app/server/appsmith-server/src/main/java/com/appsmith/server/aspect/GitRouteAspect.md diff --git a/Dockerfile b/Dockerfile index ca7dc3e9fe..3686f08599 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,9 +11,11 @@ ENV APPSMITH_SEGMENT_CE_KEY=${APPSMITH_SEGMENT_CE_KEY} COPY deploy/docker/fs / -# Install git RUN apt-get update && \ - apt-get install -y git && \ + apt-get install -y software-properties-common && \ + add-apt-repository -y ppa:git-core/ppa && \ + apt-get update && \ + apt-get install -y git tar zstd openssh-client && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* @@ -35,6 +37,9 @@ COPY ./app/client/build editor/ # Add RTS - Application Layer COPY ./app/client/packages/rts/dist rts/ +# Create the git-storage directory with group writeable permissions so non-root users can write to it. +RUN mkdir --mode 775 "/dev/shm/git-storage" + ENV PATH /opt/bin:/opt/java/bin:/opt/node/bin:$PATH RUN <spring-boot-test test + + org.apache.commons + commons-exec + 1.3 + + + io.projectreactor + reactor-core + diff --git a/app/server/appsmith-git/src/main/java/com/appsmith/git/configurations/GitServiceConfig.java b/app/server/appsmith-git/src/main/java/com/appsmith/git/configurations/GitServiceConfig.java index 35fc7ffe24..7d12f1b29e 100644 --- a/app/server/appsmith-git/src/main/java/com/appsmith/git/configurations/GitServiceConfig.java +++ b/app/server/appsmith-git/src/main/java/com/appsmith/git/configurations/GitServiceConfig.java @@ -8,9 +8,13 @@ import org.springframework.context.annotation.Configuration; @Configuration public class GitServiceConfig { - @Value("${appsmith.git.root:/data/git-storage}") + @Value("${appsmith.git.root}") private String gitRootPath; @Value("gitInitializeRepo/GitConnect-Initialize-Repo-Template") private String readmeTemplatePath; + + public Boolean isGitInMemory() { + return gitRootPath.startsWith("/dev/shm/"); + } } diff --git a/app/server/appsmith-git/src/main/java/com/appsmith/git/dto/BashFunctionResult.java b/app/server/appsmith-git/src/main/java/com/appsmith/git/dto/BashFunctionResult.java new file mode 100644 index 0000000000..7a641d7a1e --- /dev/null +++ b/app/server/appsmith-git/src/main/java/com/appsmith/git/dto/BashFunctionResult.java @@ -0,0 +1,14 @@ +package com.appsmith.git.dto; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.RequiredArgsConstructor; + +@Data +@AllArgsConstructor +@RequiredArgsConstructor +public class BashFunctionResult { + private String output; + private int exitCode; + private String error; +} diff --git a/app/server/appsmith-git/src/main/java/com/appsmith/git/handler/ce/FSGitHandlerCEImpl.java b/app/server/appsmith-git/src/main/java/com/appsmith/git/handler/ce/FSGitHandlerCEImpl.java index 14f08fb935..281871c716 100644 --- a/app/server/appsmith-git/src/main/java/com/appsmith/git/handler/ce/FSGitHandlerCEImpl.java +++ b/app/server/appsmith-git/src/main/java/com/appsmith/git/handler/ce/FSGitHandlerCEImpl.java @@ -21,6 +21,7 @@ import com.appsmith.git.constants.GitDirectories; import com.appsmith.git.helpers.RepositoryHelper; import com.appsmith.git.helpers.SshTransportConfigCallback; import com.appsmith.git.helpers.StopwatchHelpers; +import com.appsmith.git.service.BashService; import io.micrometer.observation.ObservationRegistry; import io.micrometer.tracing.Span; import lombok.RequiredArgsConstructor; @@ -112,6 +113,8 @@ public class FSGitHandlerCEImpl implements FSGitHandler { private static final String SUCCESS_MERGE_STATUS = "This branch has no conflicts with the base branch."; private final ObservationHelper observationHelper; + private final BashService bashService = new BashService(); + /** * This method will handle the git-commit functionality. Under the hood it checks if the repo has already been * initialised and will be initialised if git repo is not present @@ -1107,6 +1110,14 @@ public class FSGitHandlerCEImpl implements FSGitHandler { return pathArray[1]; } + @Override + public Mono mergeBranch(Path repoSuffix, String sourceBranch, String destinationBranch) { + String repoPath = createRepoPath(repoSuffix).toString(); + return bashService + .callFunction("git.sh", "git_merge_branch", repoPath, sourceBranch, destinationBranch) + .map(result -> result.getOutput()); + } + @Override public Mono mergeBranch( Path repoSuffix, String sourceBranch, String destinationBranch, boolean keepWorkingDirChanges) { diff --git a/app/server/appsmith-git/src/main/java/com/appsmith/git/service/BashService.java b/app/server/appsmith-git/src/main/java/com/appsmith/git/service/BashService.java new file mode 100644 index 0000000000..de1a3a41b2 --- /dev/null +++ b/app/server/appsmith-git/src/main/java/com/appsmith/git/service/BashService.java @@ -0,0 +1,112 @@ +package com.appsmith.git.service; + +import com.appsmith.git.dto.BashFunctionResult; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.exec.CommandLine; +import org.apache.commons.exec.DefaultExecutor; +import org.apache.commons.exec.PumpStreamHandler; +import reactor.core.publisher.Mono; +import reactor.core.scheduler.Schedulers; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +@Slf4j +public class BashService { + + // Executes bash function from classpath resource + public Mono callFunction(String classpathResource, String functionName, String... args) { + return Mono.fromCallable(() -> callFunctionUnBounded(classpathResource, functionName, args)) + .subscribeOn(Schedulers.boundedElastic()); + } + + // Executes bash script and returns result + private BashFunctionResult callFunctionUnBounded(String classpathResource, String functionName, String... args) + throws IOException { + InputStream scriptContentInputStream = + BashService.class.getClassLoader().getResourceAsStream(classpathResource); + if (scriptContentInputStream == null) { + throw new FileNotFoundException("Resource not found: " + classpathResource); + } + String scriptContent = new String(scriptContentInputStream.readAllBytes(), StandardCharsets.UTF_8); + + String fullScript = buildFullCommand(scriptContent, functionName, args); + + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + ByteArrayOutputStream errorStream = new ByteArrayOutputStream(); + ByteArrayInputStream inputStream = new ByteArrayInputStream(fullScript.getBytes(StandardCharsets.UTF_8)); + + CommandLine cmdLine = new CommandLine("bash"); + + DefaultExecutor executor = new DefaultExecutor(); + executor.setStreamHandler(new PumpStreamHandler(outputStream, errorStream, inputStream)); + + Integer exitCode = null; + String exceptionError = null; + try { + exitCode = executor.execute(cmdLine); + } catch (Exception e) { + exceptionError = e.getMessage(); + } + + String output = outputStream.toString(StandardCharsets.UTF_8).trim(); + String error = errorStream.toString(StandardCharsets.UTF_8).trim(); + + if (exceptionError != null || exitCode != 0) { + throw new RuntimeException( + "Bash execution failed: " + buildErrorDetails(output, error, exceptionError, exitCode)); + } + + log.info("Script: {}", fullScript); + log.info("Output: {}", output); + log.info("Error: {}", error); + log.info("Exit code: {}", exitCode); + + outputStream.close(); + errorStream.close(); + inputStream.close(); + + return new BashFunctionResult(output, exitCode, error); + } + + // Builds complete bash command with args + private String buildFullCommand(String scriptContent, String functionName, String... args) { + String variableAssignments = IntStream.range(0, args.length) + .mapToObj(i -> String.format("arg%d=\"%s\"", i + 1, args[i])) + .collect(Collectors.joining("\n")); + + String functionCall = functionName + + " " + + IntStream.range(0, args.length) + .mapToObj(i -> String.format("\"$arg%d\"", i + 1)) + .collect(Collectors.joining(" ")); + + return scriptContent + "\n" + variableAssignments + "\n" + functionCall; + } + + // Returns fallback if string is blank + private String fallbackIfBlank(String value, String fallback) { + return (value == null || value.isBlank()) ? fallback : value; + } + + // Returns fallback if integer is null + private String fallbackIfBlank(Integer value, String fallback) { + return (value == null) ? fallback : value.toString(); + } + + // Formats error details for exception + private String buildErrorDetails(String output, String error, String exceptionError, Integer exitCode) { + return "EXITCODE: %s\nEXCEPTION: %s\nSTDERR: %s\nSTDOUT: %s" + .formatted( + fallbackIfBlank(exitCode, "(empty)"), + fallbackIfBlank(output, "(empty)"), + fallbackIfBlank(error, "(empty)"), + fallbackIfBlank(exceptionError, "(none)")); + } +} diff --git a/app/server/appsmith-git/src/main/resources/application.properties b/app/server/appsmith-git/src/main/resources/application.properties index a9293728b4..8f9762a60b 100644 --- a/app/server/appsmith-git/src/main/resources/application.properties +++ b/app/server/appsmith-git/src/main/resources/application.properties @@ -1,2 +1,2 @@ # Local git repo path -appsmith.git.root = ${APPSMITH_GIT_ROOT:} \ No newline at end of file +appsmith.git.root = ${APPSMITH_GIT_ROOT:} diff --git a/app/server/appsmith-git/src/main/resources/git.sh b/app/server/appsmith-git/src/main/resources/git.sh new file mode 100644 index 0000000000..1c643c0b45 --- /dev/null +++ b/app/server/appsmith-git/src/main/resources/git.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# Time-to-live for git artifacts in Redis (24 hours in seconds) +GIT_ARTIFACT_TTL=86400 + +# Returns Redis lock key for given key +get_lock_key() { + local redis_key="$1" + + echo "lock:${redis_key}" +} + +# Clones git repo using SSH key +git_clone() { + local private_key="$1" + local remote_url="$2" + local target_folder="$3" + + local temp_private_key=$(mktemp /dev/shm/tmp.XXXXXX) + trap 'rm -rf "'"$temp_private_key"'"' EXIT ERR + + echo "$private_key" > "$temp_private_key" + + git -C "$target_folder" init "$target_folder" --initial-branch=none + git -C "$target_folder" remote add origin "$remote_url" + GIT_SSH_COMMAND="ssh -i $temp_private_key -o StrictHostKeyChecking=no" git -C "$target_folder" fetch origin --depth=1 +} + +# Uploads git repo to Redis as compressed archive +git_upload() { + local redis_key="$1" + local redis_url="$2" + local target_folder="$3" + + trap 'rm -rf "'"$target_folder"'"' EXIT ERR + + rm -f "$target_folder/.git/index.lock" + + tar -cf - -C "$target_folder" . | zstd -q --threads=0 | base64 -w 0 | redis-cli -u "$redis_url" --raw -x SETEX "$redis_key" "$GIT_ARTIFACT_TTL" +} + +# Downloads git repo from Redis or clones if not cached +git_download() { + local author_email="$1" + local author_name="$2" + local private_key="$3" + local redis_key="$4" + local redis_url="$5" + local remote_url="$6" + local target_folder="$7" + + rm -rf "$target_folder" + mkdir -p "$target_folder" + + if [ "$(redis-cli -u "$redis_url" --raw EXISTS "$redis_key")" = "1" ]; then + redis-cli -u "$redis_url" --raw GET "$redis_key" | base64 -d | zstd -d --threads=0 | tar -xf - -C "$target_folder" + else + git_clone "$private_key" "$remote_url" "$target_folder" + fi + + rm -f "$target_folder/.git/index.lock" + + git -C "$target_folder" config user.name "$author_name" + git -C "$target_folder" config user.email "$author_email" + git -C "$target_folder" config fetch.parallel 4 + + git -C "$target_folder" reset --hard + + # Checkout all branches + for remote in $(git -C "$target_folder" branch -r | grep -vE 'origin/HEAD'); do + branch=${remote#origin/} + if ! git -C "$target_folder" show-ref --quiet "refs/heads/$branch"; then + git -C "$target_folder" checkout -b "$branch" "$remote" || true + fi + done +} + +git_merge_branch() { + local target_folder="$1" + local source_branch="$2" + local destination_branch="$3" + + git -C "$target_folder" checkout "$destination_branch" + git -C "$target_folder" merge "$source_branch" --strategy=recursive --allow-unrelated-histories --no-edit +} diff --git a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/git/handler/FSGitHandler.java b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/git/handler/FSGitHandler.java index dd86807c08..088a5ee158 100644 --- a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/git/handler/FSGitHandler.java +++ b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/git/handler/FSGitHandler.java @@ -160,6 +160,16 @@ public interface FSGitHandler { */ Mono getStatus(Path repoPath, String branchName, boolean keepWorkingDirChanges); + /** + * This method merges source branch into destination branch for a git repository which is present on the partial + * path provided. This assumes that the branch on which the merge will happen is already checked out + * @param repoSuffix suffixedPath used to generate the base repo path this includes workspaceId, defaultAppId, repoName + * @param sourceBranch name of the branch whose commits will be referred amd merged to destinationBranch + * @param destinationBranch Merge operation is performed on this branch + * @return Merge status + */ + Mono mergeBranch(Path repoSuffix, String sourceBranch, String destinationBranch); + /** * This method merges source branch into destination branch for a git repository which is present on the partial * path provided. This assumes that the branch on which the merge will happen is already checked out diff --git a/app/server/appsmith-server/pom.xml b/app/server/appsmith-server/pom.xml index 7eb7287df4..855d5f7198 100644 --- a/app/server/appsmith-server/pom.xml +++ b/app/server/appsmith-server/pom.xml @@ -444,6 +444,19 @@ org.springframework.boot spring-boot-maven-plugin + + + org.apache.maven.plugins + maven-compiler-plugin + 3.10.1 + + + -parameters + + 17 + 17 + + diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/annotations/GitRoute.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/annotations/GitRoute.java new file mode 100644 index 0000000000..720963956e --- /dev/null +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/annotations/GitRoute.java @@ -0,0 +1,18 @@ +package com.appsmith.server.annotations; + +import com.appsmith.server.constants.ArtifactType; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Target({ElementType.METHOD, ElementType.TYPE}) +@Retention(RetentionPolicy.RUNTIME) +@Documented +public @interface GitRoute { + String fieldName(); + + ArtifactType artifactType(); +} diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/artifacts/gitRoute/GitRouteArtifact.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/artifacts/gitRoute/GitRouteArtifact.java new file mode 100644 index 0000000000..2b11e85354 --- /dev/null +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/artifacts/gitRoute/GitRouteArtifact.java @@ -0,0 +1,12 @@ +package com.appsmith.server.artifacts.gitRoute; + +import com.appsmith.server.repositories.ApplicationRepository; +import org.springframework.stereotype.Component; + +@Component +public class GitRouteArtifact extends GitRouteArtifactCE { + + public GitRouteArtifact(ApplicationRepository applicationRepository) { + super(applicationRepository); + } +} diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/artifacts/gitRoute/GitRouteArtifactCE.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/artifacts/gitRoute/GitRouteArtifactCE.java new file mode 100644 index 0000000000..d81b513dd9 --- /dev/null +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/artifacts/gitRoute/GitRouteArtifactCE.java @@ -0,0 +1,28 @@ +package com.appsmith.server.artifacts.gitRoute; + +import com.appsmith.server.constants.ArtifactType; +import com.appsmith.server.domains.Artifact; +import com.appsmith.server.exceptions.AppsmithError; +import com.appsmith.server.exceptions.AppsmithException; +import com.appsmith.server.repositories.ApplicationRepository; +import org.springframework.stereotype.Component; +import reactor.core.publisher.Mono; + +@Component +public abstract class GitRouteArtifactCE { + protected final ApplicationRepository applicationRepository; + + public GitRouteArtifactCE(ApplicationRepository applicationRepository) { + this.applicationRepository = applicationRepository; + } + + public Mono getArtifact(ArtifactType artifactType, String artifactId) { + return switch (artifactType) { + case APPLICATION -> applicationRepository + .findById(artifactId) + .switchIfEmpty(Mono.error(new AppsmithException(AppsmithError.NO_RESOURCE_FOUND))) + .map(app -> (Artifact) app); + default -> Mono.error(new AppsmithException(AppsmithError.GIT_ROUTE_HANDLER_NOT_FOUND, artifactType)); + }; + } +} diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/aspect/GitRouteAspect.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/aspect/GitRouteAspect.java new file mode 100644 index 0000000000..56f520d00b --- /dev/null +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/aspect/GitRouteAspect.java @@ -0,0 +1,393 @@ +package com.appsmith.server.aspect; + +import com.appsmith.git.configurations.GitServiceConfig; +import com.appsmith.git.service.BashService; +import com.appsmith.server.annotations.GitRoute; +import com.appsmith.server.artifacts.gitRoute.GitRouteArtifact; +import com.appsmith.server.constants.ArtifactType; +import com.appsmith.server.domains.Artifact; +import com.appsmith.server.domains.GitArtifactMetadata; +import com.appsmith.server.domains.GitAuth; +import com.appsmith.server.domains.GitProfile; +import com.appsmith.server.exceptions.AppsmithError; +import com.appsmith.server.exceptions.AppsmithException; +import com.appsmith.server.git.utils.GitProfileUtils; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.experimental.Accessors; +import lombok.extern.slf4j.Slf4j; +import org.aspectj.lang.ProceedingJoinPoint; +import org.aspectj.lang.annotation.Around; +import org.aspectj.lang.annotation.Aspect; +import org.aspectj.lang.reflect.CodeSignature; +import org.bouncycastle.jcajce.spec.OpenSSHPrivateKeySpec; +import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.bouncycastle.util.io.pem.PemReader; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.data.redis.core.ReactiveRedisTemplate; +import org.springframework.stereotype.Component; +import reactor.core.publisher.Mono; + +import java.io.StringReader; +import java.nio.file.Paths; +import java.security.KeyFactory; +import java.security.PrivateKey; +import java.security.spec.PKCS8EncodedKeySpec; +import java.time.Duration; +import java.util.Base64; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.IntStream; + +@Aspect +@Component +@RequiredArgsConstructor +@Slf4j +public class GitRouteAspect { + + private static final Duration LOCK_TTL = Duration.ofSeconds(90); + private static final String REDIS_REPO_KEY_FORMAT = "purpose=repo/v=1/workspace=%s/artifact=%s/repository=%s/"; + + private final ReactiveRedisTemplate redis; + private final GitProfileUtils gitProfileUtils; + private final GitServiceConfig gitServiceConfig; + private final GitRouteArtifact gitRouteArtifact; + private final BashService bashService = new BashService(); + + @Value("${appsmith.redis.git.url}") + private String redisUrl; + + @Value("${appsmith.git.root}") + private String gitRootPath; + + /* + * FSM: Definitions + */ + + private enum State { + ARTIFACT, + PARENT, + GIT_META, + REPO_KEY, + LOCK_KEY, + LOCK, + GIT_PROFILE, + GIT_AUTH, + GIT_KEY, + REPO_PATH, + DOWNLOAD, + EXECUTE, + UPLOAD, + UNLOCK, + RESULT, + DONE + } + + private enum Outcome { + SUCCESS, + FAIL + } + + @Getter + @AllArgsConstructor + private static class StateConfig { + private final State onSuccess; + private final State onFail; + private final String contextField; + private final Function> function; + + private State next(Outcome outcome) { + return outcome == Outcome.SUCCESS ? onSuccess : onFail; + } + } + + @Data + @Accessors(chain = true) + private static class Context { + // Inputs + private ProceedingJoinPoint joinPoint; + private GitRoute gitRoute; + + // Intermediate Inputs + private String fieldValue; + + // Tasks + private Artifact artifact; + private Artifact parent; + private GitArtifactMetadata gitMeta; + private String repoKey; + private String lockKey; + private Boolean lock; + private GitProfile gitProfile; + private GitAuth gitAuth; + private String gitKey; + private String repoPath; + private Object download; + private Object execute; + private Object upload; + private Boolean unlock; + private Object result; + + // Errors + private Throwable error; + } + + // Refer to GitRouteAspect.md#gitroute-fsm-execution-flow for the FSM diagram. + private final Map FSM = Map.ofEntries( + Map.entry(State.ARTIFACT, new StateConfig(State.PARENT, State.RESULT, "artifact", this::artifact)), + Map.entry(State.PARENT, new StateConfig(State.GIT_META, State.RESULT, "parent", this::parent)), + Map.entry(State.GIT_META, new StateConfig(State.REPO_KEY, State.RESULT, "gitMeta", this::gitMeta)), + Map.entry(State.REPO_KEY, new StateConfig(State.LOCK_KEY, State.RESULT, "repoKey", this::repoKey)), + Map.entry(State.LOCK_KEY, new StateConfig(State.LOCK, State.RESULT, "lockKey", this::lockKey)), + Map.entry(State.LOCK, new StateConfig(State.GIT_PROFILE, State.RESULT, "lock", this::lock)), + Map.entry(State.GIT_PROFILE, new StateConfig(State.GIT_AUTH, State.UNLOCK, "gitProfile", this::gitProfile)), + Map.entry(State.GIT_AUTH, new StateConfig(State.GIT_KEY, State.UNLOCK, "gitAuth", this::gitAuth)), + Map.entry(State.GIT_KEY, new StateConfig(State.REPO_PATH, State.UNLOCK, "gitKey", this::gitKey)), + Map.entry(State.REPO_PATH, new StateConfig(State.DOWNLOAD, State.UNLOCK, "repoPath", this::repoPath)), + Map.entry(State.DOWNLOAD, new StateConfig(State.EXECUTE, State.UNLOCK, "download", this::download)), + Map.entry(State.EXECUTE, new StateConfig(State.UPLOAD, State.UPLOAD, "execute", this::execute)), + Map.entry(State.UPLOAD, new StateConfig(State.UNLOCK, State.UNLOCK, "upload", this::upload)), + Map.entry(State.UNLOCK, new StateConfig(State.RESULT, State.RESULT, "unlock", this::unlock)), + Map.entry(State.RESULT, new StateConfig(State.DONE, State.DONE, "result", this::result))); + + /* + * FSM: Runners + */ + + // Entry point for Git operations + @Around("@annotation(gitRoute)") + public Object handleGitRoute(ProceedingJoinPoint joinPoint, GitRoute gitRoute) { + Context ctx = new Context().setJoinPoint(joinPoint).setGitRoute(gitRoute); + + // If Git is not in memory, we can just execute the join point + if (!gitServiceConfig.isGitInMemory()) { + return execute(ctx); + } + + String fieldValue = extractFieldValue(joinPoint, gitRoute.fieldName()); + + ctx.setFieldValue(fieldValue); + + return run(ctx, State.ARTIFACT) + .flatMap(unused -> ctx.getError() != null ? Mono.error(ctx.getError()) : Mono.just(ctx.getResult())); + } + + // State machine executor + private Mono run(Context ctx, State current) { + if (current == State.DONE) { + return Mono.just(true); + } + + StateConfig config = FSM.get(current); + long startTime = System.currentTimeMillis(); + + return config.getFunction() + .apply(ctx) + .flatMap(result -> { + setContextField(ctx, config.getContextField(), result); + long duration = System.currentTimeMillis() - startTime; + log.info("State: {}, SUCCESS: {}, Time: {}ms", current, result, duration); + return run(ctx, config.next(Outcome.SUCCESS)); + }) + .onErrorResume(e -> { + ctx.setError(e); + long duration = System.currentTimeMillis() - startTime; + log.info("State: {}, FAIL: {}, Time: {}ms", current, e.getMessage(), duration); + return run(ctx, config.next(Outcome.FAIL)); + }); + } + + /* + * FSM: Tasks + */ + + // Acquires Redis lock + private Mono lock(Context ctx) { + return redis.opsForValue() + .setIfAbsent(ctx.getLockKey(), "1", LOCK_TTL) + .flatMap(locked -> locked + ? Mono.just(true) + : Mono.error(new AppsmithException(AppsmithError.GIT_FILE_IN_USE, ctx.getLockKey()))); + } + + // Finds artifact + private Mono artifact(Context ctx) { + ArtifactType artifactType = ctx.getGitRoute().artifactType(); + String artifactId = ctx.getFieldValue(); + return gitRouteArtifact.getArtifact(artifactType, artifactId); + } + + // Finds parent artifact + private Mono parent(Context ctx) { + ArtifactType artifactType = ctx.getGitRoute().artifactType(); + String parentArtifactId = ctx.getArtifact().getGitArtifactMetadata().getDefaultArtifactId(); + return gitRouteArtifact.getArtifact(artifactType, parentArtifactId); + } + + // Validates Git metadata + private Mono gitMeta(Context ctx) { + return Mono.justOrEmpty(ctx.getParent().getGitArtifactMetadata()) + .switchIfEmpty(Mono.error(new AppsmithException( + AppsmithError.INVALID_GIT_CONFIGURATION, "Git metadata is not configured"))); + } + + // Generates Redis repo key + private Mono repoKey(Context ctx) { + String key = String.format( + REDIS_REPO_KEY_FORMAT, + ctx.getArtifact().getWorkspaceId(), + ctx.getGitMeta().getDefaultArtifactId(), + ctx.getGitMeta().getRepoName()); + return Mono.just(key); + } + + // Generates Redis lock key + private Mono lockKey(Context ctx) { + String key = String.format("purpose=lock/%s", ctx.getRepoKey()); + return Mono.just(key); + } + + // Gets Git user profile + private Mono gitProfile(Context ctx) { + return gitProfileUtils + .getGitProfileForUser(ctx.getFieldValue()) + .switchIfEmpty(Mono.error(new AppsmithException( + AppsmithError.INVALID_GIT_CONFIGURATION, "Git profile is not configured"))); + } + + // Validates Git auth + private Mono gitAuth(Context ctx) { + return Mono.justOrEmpty(ctx.getGitMeta().getGitAuth()) + .switchIfEmpty(Mono.error(new AppsmithException( + AppsmithError.INVALID_GIT_CONFIGURATION, "Git authentication is not configured"))); + } + + // Processes Git SSH key + private Mono gitKey(Context ctx) { + try { + return Mono.just(processPrivateKey( + ctx.getGitAuth().getPrivateKey(), ctx.getGitAuth().getPublicKey())); + } catch (Exception e) { + return Mono.error(new AppsmithException( + AppsmithError.INVALID_GIT_CONFIGURATION, "Failed to process private key: " + e.getMessage())); + } + } + + // Gets local repo path + private Mono repoPath(Context ctx) { + var path = Paths.get( + gitRootPath, + ctx.getArtifact().getWorkspaceId(), + ctx.getGitMeta().getDefaultArtifactId(), + ctx.getGitMeta().getRepoName()); + return Mono.just(path.toString()); + } + + // Downloads Git repo + private Mono download(Context ctx) { + return bashService.callFunction( + "git.sh", + "git_download", + ctx.getGitProfile().getAuthorEmail(), + ctx.getGitProfile().getAuthorName(), + ctx.getGitKey(), + ctx.getRepoKey(), + redisUrl, + ctx.getGitMeta().getRemoteUrl(), + ctx.getRepoPath()); + } + + // Executes Git operation + private Mono execute(Context ctx) { + try { + return (Mono) ctx.getJoinPoint().proceed(); + } catch (Throwable e) { + return Mono.error(e); + } + } + + // Uploads Git changes + private Mono upload(Context ctx) { + return bashService.callFunction("git.sh", "git_upload", ctx.getRepoKey(), redisUrl, ctx.getRepoPath()); + } + + // Releases Redis lock + private Mono unlock(Context ctx) { + return redis.delete(ctx.getLockKey()).map(count -> count > 0); + } + + // Returns operation result + private Mono result(Context ctx) { + return ctx.getError() != null ? Mono.error(ctx.getError()) : Mono.just(ctx.getExecute()); + } + + /* + * Helpers: Git Route + */ + + // Extracts field from join point + private static String extractFieldValue(ProceedingJoinPoint jp, String target) { + String[] names = ((CodeSignature) jp.getSignature()).getParameterNames(); + Object[] values = jp.getArgs(); + return IntStream.range(0, names.length) + .filter(i -> names[i].equals(target)) + .mapToObj(i -> String.valueOf(values[i])) + .findFirst() + .orElse(null); + } + + // Sets context field value + private static void setContextField(Context ctx, String fieldName, Object value) { + try { + var field = ctx.getClass().getDeclaredField(fieldName); + field.setAccessible(true); + field.set(ctx, value); + } catch (Exception e) { + throw new RuntimeException("Failed to set field " + fieldName, e); + } + } + + /* + * Helpers: Git Private Key + * Reference: SshTransportConfigCallback.java + */ + + // Processes SSH private key + private static String processPrivateKey(String privateKey, String publicKey) throws Exception { + String[] splitKeys = privateKey.split("-----.*-----\n"); + return splitKeys.length > 1 + ? handlePemFormat(privateKey, publicKey) + : handleBase64Format(privateKey, publicKey); + } + + // Handles PEM format key + private static String handlePemFormat(String privateKey, String publicKey) throws Exception { + byte[] content = + new PemReader(new StringReader(privateKey)).readPemObject().getContent(); + OpenSSHPrivateKeySpec privateKeySpec = new OpenSSHPrivateKeySpec(content); + KeyFactory keyFactory = getKeyFactory(publicKey); + PrivateKey generatedPrivateKey = keyFactory.generatePrivate(privateKeySpec); + return Base64.getEncoder().encodeToString(generatedPrivateKey.getEncoded()); + } + + // Handles Base64 format key + private static String handleBase64Format(String privateKey, String publicKey) throws Exception { + PKCS8EncodedKeySpec privateKeySpec = + new PKCS8EncodedKeySpec(Base64.getDecoder().decode(privateKey)); + PrivateKey generatedPrivateKey = getKeyFactory(publicKey).generatePrivate(privateKeySpec); + return formatPrivateKey(Base64.getEncoder().encodeToString(generatedPrivateKey.getEncoded())); + } + + // Gets key factory for algorithm + private static KeyFactory getKeyFactory(String publicKey) throws Exception { + String algo = publicKey.startsWith("ssh-rsa") ? "RSA" : "ECDSA"; + return KeyFactory.getInstance(algo, new BouncyCastleProvider()); + } + + // Formats private key string + private static String formatPrivateKey(String privateKey) { + return "-----BEGIN PRIVATE KEY-----\n" + privateKey + "\n-----END PRIVATE KEY-----"; + } +} diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/aspect/GitRouteAspect.md b/app/server/appsmith-server/src/main/java/com/appsmith/server/aspect/GitRouteAspect.md new file mode 100644 index 0000000000..063b17ba85 --- /dev/null +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/aspect/GitRouteAspect.md @@ -0,0 +1,44 @@ +## GitRoute FSM Execution Flow + +```mermaid +flowchart TD + START([START]) --> ARTIFACT + + ARTIFACT -->|SUCCESS| PARENT + ARTIFACT -->|FAIL| RESULT + + PARENT -->|SUCCESS| GIT_META + PARENT -->|FAIL| RESULT + + GIT_META -->|SUCCESS| REPO_KEY + GIT_META -->|FAIL| RESULT + + REPO_KEY -->|SUCCESS| LOCK_KEY + REPO_KEY -->|FAIL| RESULT + + LOCK_KEY -->|SUCCESS| LOCK + LOCK_KEY -->|FAIL| RESULT + + LOCK -->|SUCCESS| GIT_PROFILE + LOCK -->|FAIL| RESULT + + GIT_PROFILE -->|SUCCESS| GIT_AUTH + GIT_PROFILE -->|FAIL| UNLOCK + + GIT_AUTH -->|SUCCESS| GIT_KEY + GIT_AUTH -->|FAIL| UNLOCK + + GIT_KEY -->|SUCCESS| REPO_PATH + GIT_KEY -->|FAIL| UNLOCK + + REPO_PATH -->|SUCCESS| DOWNLOAD + REPO_PATH -->|FAIL| UNLOCK + + DOWNLOAD -->|SUCCESS| EXECUTE + DOWNLOAD -->|FAIL| UNLOCK + + EXECUTE -->|SUCCESS or FAIL| UPLOAD + UPLOAD -->|SUCCESS or FAIL| UNLOCK + UNLOCK -->|SUCCESS or FAIL| RESULT + RESULT --> DONE([DONE]) +``` diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/exceptions/AppsmithError.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/exceptions/AppsmithError.java index b3f2be7859..2877b7fcd2 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/exceptions/AppsmithError.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/exceptions/AppsmithError.java @@ -1020,6 +1020,46 @@ public enum AppsmithError { "Insufficient password strength", ErrorType.ARGUMENT_ERROR, null), + GIT_ROUTE_HANDLER_NOT_FOUND( + 500, + AppsmithErrorCode.GIT_ROUTE_HANDLER_NOT_FOUND.getCode(), + "No handler found for Git route type: {0}", + AppsmithErrorAction.DEFAULT, + "Git route handler not found", + ErrorType.GIT_CONFIGURATION_ERROR, + null), + GIT_ROUTE_INVALID_FIELD_VALUE( + 400, + AppsmithErrorCode.GIT_ROUTE_INVALID_FIELD_VALUE.getCode(), + "Invalid field value provided for Git route: {0}", + AppsmithErrorAction.DEFAULT, + "Invalid Git route field value", + ErrorType.GIT_CONFIGURATION_ERROR, + null), + GIT_ROUTE_CONTEXT_BUILD_ERROR( + 500, + AppsmithErrorCode.GIT_ROUTE_CONTEXT_BUILD_ERROR.getCode(), + "Failed to build Git route context: {0}", + AppsmithErrorAction.DEFAULT, + "Git route context build failed", + ErrorType.GIT_CONFIGURATION_ERROR, + null), + GIT_ROUTE_ARTIFACT_NOT_FOUND( + 404, + AppsmithErrorCode.GIT_ROUTE_ARTIFACT_NOT_FOUND.getCode(), + "Artifact not found for Git route: {0}", + AppsmithErrorAction.DEFAULT, + "Git route artifact not found", + ErrorType.GIT_CONFIGURATION_ERROR, + null), + GIT_ROUTE_INVALID_PRIVATE_KEY( + 400, + AppsmithErrorCode.GIT_ROUTE_INVALID_PRIVATE_KEY.getCode(), + "Invalid private key format for Git route: {0}", + AppsmithErrorAction.DEFAULT, + "Invalid Git private key", + ErrorType.GIT_CONFIGURATION_ERROR, + null), ; private final Integer httpErrorCode; diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/exceptions/AppsmithErrorCode.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/exceptions/AppsmithErrorCode.java index c493a05a19..e19a9f0353 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/exceptions/AppsmithErrorCode.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/exceptions/AppsmithErrorCode.java @@ -131,6 +131,14 @@ public enum AppsmithErrorCode { INVALID_METHOD_LEVEL_ANNOTATION_USAGE("AE-APP-4094", "Invalid usage for custom annotation"), FEATURE_FLAG_MIGRATION_FAILURE("AE-APP-5045", "Feature flag based migration error"), + + // Git route related error codes + GIT_ROUTE_HANDLER_NOT_FOUND("AE-GIT-5005", "Git route handler not found"), + GIT_ROUTE_INVALID_FIELD_VALUE("AE-GIT-5006", "Git route invalid field value"), + GIT_ROUTE_CONTEXT_BUILD_ERROR("AE-GIT-5007", "Git route context build error"), + GIT_ROUTE_ARTIFACT_NOT_FOUND("AE-GIT-5008", "Git route artifact not found"), + GIT_ROUTE_INVALID_PRIVATE_KEY("AE-GIT-5009", "Git route invalid private key"), + DATASOURCE_CONNECTION_RATE_LIMIT_BLOCKING_FAILED( "AE-TMR-4031", "Rate limit exhausted, blocking the host name failed"), TRIGGER_PARAMETERS_EMPTY("AE-DS-4001", "Trigger parameters empty."), diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/git/GitRedisUtils.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/git/GitRedisUtils.java index 40e903568f..50c3982e69 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/git/GitRedisUtils.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/git/GitRedisUtils.java @@ -1,6 +1,7 @@ package com.appsmith.server.git; import com.appsmith.external.git.constants.GitSpan; +import com.appsmith.git.configurations.GitServiceConfig; import com.appsmith.server.constants.ArtifactType; import com.appsmith.server.exceptions.AppsmithError; import com.appsmith.server.exceptions.AppsmithException; @@ -18,11 +19,13 @@ import static com.appsmith.server.helpers.GitUtils.RETRY_DELAY; @Slf4j @Component +@Deprecated @RequiredArgsConstructor public class GitRedisUtils { private final RedisUtils redisUtils; private final ObservationRegistry observationRegistry; + private final GitServiceConfig gitServiceConfig; /** * Adds a baseArtifact id as a key in redis, the presence of this key represents a symbolic lock, essentially meaning that no new operations @@ -33,7 +36,11 @@ public class GitRedisUtils { * @param isRetryAllowed : Boolean for whether retries for adding the value is allowed * @return a boolean publisher for the added file locks */ + @Deprecated public Mono addFileLock(String key, String commandName, Boolean isRetryAllowed) { + if (gitServiceConfig.isGitInMemory()) { + return Mono.just(true); + } long numberOfRetries = Boolean.TRUE.equals(isRetryAllowed) ? MAX_RETRIES : 0L; log.info("Git command {} is trying to acquire the lock for identity {}", commandName, key); @@ -51,12 +58,20 @@ public class GitRedisUtils { .tap(Micrometer.observation(observationRegistry)); } + @Deprecated public Mono addFileLock(String baseArtifactId, String commandName) { + if (gitServiceConfig.isGitInMemory()) { + return Mono.just(true); + } String key = generateRedisKey(ArtifactType.APPLICATION, baseArtifactId); return addFileLock(key, commandName, true); } + @Deprecated public Mono releaseFileLock(String baseArtifactId) { + if (gitServiceConfig.isGitInMemory()) { + return Mono.just(true); + } String key = generateRedisKey(ArtifactType.APPLICATION, baseArtifactId); return redisUtils @@ -75,8 +90,12 @@ public class GitRedisUtils { * @param isLockRequired : is lock really required or is it a proxy function * @return : Boolean for whether the lock is acquired */ + @Deprecated public Mono acquireGitLock( ArtifactType artifactType, String baseArtifactId, String commandName, Boolean isLockRequired) { + if (gitServiceConfig.isGitInMemory()) { + return Mono.just(true); + } if (!Boolean.TRUE.equals(isLockRequired)) { return Mono.just(Boolean.TRUE); } @@ -96,7 +115,11 @@ public class GitRedisUtils { * @param isLockRequired : is lock really required or is it a proxy function * @return : Boolean for whether the lock is released */ + @Deprecated public Mono releaseFileLock(ArtifactType artifactType, String baseArtifactId, boolean isLockRequired) { + if (gitServiceConfig.isGitInMemory()) { + return Mono.just(true); + } if (!Boolean.TRUE.equals(isLockRequired)) { return Mono.just(Boolean.TRUE); } @@ -109,6 +132,7 @@ public class GitRedisUtils { .tap(Micrometer.observation(observationRegistry)); } + @Deprecated private String generateRedisKey(ArtifactType artifactType, String artifactId) { return artifactType.lowerCaseName() + "-" + artifactId; } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/git/controllers/GitApplicationControllerCE.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/git/controllers/GitApplicationControllerCE.java index ee8000e15f..8d509911a4 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/git/controllers/GitApplicationControllerCE.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/git/controllers/GitApplicationControllerCE.java @@ -6,6 +6,7 @@ import com.appsmith.external.dtos.MergeStatusDTO; import com.appsmith.external.git.constants.ce.RefType; import com.appsmith.external.views.Views; import com.appsmith.git.dto.CommitDTO; +import com.appsmith.server.annotations.GitRoute; import com.appsmith.server.artifacts.base.ArtifactService; import com.appsmith.server.constants.ArtifactType; import com.appsmith.server.constants.FieldName; @@ -59,6 +60,7 @@ public class GitApplicationControllerCE { @JsonView({Views.Metadata.class}) @GetMapping("/{baseApplicationId}/metadata") + @GitRoute(fieldName = "baseApplicationId", artifactType = ArtifactType.APPLICATION) public Mono> getGitMetadata(@PathVariable String baseApplicationId) { return centralGitService .getGitArtifactMetadata(baseApplicationId, ARTIFACT_TYPE) @@ -67,6 +69,7 @@ public class GitApplicationControllerCE { @JsonView(Views.Public.class) @PostMapping("/{applicationId}/connect") + @GitRoute(fieldName = "applicationId", artifactType = ArtifactType.APPLICATION) public Mono> connectApplicationToRemoteRepo( @PathVariable String applicationId, @RequestBody GitConnectDTO gitConnectDTO, @@ -79,6 +82,7 @@ public class GitApplicationControllerCE { @JsonView(Views.Public.class) @PostMapping("/{branchedApplicationId}/commit") @ResponseStatus(HttpStatus.CREATED) + @GitRoute(fieldName = "branchedApplicationId", artifactType = ArtifactType.APPLICATION) public Mono> commit( @RequestBody CommitDTO commitDTO, @PathVariable String branchedApplicationId) { log.info("Going to commit branchedApplicationId {}", branchedApplicationId); @@ -90,6 +94,7 @@ public class GitApplicationControllerCE { @JsonView(Views.Public.class) @PostMapping("/{referencedApplicationId}/create-ref") @ResponseStatus(HttpStatus.CREATED) + @GitRoute(fieldName = "referencedApplicationId", artifactType = ArtifactType.APPLICATION) public Mono> createReference( @PathVariable String referencedApplicationId, @RequestHeader(name = FieldName.BRANCH_NAME, required = false) String srcBranch, @@ -105,6 +110,7 @@ public class GitApplicationControllerCE { @JsonView(Views.Public.class) @PostMapping("/{referencedApplicationId}/checkout-ref") + @GitRoute(fieldName = "referencedApplicationId", artifactType = ArtifactType.APPLICATION) public Mono> checkoutReference( @PathVariable String referencedApplicationId, @RequestBody GitRefDTO gitRefDTO) { return centralGitService @@ -114,6 +120,7 @@ public class GitApplicationControllerCE { @JsonView(Views.Public.class) @PostMapping("/{branchedApplicationId}/disconnect") + @GitRoute(fieldName = "branchedApplicationId", artifactType = ArtifactType.APPLICATION) public Mono> disconnectFromRemote(@PathVariable String branchedApplicationId) { log.info("Going to remove the remoteUrl for application {}", branchedApplicationId); return centralGitService @@ -123,6 +130,7 @@ public class GitApplicationControllerCE { @JsonView(Views.Public.class) @GetMapping("/{branchedApplicationId}/pull") + @GitRoute(fieldName = "branchedApplicationId", artifactType = ArtifactType.APPLICATION) public Mono> pull(@PathVariable String branchedApplicationId) { log.info("Going to pull the latest for branchedApplicationId {}", branchedApplicationId); return centralGitService @@ -132,6 +140,7 @@ public class GitApplicationControllerCE { @JsonView(Views.Public.class) @GetMapping("/{branchedApplicationId}/status") + @GitRoute(fieldName = "branchedApplicationId", artifactType = ArtifactType.APPLICATION) public Mono> getStatus( @PathVariable String branchedApplicationId, @RequestParam(required = false, defaultValue = "true") Boolean compareRemote) { @@ -143,6 +152,7 @@ public class GitApplicationControllerCE { @JsonView(Views.Public.class) @GetMapping("/{referencedApplicationId}/fetch/remote") + @GitRoute(fieldName = "referencedApplicationId", artifactType = ArtifactType.APPLICATION) public Mono> fetchRemoteChanges( @PathVariable String referencedApplicationId, @RequestHeader(required = false, defaultValue = "branch") RefType refType) { @@ -154,6 +164,7 @@ public class GitApplicationControllerCE { @JsonView(Views.Public.class) @PostMapping("/{branchedApplicationId}/merge") + @GitRoute(fieldName = "branchedApplicationId", artifactType = ArtifactType.APPLICATION) public Mono> merge( @PathVariable String branchedApplicationId, @RequestBody GitMergeDTO gitMergeDTO) { log.debug( @@ -168,6 +179,7 @@ public class GitApplicationControllerCE { @JsonView(Views.Public.class) @PostMapping("/{branchedApplicationId}/merge/status") + @GitRoute(fieldName = "branchedApplicationId", artifactType = ArtifactType.APPLICATION) public Mono> mergeStatus( @PathVariable String branchedApplicationId, @RequestBody GitMergeDTO gitMergeDTO) { log.info( @@ -182,6 +194,7 @@ public class GitApplicationControllerCE { @JsonView(Views.Public.class) @DeleteMapping("/{baseArtifactId}/ref") + @GitRoute(fieldName = "baseArtifactId", artifactType = ArtifactType.APPLICATION) public Mono> deleteBranch( @PathVariable String baseArtifactId, @RequestParam String refName, @@ -194,6 +207,7 @@ public class GitApplicationControllerCE { @JsonView(Views.Public.class) @PutMapping("/{branchedApplicationId}/discard") + @GitRoute(fieldName = "branchedApplicationId", artifactType = ArtifactType.APPLICATION) public Mono> discardChanges(@PathVariable String branchedApplicationId) { log.info("Going to discard changes for branchedApplicationId {}", branchedApplicationId); return centralGitService @@ -203,6 +217,7 @@ public class GitApplicationControllerCE { @JsonView(Views.Public.class) @PostMapping("/{baseArtifactId}/protected-branches") + @GitRoute(fieldName = "baseArtifactId", artifactType = ArtifactType.APPLICATION) public Mono>> updateProtectedBranches( @PathVariable String baseArtifactId, @RequestBody @Valid BranchProtectionRequestDTO branchProtectionRequestDTO) { @@ -213,6 +228,7 @@ public class GitApplicationControllerCE { @JsonView(Views.Public.class) @GetMapping("/{baseArtifactId}/protected-branches") + @GitRoute(fieldName = "baseArtifactId", artifactType = ArtifactType.APPLICATION) public Mono>> getProtectedBranches(@PathVariable String baseArtifactId) { return centralGitService .getProtectedBranches(baseArtifactId, ARTIFACT_TYPE) @@ -221,6 +237,7 @@ public class GitApplicationControllerCE { @JsonView(Views.Public.class) @PostMapping("/{branchedApplicationId}/auto-commit") + @GitRoute(fieldName = "branchedApplicationId", artifactType = ArtifactType.APPLICATION) public Mono> autoCommitApplication(@PathVariable String branchedApplicationId) { return autoCommitService .autoCommitApplication(branchedApplicationId) @@ -229,6 +246,7 @@ public class GitApplicationControllerCE { @JsonView(Views.Public.class) @GetMapping("/{baseApplicationId}/auto-commit/progress") + @GitRoute(fieldName = "baseApplicationId", artifactType = ArtifactType.APPLICATION) public Mono> getAutoCommitProgress( @PathVariable String baseApplicationId, @RequestHeader(name = FieldName.BRANCH_NAME, required = false) String branchName) { @@ -239,6 +257,7 @@ public class GitApplicationControllerCE { @JsonView(Views.Public.class) @PatchMapping("/{baseArtifactId}/auto-commit/toggle") + @GitRoute(fieldName = "baseArtifactId", artifactType = ArtifactType.APPLICATION) public Mono> toggleAutoCommitEnabled(@PathVariable String baseArtifactId) { return centralGitService .toggleAutoCommitEnabled(baseArtifactId, ARTIFACT_TYPE) @@ -247,6 +266,7 @@ public class GitApplicationControllerCE { @JsonView(Views.Public.class) @GetMapping("/{branchedApplicationId}/refs") + @GitRoute(fieldName = "branchedApplicationId", artifactType = ArtifactType.APPLICATION) public Mono>> getReferences( @PathVariable String branchedApplicationId, @RequestParam(required = false, defaultValue = "branch") RefType refType, @@ -259,6 +279,7 @@ public class GitApplicationControllerCE { @JsonView(Views.Public.class) @GetMapping("/{branchedApplicationId}/ssh-keypair") + @GitRoute(fieldName = "branchedApplicationId", artifactType = ArtifactType.APPLICATION) public Mono> getSSHKey(@PathVariable String branchedApplicationId) { return artifactService .getSshKey(ARTIFACT_TYPE, branchedApplicationId) @@ -267,6 +288,7 @@ public class GitApplicationControllerCE { @JsonView(Views.Public.class) @PostMapping("/{branchedApplicationId}/ssh-keypair") + @GitRoute(fieldName = "branchedApplicationId", artifactType = ArtifactType.APPLICATION) public Mono> generateSSHKeyPair( @PathVariable String branchedApplicationId, @RequestParam(required = false) String keyType) { return artifactService diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/git/fs/GitFSServiceCECompatibleImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/git/fs/GitFSServiceCECompatibleImpl.java index c78337d603..f6746c2d1e 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/git/fs/GitFSServiceCECompatibleImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/git/fs/GitFSServiceCECompatibleImpl.java @@ -1,6 +1,7 @@ package com.appsmith.server.git.fs; import com.appsmith.external.git.handler.FSGitHandler; +import com.appsmith.git.configurations.GitServiceConfig; import com.appsmith.server.git.GitRedisUtils; import com.appsmith.server.git.central.GitHandlingServiceCECompatible; import com.appsmith.server.git.resolver.GitArtifactHelperResolver; @@ -28,7 +29,8 @@ public class GitFSServiceCECompatibleImpl extends GitFSServiceCEImpl implements FSGitHandler fsGitHandler, GitAnalyticsUtils gitAnalyticsUtils, GitArtifactHelperResolver gitArtifactHelperResolver, - FeatureFlagService featureFlagService) { + FeatureFlagService featureFlagService, + GitServiceConfig gitServiceConfig) { super( gitDeployKeysRepository, commonGitFileUtils, @@ -39,6 +41,7 @@ public class GitFSServiceCECompatibleImpl extends GitFSServiceCEImpl implements fsGitHandler, gitAnalyticsUtils, gitArtifactHelperResolver, - featureFlagService); + featureFlagService, + gitServiceConfig); } } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/git/fs/GitFSServiceCEImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/git/fs/GitFSServiceCEImpl.java index 4ecd79d268..3c9bba2d35 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/git/fs/GitFSServiceCEImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/git/fs/GitFSServiceCEImpl.java @@ -10,6 +10,7 @@ import com.appsmith.external.git.constants.GitSpan; import com.appsmith.external.git.constants.ce.RefType; import com.appsmith.external.git.dtos.FetchRemoteDTO; import com.appsmith.external.git.handler.FSGitHandler; +import com.appsmith.git.configurations.GitServiceConfig; import com.appsmith.git.dto.CommitDTO; import com.appsmith.server.constants.ArtifactType; import com.appsmith.server.domains.Artifact; @@ -79,6 +80,7 @@ public class GitFSServiceCEImpl implements GitHandlingServiceCE { protected final GitArtifactHelperResolver gitArtifactHelperResolver; private final FeatureFlagService featureFlagService; + private final GitServiceConfig gitServiceConfig; private static final String ORIGIN = "origin/"; private static final String REMOTE_NAME_REPLACEMENT = ""; @@ -698,6 +700,12 @@ public class GitFSServiceCEImpl implements GitHandlingServiceCE { ArtifactType artifactType = jsonTransformationDTO.getArtifactType(); GitArtifactHelper gitArtifactHelper = gitArtifactHelperResolver.getArtifactHelper(artifactType); Path repoSuffix = gitArtifactHelper.getRepoSuffixPath(workspaceId, baseArtifactId, repoName); + + if (gitServiceConfig.isGitInMemory()) { + return fsGitHandler.mergeBranch( + repoSuffix, gitMergeDTO.getSourceBranch(), gitMergeDTO.getDestinationBranch()); + } + Mono keepWorkingDirChangesMono = featureFlagService.check(FeatureFlagEnum.release_git_reset_optimization_enabled); diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/git/fs/GitFSServiceImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/git/fs/GitFSServiceImpl.java index 18c21d864e..d7b9ea93ea 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/git/fs/GitFSServiceImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/git/fs/GitFSServiceImpl.java @@ -1,6 +1,7 @@ package com.appsmith.server.git.fs; import com.appsmith.external.git.handler.FSGitHandler; +import com.appsmith.git.configurations.GitServiceConfig; import com.appsmith.server.git.GitRedisUtils; import com.appsmith.server.git.central.GitHandlingService; import com.appsmith.server.git.resolver.GitArtifactHelperResolver; @@ -28,7 +29,8 @@ public class GitFSServiceImpl extends GitFSServiceCECompatibleImpl implements Gi FSGitHandler fsGitHandler, GitAnalyticsUtils gitAnalyticsUtils, GitArtifactHelperResolver gitArtifactHelperResolver, - FeatureFlagService featureFlagService) { + FeatureFlagService featureFlagService, + GitServiceConfig gitServiceConfig) { super( gitDeployKeysRepository, commonGitFileUtils, @@ -39,6 +41,7 @@ public class GitFSServiceImpl extends GitFSServiceCECompatibleImpl implements Gi fsGitHandler, gitAnalyticsUtils, gitArtifactHelperResolver, - featureFlagService); + featureFlagService, + gitServiceConfig); } } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/RedisUtils.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/RedisUtils.java index 79ca677d38..1f6e2859fe 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/RedisUtils.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/RedisUtils.java @@ -1,5 +1,6 @@ package com.appsmith.server.helpers; +import com.appsmith.git.configurations.GitServiceConfig; import com.appsmith.server.exceptions.AppsmithError; import com.appsmith.server.exceptions.AppsmithException; import lombok.RequiredArgsConstructor; @@ -23,6 +24,7 @@ import static org.springframework.util.StringUtils.hasText; @Slf4j public class RedisUtils { private final ReactiveRedisOperations redisOperations; + private final GitServiceConfig gitServiceConfig; private static final String REDIS_FILE_LOCK_VALUE = "inUse"; @@ -47,7 +49,11 @@ public class RedisUtils { }); } + @Deprecated public Mono addFileLock(String key, Duration expirationPeriod, AppsmithException exception) { + if (gitServiceConfig.isGitInMemory()) { + return Mono.just(true); + } return redisOperations.hasKey(key).flatMap(isKeyPresent -> { if (Boolean.TRUE.equals(isKeyPresent)) { return Mono.error(exception); @@ -56,15 +62,27 @@ public class RedisUtils { }); } + @Deprecated public Mono releaseFileLock(String key) { + if (gitServiceConfig.isGitInMemory()) { + return Mono.just(true); + } return redisOperations.opsForValue().delete(key); } + @Deprecated public Mono hasKey(String key) { + if (gitServiceConfig.isGitInMemory()) { + return Mono.just(false); + } return redisOperations.hasKey(key); } + @Deprecated public Mono startAutoCommit(String defaultApplicationId, String branchName) { + if (gitServiceConfig.isGitInMemory()) { + return Mono.just(true); + } String key = String.format(AUTO_COMMIT_KEY_FORMAT, defaultApplicationId); return redisOperations.hasKey(key).flatMap(isKeyPresent -> { if (Boolean.TRUE.equals(isKeyPresent)) { @@ -84,12 +102,20 @@ public class RedisUtils { return redisOperations.opsForValue().get(key).map(Integer::valueOf); } + @Deprecated public Mono finishAutoCommit(String defaultApplicationId) { + if (gitServiceConfig.isGitInMemory()) { + return Mono.just(true); + } String key = String.format(AUTO_COMMIT_KEY_FORMAT, defaultApplicationId); return redisOperations.opsForValue().delete(key); } + @Deprecated public Mono getRunningAutoCommitBranchName(String defaultApplicationId) { + if (gitServiceConfig.isGitInMemory()) { + return Mono.empty(); + } String key = String.format(AUTO_COMMIT_KEY_FORMAT, defaultApplicationId); return redisOperations.hasKey(key).flatMap(hasKey -> { if (hasKey) { @@ -105,7 +131,11 @@ public class RedisUtils { * This would be required for whenever any attribute related to sessions becomes invalid at a systemic level. * Use with caution, every user will be logged out. */ + @Deprecated public Mono deleteAllSessionsIncludingCurrentUser() { + if (gitServiceConfig.isGitInMemory()) { + return Mono.empty(); + } AtomicInteger deletedKeysCount = new AtomicInteger(0); return redisOperations diff --git a/app/server/appsmith-server/src/main/resources/application-ce.properties b/app/server/appsmith-server/src/main/resources/application-ce.properties index 6805b8cea4..e22d2c50f2 100644 --- a/app/server/appsmith-server/src/main/resources/application-ce.properties +++ b/app/server/appsmith-server/src/main/resources/application-ce.properties @@ -66,6 +66,7 @@ sentry.environment=${APPSMITH_SERVER_SENTRY_ENVIRONMENT:} # Redis Properties appsmith.redis.url=${APPSMITH_REDIS_URL} +appsmith.redis.git.url=${APPSMITH_REDIS_GIT_URL:${APPSMITH_REDIS_URL}} # Mail Properties # Email defaults to false, because, when true and the other SMTP properties are not set, Spring will try to use a diff --git a/app/server/appsmith-server/src/test/java/com/appsmith/server/git/autocommit/AutoCommitServiceTest.java b/app/server/appsmith-server/src/test/java/com/appsmith/server/git/autocommit/AutoCommitServiceTest.java index 1c5809f347..26997da912 100644 --- a/app/server/appsmith-server/src/test/java/com/appsmith/server/git/autocommit/AutoCommitServiceTest.java +++ b/app/server/appsmith-server/src/test/java/com/appsmith/server/git/autocommit/AutoCommitServiceTest.java @@ -44,7 +44,6 @@ import org.springframework.boot.test.mock.mockito.SpyBean; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import reactor.test.StepVerifier; -import reactor.util.retry.Retry; import java.io.IOException; import java.net.URISyntaxException; @@ -527,168 +526,4 @@ public class AutoCommitServiceTest { }) .verifyComplete(); } - - @Test - public void - testAutoCommit_whenServerIsRunningMigrationCallsAutocommitAgainOnSameBranch_ReturnsAutoCommitInProgress() - throws URISyntaxException, IOException, GitAPIException { - - ApplicationJson applicationJson = - gitFileSystemTestHelper.getApplicationJson(this.getClass().getResource(APP_JSON_NAME)); - - mockAutoCommitTriggerResponse(TRUE, FALSE); - - ApplicationJson applicationJson1 = new ApplicationJson(); - AppsmithBeanUtils.copyNewFieldValuesIntoOldObject(applicationJson, applicationJson1); - applicationJson1.setServerSchemaVersion(jsonSchemaVersions.getServerVersion() + 1); - - doReturn(Mono.just(applicationJson1)) - .when(jsonSchemaMigration) - .migrateApplicationJsonToLatestSchema( - any(ApplicationJson.class), Mockito.anyString(), Mockito.anyString(), any(RefType.class)); - - gitFileSystemTestHelper.setupGitRepository( - WORKSPACE_ID, DEFAULT_APP_ID, BRANCH_NAME, REPO_NAME, applicationJson); - - // verifying the initial number of commits - StepVerifier.create(fsGitHandler.getCommitHistory(baseRepoSuffix)) - .assertNext(gitLogDTOs -> { - assertThat(gitLogDTOs).isNotEmpty(); - assertThat(gitLogDTOs.size()).isEqualTo(2); - - Set commitMessages = - gitLogDTOs.stream().map(GitLogDTO::getCommitMessage).collect(Collectors.toSet()); - assertThat(commitMessages).doesNotContain(String.format(AUTO_COMMIT_MSG_FORMAT, "UNKNOWN")); - }) - .verifyComplete(); - - // redis-utils fixing - Mockito.when(redisUtils.getRunningAutoCommitBranchName(DEFAULT_APP_ID)).thenReturn(Mono.empty()); - Mockito.when(redisUtils.getAutoCommitProgress(DEFAULT_APP_ID)).thenReturn(Mono.empty()); - - Mono autoCommitResponseDTOMono = - autoCommitService.autoCommitApplication(testApplication.getId()); - - StepVerifier.create(autoCommitResponseDTOMono) - .assertNext(autoCommitResponseDTO -> assertThat(autoCommitResponseDTO.getAutoCommitResponse()) - .isEqualTo(AutoCommitResponseDTO.AutoCommitResponse.PUBLISHED)) - .verifyComplete(); - - // redis-utils fixing - Mockito.when(redisUtils.getRunningAutoCommitBranchName(DEFAULT_APP_ID)).thenReturn(Mono.just(BRANCH_NAME)); - Mockito.when(redisUtils.getAutoCommitProgress(DEFAULT_APP_ID)).thenReturn(Mono.just(20)); - - StepVerifier.create(autoCommitService.autoCommitApplication(testApplication.getId())) - .assertNext(autoCommitResponseDTO -> { - assertThat(autoCommitResponseDTO.getAutoCommitResponse()) - .isEqualTo(AutoCommitResponseDTO.AutoCommitResponse.IN_PROGRESS); - assertThat(autoCommitResponseDTO.getBranchName()).isEqualTo(BRANCH_NAME); - }) - .verifyComplete(); - - // this would trigger autocommit - Mono> gitlogDTOsMono = Mono.delay(Duration.ofSeconds(WAIT_DURATION_FOR_ASYNC_EVENT)) - .then(fsGitHandler.getCommitHistory(baseRepoSuffix)); - - // verifying final number of commits - StepVerifier.create(gitlogDTOsMono) - .assertNext(gitLogDTOs -> { - assertThat(gitLogDTOs).isNotEmpty(); - assertThat(gitLogDTOs.size()).isEqualTo(3); - - Set commitMessages = - gitLogDTOs.stream().map(GitLogDTO::getCommitMessage).collect(Collectors.toSet()); - assertThat(commitMessages).contains(String.format(AUTO_COMMIT_MSG_FORMAT, "UNKNOWN")); - }) - .verifyComplete(); - } - - @Test - public void testAutoCommit_whenServerIsRunningMigrationCallsAutocommitAgainOnDiffBranch_ReturnsAutoCommitLocked() - throws URISyntaxException, IOException, GitAPIException { - - ApplicationJson applicationJson = - gitFileSystemTestHelper.getApplicationJson(this.getClass().getResource(APP_JSON_NAME)); - - // setup repository for test - gitFileSystemTestHelper.setupGitRepository( - WORKSPACE_ID, DEFAULT_APP_ID, BRANCH_NAME, REPO_NAME, applicationJson); - - ApplicationJson applicationJson1 = new ApplicationJson(); - AppsmithBeanUtils.copyNewFieldValuesIntoOldObject(applicationJson, applicationJson1); - applicationJson1.setServerSchemaVersion(jsonSchemaVersions.getServerVersion() + 1); - - // bump up server-version by one for metadata changes - doReturn(Mono.just(applicationJson1)) - .when(jsonSchemaMigration) - .migrateApplicationJsonToLatestSchema( - any(ApplicationJson.class), Mockito.anyString(), Mockito.anyString(), any(RefType.class)); - - // mock server migration as true and client migration as false - mockAutoCommitTriggerResponse(TRUE, FALSE); - - // verifying the initial number of commits - StepVerifier.create(fsGitHandler.getCommitHistory(baseRepoSuffix)) - .assertNext(gitLogDTOs -> { - assertThat(gitLogDTOs).isNotEmpty(); - assertThat(gitLogDTOs.size()).isEqualTo(2); - - Set commitMessages = - gitLogDTOs.stream().map(GitLogDTO::getCommitMessage).collect(Collectors.toSet()); - assertThat(commitMessages).doesNotContain(String.format(AUTO_COMMIT_MSG_FORMAT, "UNKNOWN")); - }) - .verifyComplete(); - - // redis-utils fixing - Mockito.when(redisUtils.getRunningAutoCommitBranchName(DEFAULT_APP_ID)).thenReturn(Mono.empty()); - Mockito.when(redisUtils.getAutoCommitProgress(DEFAULT_APP_ID)).thenReturn(Mono.empty()); - - Mono autoCommitResponseDTOMono = - autoCommitService.autoCommitApplication(testApplication.getId()); - - StepVerifier.create(autoCommitResponseDTOMono) - .assertNext(autoCommitResponseDTO -> assertThat(autoCommitResponseDTO.getAutoCommitResponse()) - .isEqualTo(AutoCommitResponseDTO.AutoCommitResponse.PUBLISHED)) - .verifyComplete(); - - testApplication.getGitApplicationMetadata().setRefName("another-branch-name"); - - // redis-utils fixing - Mockito.when(redisUtils.getRunningAutoCommitBranchName(DEFAULT_APP_ID)).thenReturn(Mono.just(BRANCH_NAME)); - Mockito.when(redisUtils.getAutoCommitProgress(DEFAULT_APP_ID)).thenReturn(Mono.just(20)); - - StepVerifier.create(autoCommitService.autoCommitApplication(testApplication.getId())) - .assertNext(autoCommitResponseDTO -> { - assertThat(autoCommitResponseDTO.getAutoCommitResponse()) - .isEqualTo(AutoCommitResponseDTO.AutoCommitResponse.LOCKED); - assertThat(autoCommitResponseDTO.getBranchName()).isEqualTo(BRANCH_NAME); - }) - .verifyComplete(); - - // wait for the event handler to complete the autocommit. - Mono> gitlogDTOsMono = Mono.delay(Duration.ofSeconds(WAIT_DURATION_FOR_ASYNC_EVENT)) - .then(fsGitHandler.getCommitHistory(baseRepoSuffix)); - - // verifying final number of commits - StepVerifier.create(gitlogDTOsMono) - .assertNext(gitLogDTOs -> { - assertThat(gitLogDTOs).isNotEmpty(); - assertThat(gitLogDTOs.size()).isEqualTo(3); - - Set commitMessages = - gitLogDTOs.stream().map(GitLogDTO::getCommitMessage).collect(Collectors.toSet()); - assertThat(commitMessages).contains(String.format(AUTO_COMMIT_MSG_FORMAT, "UNKNOWN")); - }) - .verifyComplete(); - } - - private Mono> getGitLog(Path artifactRepositorySuffix) { - return redisUtils - .getAutoCommitProgress(DEFAULT_APP_ID) - .retryWhen(Retry.fixedDelay(MAX_RETRIES, Duration.ofSeconds(RETRY_DELAY)) - .onRetryExhaustedThrow((retryBackoffSpec, retrySignal) -> { - throw new RuntimeException(); - })) - .then(fsGitHandler.getCommitHistory(baseRepoSuffix)); - } } diff --git a/app/server/appsmith-server/src/test/java/com/appsmith/server/git/autocommit/helpers/GitAutoCommitHelperImplTest.java b/app/server/appsmith-server/src/test/java/com/appsmith/server/git/autocommit/helpers/GitAutoCommitHelperImplTest.java index 711ba91af5..f080ac1ba1 100644 --- a/app/server/appsmith-server/src/test/java/com/appsmith/server/git/autocommit/helpers/GitAutoCommitHelperImplTest.java +++ b/app/server/appsmith-server/src/test/java/com/appsmith/server/git/autocommit/helpers/GitAutoCommitHelperImplTest.java @@ -31,7 +31,6 @@ import reactor.core.publisher.Mono; import reactor.test.StepVerifier; import static com.appsmith.server.dtos.AutoCommitResponseDTO.AutoCommitResponse.IDLE; -import static com.appsmith.server.dtos.AutoCommitResponseDTO.AutoCommitResponse.IN_PROGRESS; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; @@ -211,22 +210,6 @@ public class GitAutoCommitHelperImplTest { .verifyComplete(); } - @Test - public void getAutoCommitProgress_WhenAutoCommitRunning_ReturnsValidResponse() { - Mono progressDTOMono = redisUtils - .startAutoCommit(defaultApplicationId, branchName) - .then(redisUtils.setAutoCommitProgress(defaultApplicationId, 20)) - .then(gitAutoCommitHelper.getAutoCommitProgress(defaultApplicationId, branchName)); - - StepVerifier.create(progressDTOMono) - .assertNext(dto -> { - assertThat(dto.getAutoCommitResponse()).isEqualTo(IN_PROGRESS); - assertThat(dto.getProgress()).isEqualTo(20); - assertThat(dto.getBranchName()).isEqualTo(branchName); - }) - .verifyComplete(); - } - @Test public void getAutoCommitProgress_WhenNoAutoCommitFinished_ReturnsValidResponse() { Mono progressDTOMono = redisUtils diff --git a/app/server/appsmith-server/src/test/java/com/appsmith/server/git/resourcemap/ExchangeJsonConversionTests.java b/app/server/appsmith-server/src/test/java/com/appsmith/server/git/resourcemap/ExchangeJsonConversionTests.java index ff9163a4d4..44de6df971 100644 --- a/app/server/appsmith-server/src/test/java/com/appsmith/server/git/resourcemap/ExchangeJsonConversionTests.java +++ b/app/server/appsmith-server/src/test/java/com/appsmith/server/git/resourcemap/ExchangeJsonConversionTests.java @@ -11,13 +11,9 @@ import com.appsmith.server.helpers.CommonGitFileUtils; import com.appsmith.server.migrations.JsonSchemaMigration; import com.fasterxml.jackson.databind.MapperFeature; import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.commons.io.FileUtils; -import org.assertj.core.api.Assertions; -import org.eclipse.jgit.api.errors.GitAPIException; import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.TestTemplate; import org.junit.jupiter.api.extension.RegisterExtension; -import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.mock.mockito.SpyBean; @@ -28,8 +24,6 @@ import reactor.util.function.Tuple2; import java.io.IOException; import java.nio.charset.Charset; -import java.nio.file.Files; -import java.nio.file.Path; import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; @@ -121,28 +115,4 @@ public class ExchangeJsonConversionTests { templateProvider.assertResourceComparisons(originalArtifactJson, artifactExchangeJson); } - - @TestTemplate - public void testSerializeArtifactExchangeJson_whenArtifactIsFullyPopulated_returnsCorrespondingBaseRepoPath( - ExchangeJsonContext context) throws IOException, GitAPIException { - ArtifactExchangeJson originalArtifactJson = createArtifactJson(context).block(); - - Mockito.doReturn(Mono.just(true)) - .when(fsGitHandler) - .resetToLastCommit(Mockito.any(), Mockito.anyString(), Mockito.anyBoolean()); - - Files.createDirectories(Path.of("./container-volumes/git-storage/test123")); - - Mono responseMono = - commonGitFileUtils.saveArtifactToLocalRepoNew(Path.of("test123"), originalArtifactJson, "irrelevant"); - - StepVerifier.create(responseMono) - .assertNext(response -> { - Assertions.assertThat(response).isNotNull(); - }) - .verifyComplete(); - - FileUtils.deleteDirectory( - Path.of("./container-volumes/git-storage/test123").toFile()); - } } diff --git a/app/server/appsmith-server/src/test/resources/application-test.properties b/app/server/appsmith-server/src/test/resources/application-test.properties index 18d30ada04..878951df30 100644 --- a/app/server/appsmith-server/src/test/resources/application-test.properties +++ b/app/server/appsmith-server/src/test/resources/application-test.properties @@ -1,4 +1,4 @@ # embedded mongo DB version which is used during junit tests de.flapdoodle.mongodb.embedded.version=5.0.5 logging.level.root=error -appsmith.git.root=./container-volumes/git-storage/ +appsmith.git.root = /dev/shm/git-storage diff --git a/deploy/docker/fs/opt/appsmith/run-with-env.sh b/deploy/docker/fs/opt/appsmith/run-with-env.sh index bb62ce275d..8934ac7d7f 100755 --- a/deploy/docker/fs/opt/appsmith/run-with-env.sh +++ b/deploy/docker/fs/opt/appsmith/run-with-env.sh @@ -30,11 +30,12 @@ fi if [[ -z "${APPSMITH_GIT_ROOT:-}" ]]; then export APPSMITH_GIT_ROOT=/appsmith-stacks/git-storage -else - tlog "WARNING: It appears a custom value has been configured for APPSMITH_GIT_ROOT. This behaviour is deprecated and will soon be removed." >&2 fi + mkdir -pv "$APPSMITH_GIT_ROOT" +echo "APPSMITH_GIT_ROOT: ${APPSMITH_GIT_ROOT}" >&2 + # Check if APPSMITH_DB_URL is set if [[ -z "${APPSMITH_DB_URL}" ]]; then # If APPSMITH_DB_URL is not set, fall back to APPSMITH_MONGODB_URI