Protei/PromucFlow_constructor
2
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

fix: Connection timeout for git connect when the git repo is behind a firewall (#12819)

Browse Source 
* Use non blocking IO for checking the repo status

* Cache the results

* Update tests for the util class method
This commit is contained in:
Anagh Hegde 2022-04-13 16:03:28 +05:30 committed by GitHub
parent 2abb7bb822
commit 5c17422956
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 81 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/GitUtils.java
View File

@ -1,13 +1,19 @@
package com.appsmith.server.helpers;
import com.appsmith.server.dtos.ResponseDTO;
import com.appsmith.server.exceptions.AppsmithError;
import com.appsmith.server.exceptions.AppsmithException;
import org.eclipse.jgit.util.StringUtils;
import org.springframework.core.ParameterizedTypeReference;
import org.springframework.web.reactive.function.client.WebClient;
import reactor.core.publisher.Mono;
import reactor.netty.http.client.HttpClientRequest;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.URL;
import java.time.Duration;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@ -57,12 +63,23 @@ public class GitUtils {
* @return if the repo is public
* @throws IOException exception thrown during openConnection
*/
public static boolean isRepoPrivate(String remoteHttpsUrl) throws IOException {
URL url = new URL(remoteHttpsUrl);
HttpURLConnection huc = (HttpURLConnection) url.openConnection();
int responseCode = huc.getResponseCode();
return !(HttpURLConnection.HTTP_OK == responseCode || HttpURLConnection.HTTP_ACCEPTED == responseCode);
public static Mono<Boolean> isRepoPrivate(String remoteHttpsUrl) {
return WebClient
.create(remoteHttpsUrl)
.get()
.httpRequest(httpRequest -> {
HttpClientRequest reactorRequest = httpRequest.getNativeRequest();
reactorRequest.responseTimeout(Duration.ofSeconds(2));
})
.exchange()
.flatMap(response -> {
if (response.statusCode().is2xxSuccessful()) {
return Mono.just(Boolean.FALSE);
} else {
return Mono.just(Boolean.TRUE);
}
})
.onErrorResume(throwable -> Mono.just(Boolean.TRUE));
}
/**

87
app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/GitServiceCEImpl.java
View File

@ -394,20 +394,19 @@ public class GitServiceCEImpl implements GitServiceCE {
// Check if the repo is public for current application and if the user have changed the access after
// the connection
final Boolean isRepoPrivate = defaultGitMetadata.getIsRepoPrivate();
Mono<Application> applicationMono = Mono.just(defaultApplication);
Mono<Application> applicationMono;
if (Boolean.FALSE.equals(isRepoPrivate)) {
try {
defaultGitMetadata.setIsRepoPrivate(
GitUtils.isRepoPrivate(defaultGitMetadata.getBrowserSupportedRemoteUrl())
);
if (!isRepoPrivate.equals(defaultGitMetadata.getIsRepoPrivate())) {
applicationMono = applicationService.save(defaultApplication);
} else {
return applicationMono;
}
} catch (IOException e) {
log.debug("Error while checking if the repo is private: ", e);
}
applicationMono = GitUtils.isRepoPrivate(defaultGitMetadata.getBrowserSupportedRemoteUrl())
.flatMap(isPrivate -> {
defaultGitMetadata.setIsRepoPrivate(isPrivate);
if (!isPrivate.equals(defaultGitMetadata.getIsRepoPrivate())) {
return applicationService.save(defaultApplication);
} else {
return Mono.just(defaultApplication);
}
});
} else {
return Mono.just(defaultApplication);
}
// Check if the private repo count is less than the allowed repo count
@ -643,17 +642,15 @@ public class GitServiceCEImpl implements GitServiceCE {
.switchIfEmpty(Mono.error(new AppsmithException(AppsmithError.INVALID_GIT_CONFIGURATION, GIT_PROFILE_ERROR)));
final String browserSupportedUrl = GitUtils.convertSshUrlToBrowserSupportedUrl(gitConnectDTO.getRemoteUrl());
boolean isRepoPrivateTemp = true;
try {
isRepoPrivateTemp = GitUtils.isRepoPrivate(browserSupportedUrl);
} catch (IOException e) {
log.debug("Error while checking if the repo is private: ", e);
}
final boolean isRepoPrivate = isRepoPrivateTemp;
Mono<Boolean> isPrivateRepoMono = GitUtils.isRepoPrivate(browserSupportedUrl).cache();
Mono<Application> connectApplicationMono = profileMono
.then(getApplicationById(defaultApplicationId))
.flatMap(application -> {
.then(getApplicationById(defaultApplicationId).zipWith(isPrivateRepoMono))
.flatMap(tuple -> {
Application application = tuple.getT1();
boolean isRepoPrivate = tuple.getT2();
// Check if the repo is public
if(!isRepoPrivate) {
return Mono.just(application);
@ -733,8 +730,10 @@ public class GitServiceCEImpl implements GitServiceCE {
final String applicationId = application.getId();
final String orgId = application.getOrganizationId();
try {
return fileUtils.checkIfDirectoryIsEmpty(repoPath)
.flatMap(isEmpty -> {
return fileUtils.checkIfDirectoryIsEmpty(repoPath).zipWith(isPrivateRepoMono)
.flatMap(objects -> {
boolean isEmpty = objects.getT1();
boolean isRepoPrivate = objects.getT2();
if (!isEmpty) {
return addAnalyticsForGitOperation(
AnalyticsEvents.GIT_CONNECT.getEventName(),
@ -1825,25 +1824,21 @@ public class GitServiceCEImpl implements GitServiceCE {
return Mono.error(new AppsmithException(AppsmithError.INVALID_PARAMETER, "Invalid organization id"));
}
boolean isRepoPrivateTemp;
try {
isRepoPrivateTemp = GitUtils.isRepoPrivate(GitUtils.convertSshUrlToBrowserSupportedUrl(gitConnectDTO.getRemoteUrl()));
} catch (IOException e) {
log.error("Error while checking if the repo is private: ", e);
isRepoPrivateTemp = true;
}
final boolean isRepoPrivate = isRepoPrivateTemp;
final String repoName = GitUtils.getRepoName(gitConnectDTO.getRemoteUrl());
Mono<Boolean> isPrivateRepoMono = GitUtils.isRepoPrivate(GitUtils.convertSshUrlToBrowserSupportedUrl(gitConnectDTO.getRemoteUrl())).cache();
Mono<ApplicationImportDTO> importedApplicationMono = getSSHKeyForCurrentUser()
.zipWith(isPrivateRepoMono)
.switchIfEmpty(Mono.error(new AppsmithException(AppsmithError.INVALID_GIT_CONFIGURATION,
"Unable to find git configuration for logged-in user. Please contact Appsmith team for support")))
//Check the limit for number of private repo
.flatMap(gitAuth -> {
.flatMap(tuple -> {
// Check if the repo is public
Application newApplication = new Application();
newApplication.setName(repoName);
newApplication.setOrganizationId(organizationId);
newApplication.setGitApplicationMetadata(new GitApplicationMetadata());
GitAuth gitAuth = tuple.getT1();
boolean isRepoPrivate = tuple.getT2();
Mono<Application> applicationMono = applicationPageService
.createOrUpdateSuffixedApplication(newApplication, newApplication.getName(), 0);
if(!isRepoPrivate) {
@ -1905,7 +1900,10 @@ public class GitServiceCEImpl implements GitServiceCE {
});
return defaultBranchMono
.flatMap(defaultBranch -> {
.zipWith(isPrivateRepoMono)
.flatMap(tuple2 -> {
String defaultBranch = tuple2.getT1();
boolean isRepoPrivate = tuple2.getT2();
GitApplicationMetadata gitApplicationMetadata = new GitApplicationMetadata();
gitApplicationMetadata.setGitAuth(gitAuth);
gitApplicationMetadata.setDefaultApplicationId(application.getId());
@ -2300,19 +2298,14 @@ public class GitServiceCEImpl implements GitServiceCE {
.flatMap(application -> {
GitApplicationMetadata gitData = application.getGitApplicationMetadata();
final Boolean isRepoPrivate = gitData.getIsRepoPrivate();
try {
// Check if user have altered the repo accessibility
gitData.setIsRepoPrivate(
GitUtils.isRepoPrivate(application.getGitApplicationMetadata().getBrowserSupportedRemoteUrl())
);
if (!isRepoPrivate.equals(gitData.getIsRepoPrivate())) {
// Repo accessibility is changed
return applicationService.save(application);
}
} catch (IOException e) {
log.debug("Error while checking if the repo is private: ", e);
}
return Mono.just(application);
return GitUtils.isRepoPrivate(application.getGitApplicationMetadata().getBrowserSupportedRemoteUrl())
.flatMap(isPrivate -> {
if (!isRepoPrivate.equals(gitData.getIsRepoPrivate())) {
// Repo accessibility is changed
return applicationService.save(application);
}
return Mono.just(application);
});
})
.then(applicationService.getGitConnectedApplicationsCountWithPrivateRepoByOrgId(organizationId));
}

27
app/server/appsmith-server/src/test/java/com/appsmith/server/helpers/GitUtilsTest.java
View File

@ -4,6 +4,7 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringRunner;
import reactor.test.StepVerifier;
import java.io.IOException;
@ -28,15 +29,23 @@ public class GitUtilsTest {
.isEqualTo("https://example.test.net/user/test/tests/testRepo");
}
@Test
public void isRepoPrivate() throws IOException {
assertThat(GitUtils.isRepoPrivate(GitUtils.convertSshUrlToBrowserSupportedUrl("git@example.com:test/testRepo.git")))
.isEqualTo(Boolean.TRUE);
assertThat(GitUtils.isRepoPrivate(GitUtils.convertSshUrlToBrowserSupportedUrl("git@example.com:test/testRepo.git")))
.isEqualTo(Boolean.TRUE);
assertThat(GitUtils.isRepoPrivate(GitUtils.convertSshUrlToBrowserSupportedUrl("git@example.org:test/testRepo.git")))
.isEqualTo(Boolean.TRUE);
assertThat(GitUtils.isRepoPrivate(GitUtils.convertSshUrlToBrowserSupportedUrl("ssh://git@appsmith.com.git")))
.isEqualTo(Boolean.FALSE);
public void isRepoPrivate() {
StepVerifier
.create(GitUtils.isRepoPrivate(GitUtils.convertSshUrlToBrowserSupportedUrl("git@github.com:test/testRepo.git")))
.assertNext(isRepoPrivate -> assertThat(isRepoPrivate).isEqualTo(Boolean.TRUE))
.verifyComplete();
StepVerifier
.create(GitUtils.isRepoPrivate(GitUtils.convertSshUrlToBrowserSupportedUrl("ssh://git@example.test.net:user/test/tests/testRepo.git")))
.assertNext(isRepoPrivate -> assertThat(isRepoPrivate).isEqualTo(Boolean.TRUE))
.verifyComplete();
StepVerifier
.create(GitUtils.isRepoPrivate(GitUtils.convertSshUrlToBrowserSupportedUrl("git@github.com:appsmithorg/appsmith.git")))
.assertNext(isRepoPrivate -> assertThat(isRepoPrivate).isEqualTo(Boolean.FALSE))
.verifyComplete();
}
@Test

1
app/server/appsmith-server/src/test/java/com/appsmith/server/services/GitServiceTest.java
View File

@ -731,7 +731,6 @@ public class GitServiceTest {
Application application1 = this.createApplicationConnectedToGit("private_repo_1", "master", limitPrivateRepoTestOrgId);
this.createApplicationConnectedToGit("private_repo_2", "master", limitPrivateRepoTestOrgId);
this.createApplicationConnectedToGit("private_repo_3", "master", limitPrivateRepoTestOrgId);
Application testApplication = new Application();
GitApplicationMetadata gitApplicationMetadata = new GitApplicationMetadata();