Protei/PromucFlow_constructor
2
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Fetching the providers from the Marketplace using webclient. Introduced a temporary end point to support the new source for Providers

Browse Source
This commit is contained in:
Trisha Anand 2020-04-01 18:53:39 +00:00
parent f51751aba8
commit f882ffcdf3
14 changed files with 250 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/Provider.java vendored
View File

@ -41,6 +41,9 @@ public class Provider extends BaseDomain {
String planSubscribed;
/**
* TODO : Once the marketplace is up, remove the default values from here. Let the Marketplace handle the defaults.
*/
Boolean isVisible = true;
Integer sortOrder = 1000;

12
app/server/appsmith-server/src/main/java/com/appsmith/server/configurations/MarketplaceConfig.java Normal file
View File

@ -0,0 +1,12 @@
package com.appsmith.server.configurations;
import lombok.Getter;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Configuration;
@Configuration
@Getter
public class MarketplaceConfig {
@Value("${marketplace.base-url}")
String base_url;
}

2
app/server/appsmith-server/src/main/java/com/appsmith/server/constants/FieldName.java
View File

@ -22,4 +22,6 @@ public class FieldName {
public static String USER = "user";
public static String PROVIDER_ID = "providerId";
public static String CATEGORY = "category";
public static String PAGE = "page";
public static String SIZE = "size";
}

2
app/server/appsmith-server/src/main/java/com/appsmith/server/controllers/ApiTemplateController.java
View File

@ -3,11 +3,13 @@ package com.appsmith.server.controllers;
import com.appsmith.external.models.ApiTemplate;
import com.appsmith.server.constants.Url;
import com.appsmith.server.services.ApiTemplateService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
@RestController
@RequestMapping(Url.API_TEMPLATE_URL)
@Slf4j
public class ApiTemplateController extends BaseController<ApiTemplateService, ApiTemplate, String> {
public ApiTemplateController(ApiTemplateService service) {
super(service);

31
app/server/appsmith-server/src/main/java/com/appsmith/server/controllers/MarketplaceController.java
View File

@ -13,11 +13,14 @@ import com.appsmith.server.domains.Action;
import com.appsmith.server.domains.Datasource;
import com.appsmith.server.dtos.ResponseDTO;
import com.appsmith.server.dtos.SearchResponseDTO;
import com.appsmith.server.services.MarketplaceService;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.extern.slf4j.Slf4j;
import org.springframework.http.HttpMethod;
import org.springframework.http.HttpStatus;
import org.springframework.util.MultiValueMap;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
@ -29,11 +32,15 @@ import java.util.List;
@RestController
@RequestMapping(Url.MARKETPLACE_URL)
@Slf4j
public class MarketplaceController {
private final ObjectMapper objectMapper;
private final MarketplaceService marketplaceService;
public MarketplaceController(ObjectMapper objectMapper) {
public MarketplaceController(ObjectMapper objectMapper,
MarketplaceService marketplaceService) {
this.objectMapper = objectMapper;
this.marketplaceService = marketplaceService;
}
@GetMapping("/search")
@ -111,4 +118,26 @@ public class MarketplaceController {
.map(result -> new ResponseDTO<>(HttpStatus.OK.value(), result, null));
}
@GetMapping("/templates")
public Mono<ResponseDTO<List<ApiTemplate>>> getAllTemplatesFromMarketplace(@RequestParam MultiValueMap<String, String> params) {
log.debug("Going to get all templates from Marketplace");
return marketplaceService.getTemplates(params)
.map(resources -> new ResponseDTO<>(HttpStatus.OK.value(), resources, null));
}
@GetMapping("/providers")
public Mono<ResponseDTO<List<Provider>>> getAllProvidersFromMarketplace(@RequestParam MultiValueMap<String, String> params) {
log.debug("Going to get all providers from Marketplace");
return marketplaceService.getProviders(params)
.map(resources -> new ResponseDTO<>(HttpStatus.OK.value(), resources, null));
}
@GetMapping("/categories")
public Mono<ResponseDTO<List<String>>> getAllCategoriesFromMarketplace() {
log.debug("Going to get all categories from Marketplace");
return marketplaceService.getCategories()
.map(resources -> new ResponseDTO<>(HttpStatus.OK.value(), resources, null));
}
}

3
app/server/appsmith-server/src/main/java/com/appsmith/server/controllers/ProviderController.java
View File

@ -4,6 +4,7 @@ import com.appsmith.external.models.Provider;
import com.appsmith.server.constants.Url;
import com.appsmith.server.dtos.ResponseDTO;
import com.appsmith.server.services.ProviderService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
@ -14,8 +15,8 @@ import java.util.List;
@RestController
@RequestMapping(Url.PROVIDER_URL)
@Slf4j
public class ProviderController extends BaseController<ProviderService, Provider, String> {
public ProviderController(ProviderService service) {
super(service);
}

6
app/server/appsmith-server/src/main/java/com/appsmith/server/exceptions/AppsmithError.java
View File

@ -24,7 +24,8 @@ public enum AppsmithError {
NO_CONFIGURATION_FOUND_IN_ACTION(400, 4016, "No action configuration found. Please configure it and try again."),
NAME_CLASH_NOT_ALLOWED_IN_REFACTOR(400, 4017, "The new name {1} already exists in the current page. Choose another name."),
PAGE_DOESNT_BELONG_TO_APPLICATION(400, 4018, "Page {0} does not belong to the application {1}"),
NO_DSL_FOUND_IN_PAGE(400, 4029, "The page {0} doesn't have a DSL. This is an unexpected state"),
PAGINATED_API_PAGE_SIZE_MISSING(400, 4019, "This is a paginated API. Page and size are mandatory paramters"),
NO_DSL_FOUND_IN_PAGE(400, 4020, "The page {0} doesn't have a DSL. This is an unexpected state"),
UNAUTHORIZED_DOMAIN(401, 4019, "Invalid email domain provided. Please sign in with a valid work email ID"),
INVALID_PASSWORD_RESET(400, 4020, "Unable to reset the password. Please initiate a request via 'forgot password' link to reset your password"),
LOGIN_INTERNAL_ERROR(401, 4021, "Internal error while trying to login"),
@ -39,7 +40,8 @@ public enum AppsmithError {
REPOSITORY_SAVE_FAILED(500, 5001, "Failed to save the repository. Try again."),
PLUGIN_INSTALLATION_FAILED_DOWNLOAD_ERROR(500, 5002, "Plugin installation failed due to an error while downloading it. Check the jar location & try again."),
PLUGIN_RUN_FAILED(500, 5003, "Plugin execution failed with error {0}"),
PLUGIN_EXECUTION_TIMEOUT(504, 5040, "Plugin Execution exceeded the maximum allowed time. Please increase the timeout in your action settings or check your backend action endpoint");
PLUGIN_EXECUTION_TIMEOUT(504, 5040, "Plugin Execution exceeded the maximum allowed time. Please increase the timeout in your action settings or check your backend action endpoint"),
MARKETPLACE_TIMEOUT(504, 5041, "Marketplace is responding too slowly. Please check the internet connection");
private Integer httpErrorCode;

16
app/server/appsmith-server/src/main/java/com/appsmith/server/services/ItemServiceImpl.java
View File

@ -22,17 +22,17 @@ public class ItemServiceImpl implements ItemService {
private final ApiTemplateService apiTemplateService;
private final ActionService actionService;
private final PluginService pluginService;
private final ProviderService providerService;
private final MarketplaceService marketplaceService;
private static final String RAPID_API_PLUGIN = "rapidapi-plugin";
public ItemServiceImpl(ApiTemplateService apiTemplateService,
ActionService actionService,
PluginService pluginService,
ProviderService providerService) {
MarketplaceService marketplaceService) {
this.apiTemplateService = apiTemplateService;
this.actionService = actionService;
this.pluginService = pluginService;
this.providerService = providerService;
this.marketplaceService = marketplaceService;
}
@Override
@ -78,9 +78,6 @@ public class ItemServiceImpl implements ItemService {
documentation.setText(apiTemplate.getApiTemplateConfiguration().getDocumentation());
documentation.setUrl(apiTemplate.getApiTemplateConfiguration().getDocumentationUrl());
action.setDocumentation(documentation);
/** TODO
* Also hit the Marketplace to update the number of imports.
*/
// Set Action Fields
action.setActionConfiguration(apiTemplate.getActionConfiguration());
@ -89,12 +86,15 @@ public class ItemServiceImpl implements ItemService {
action.setCacheResponse(apiTemplate.getApiTemplateConfiguration().getSampleResponse().getBody().toString());
}
return pluginService
return marketplaceService
// First hit the marketplace to update the statistics and to subscribe to the provider in case it hasn't
.subscribeAndUpdateStatisticsOfProvider(action.getProviderId())
// Assume that we are only adding rapid api templates right now. Set the package to rapid-api forcibly
/** TODO
* Scraper should set the correct package name (rapidapi-plugin) instead of restapi-plugin
*/
.findByPackageName(RAPID_API_PLUGIN)
.then(pluginService.findByPackageName(RAPID_API_PLUGIN))
.map(plugin -> {
//Set Datasource
Datasource datasource = new Datasource();

15
app/server/appsmith-server/src/main/java/com/appsmith/server/services/MarketplaceService.java Normal file
View File

@ -0,0 +1,15 @@
package com.appsmith.server.services;
import com.appsmith.external.models.ApiTemplate;
import com.appsmith.external.models.Provider;
import org.springframework.util.MultiValueMap;
import reactor.core.publisher.Mono;
import java.util.List;
public interface MarketplaceService {
Mono<List<Provider>> getProviders(MultiValueMap<String, String> params);
Mono<List<ApiTemplate>> getTemplates(MultiValueMap<String, String> params);
Mono<List<String>> getCategories();
Mono<Provider> subscribeAndUpdateStatisticsOfProvider(String providerId);
}

164
app/server/appsmith-server/src/main/java/com/appsmith/server/services/MarketplaceServiceImpl.java Normal file
View File

@ -0,0 +1,164 @@
package com.appsmith.server.services;
import com.appsmith.external.models.ApiTemplate;
import com.appsmith.external.models.Provider;
import com.appsmith.server.configurations.MarketplaceConfig;
import com.appsmith.server.constants.FieldName;
import com.appsmith.server.exceptions.AppsmithError;
import com.appsmith.server.exceptions.AppsmithException;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap;
import org.springframework.web.reactive.function.client.WebClient;
import org.springframework.web.util.UriComponentsBuilder;
import reactor.core.publisher.Mono;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URLEncoder;
import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
@Slf4j
@Service
public class MarketplaceServiceImpl implements MarketplaceService {
private final WebClient webClient;
private final MarketplaceConfig marketplaceConfig;
private static String PROVIDER_PATH = "/providers";
private static String TEMPLATE_PATH = "/templates";
private static String CATEGORIES_PATH = PROVIDER_PATH + "/categories";
private static String MARKETPLACE_USERNAME = "appsmith-server";
private static String MARKETPLACE_PASSWORD = "g:bj{64<$[k>hHBV";
private final ObjectMapper objectMapper;
private final Long timeoutInMillis = Long.valueOf(10000);
@Autowired
public MarketplaceServiceImpl(WebClient.Builder webClientBuilder,
MarketplaceConfig marketplaceConfig, ObjectMapper objectMapper) {
this.marketplaceConfig = marketplaceConfig;
this.webClient = webClientBuilder
.defaultHeaders(header -> header.setBasicAuth(MARKETPLACE_USERNAME, MARKETPLACE_PASSWORD))
.baseUrl(marketplaceConfig.getBase_url())
.build();
this.objectMapper = objectMapper;
}
@Override
public Mono<List<Provider>> getProviders(MultiValueMap<String, String> params) {
if (params.getFirst(FieldName.PAGE) == null || params.getFirst(FieldName.SIZE) == null) {
return Mono.error(new AppsmithException(AppsmithError.PAGINATED_API_PAGE_SIZE_MISSING));
}
URI uri = buildFullURI(params, PROVIDER_PATH);
return webClient
.get()
.uri(uri)
.retrieve()
.bodyToMono(String.class)
.flatMap(stringBody -> {
List<Provider> providerList = null;
try {
providerList = objectMapper.readValue(stringBody, ArrayList.class);
} catch (JsonProcessingException e) {
return Mono.error(new AppsmithException(AppsmithError.JSON_PROCESSING_ERROR, e));
}
return Mono.just(providerList);
})
.timeout(Duration.ofMillis(timeoutInMillis))
.doOnError(error -> Mono.error(new AppsmithException(AppsmithError.MARKETPLACE_TIMEOUT)));
}
@Override
public Mono<List<ApiTemplate>> getTemplates(MultiValueMap<String, String> params) {
URI uri = buildFullURI(params, TEMPLATE_PATH);
return webClient
.get()
.uri(uri)
.retrieve()
.bodyToMono(String.class)
.flatMap(stringBody -> {
List<ApiTemplate> templates = null;
try {
templates = objectMapper.readValue(stringBody, ArrayList.class);
} catch (JsonProcessingException e) {
return Mono.error(new AppsmithException(AppsmithError.JSON_PROCESSING_ERROR, e));
}
return Mono.just(templates);
})
.timeout(Duration.ofMillis(timeoutInMillis))
.doOnError(error -> Mono.error(new AppsmithException(AppsmithError.MARKETPLACE_TIMEOUT)));
}
@Override
public Mono<List<String>> getCategories() {
URI uri = buildFullURI(null, CATEGORIES_PATH);
return webClient
.get()
.uri(uri)
.retrieve()
.bodyToMono(String.class)
.flatMap(stringBody -> {
List<String> categories = null;
try {
categories = objectMapper.readValue(stringBody, ArrayList.class);
} catch (JsonProcessingException e) {
return Mono.error(new AppsmithException(AppsmithError.JSON_PROCESSING_ERROR, e));
}
return Mono.just(categories);
})
.timeout(Duration.ofMillis(timeoutInMillis))
.doOnError(error -> Mono.error(new AppsmithException(AppsmithError.MARKETPLACE_TIMEOUT)));
}
@Override
public Mono<Provider> subscribeAndUpdateStatisticsOfProvider(String providerId) {
MultiValueMap<String, String> params = new LinkedMultiValueMap<>();
params.add("id", providerId);
URI uri = buildFullURI(params, PROVIDER_PATH);
if (uri == null) {
// Throw an internal server error because the URL is hard coded and must be correct
return Mono.error(new AppsmithException(AppsmithError.INTERNAL_SERVER_ERROR));
}
return webClient
.get()
.uri(uri)
.retrieve()
.bodyToMono(Provider.class);
}
private URI buildFullURI(MultiValueMap<String, String> params, String path) {
UriComponentsBuilder uriBuilder = UriComponentsBuilder.newInstance();
try {
uriBuilder.uri(new URI(marketplaceConfig.getBase_url() + path));
} catch (URISyntaxException e) {
log.error(e.getMessage());
return null;
}
if (params != null) {
for (String key : params.keySet()) {
uriBuilder.queryParam(key, URLEncoder.encode(params.getFirst(key)));
}
}
return uriBuilder.build(true).toUri();
}
}

2
app/server/appsmith-server/src/main/resources/application-docker.properties
View File

@ -8,6 +8,8 @@ spring.data.mongodb.port=27017
#spring.data.mongodb.username=
#spring.data.mongodb.password=
marketplace.base-url = http://localhost:8081/api/v1
# Log properties
logging.level.root=info
logging.level.com.appsmith=debug

2
app/server/appsmith-server/src/main/resources/application-local.properties
View File

@ -8,6 +8,8 @@ spring.data.mongodb.port=27017
#spring.data.mongodb.username=
#spring.data.mongodb.password=
marketplace.base-url = http://localhost:8081/api/v1
# Log properties
logging.level.root=info
logging.level.com.appsmith=debug

2
app/server/appsmith-server/src/main/resources/application-staging.properties
View File

@ -6,6 +6,8 @@ spring.data.mongodb.database=mobtools
spring.data.mongodb.uri=mongodb+srv://admin:Y9PuxM52gcP3Dgfo@mobtools-test-cluster-swrsq.mongodb.net/mobtools?retryWrites=true&minPoolSize=1&maxPoolSize=10&maxIdleTimeMS=900000
spring.data.mongodb.authentication-database=admin
marketplace.base-url = http://localhost:8081/api/v1
# Log properties
logging.level.root=info
logging.level.com.appsmith=debug

2
app/server/appsmith-server/src/test/resources/application-test.properties
View File

@ -5,6 +5,8 @@ spring.data.mongodb.port=27017
#spring.data.mongodb.username=
#spring.data.mongodb.password=
marketplace.base-url = http://localhost:8081/api/v1
# Log properties
logging.level.root=error
logging.level.com.appsmith=debug