diff --git a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/Provider.java b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/Provider.java index 946364cfe9..42fdd36c3d 100644 --- a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/Provider.java +++ b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/Provider.java @@ -41,6 +41,9 @@ public class Provider extends BaseDomain { String planSubscribed; + /** + * TODO : Once the marketplace is up, remove the default values from here. Let the Marketplace handle the defaults. + */ Boolean isVisible = true; Integer sortOrder = 1000; diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/configurations/MarketplaceConfig.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/configurations/MarketplaceConfig.java new file mode 100644 index 0000000000..53aac0f72c --- /dev/null +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/configurations/MarketplaceConfig.java @@ -0,0 +1,12 @@ +package com.appsmith.server.configurations; + +import lombok.Getter; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Configuration; + +@Configuration +@Getter +public class MarketplaceConfig { + @Value("${marketplace.base-url}") + String base_url; +} diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/constants/FieldName.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/constants/FieldName.java index 06252c3c62..41f3ce3d70 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/constants/FieldName.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/constants/FieldName.java @@ -22,4 +22,6 @@ public class FieldName { public static String USER = "user"; public static String PROVIDER_ID = "providerId"; public static String CATEGORY = "category"; + public static String PAGE = "page"; + public static String SIZE = "size"; } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/controllers/ApiTemplateController.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/controllers/ApiTemplateController.java index 61f40cf8bc..3646d854f8 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/controllers/ApiTemplateController.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/controllers/ApiTemplateController.java @@ -3,11 +3,13 @@ package com.appsmith.server.controllers; import com.appsmith.external.models.ApiTemplate; import com.appsmith.server.constants.Url; import com.appsmith.server.services.ApiTemplateService; +import lombok.extern.slf4j.Slf4j; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; @RestController @RequestMapping(Url.API_TEMPLATE_URL) +@Slf4j public class ApiTemplateController extends BaseController { public ApiTemplateController(ApiTemplateService service) { super(service); diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/controllers/MarketplaceController.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/controllers/MarketplaceController.java index 814b4817e3..02621536ea 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/controllers/MarketplaceController.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/controllers/MarketplaceController.java @@ -13,11 +13,14 @@ import com.appsmith.server.domains.Action; import com.appsmith.server.domains.Datasource; import com.appsmith.server.dtos.ResponseDTO; import com.appsmith.server.dtos.SearchResponseDTO; +import com.appsmith.server.services.MarketplaceService; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.extern.slf4j.Slf4j; import org.springframework.http.HttpMethod; import org.springframework.http.HttpStatus; +import org.springframework.util.MultiValueMap; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; @@ -29,11 +32,15 @@ import java.util.List; @RestController @RequestMapping(Url.MARKETPLACE_URL) +@Slf4j public class MarketplaceController { private final ObjectMapper objectMapper; + private final MarketplaceService marketplaceService; - public MarketplaceController(ObjectMapper objectMapper) { + public MarketplaceController(ObjectMapper objectMapper, + MarketplaceService marketplaceService) { this.objectMapper = objectMapper; + this.marketplaceService = marketplaceService; } @GetMapping("/search") @@ -111,4 +118,26 @@ public class MarketplaceController { .map(result -> new ResponseDTO<>(HttpStatus.OK.value(), result, null)); } + + @GetMapping("/templates") + public Mono>> getAllTemplatesFromMarketplace(@RequestParam MultiValueMap params) { + log.debug("Going to get all templates from Marketplace"); + return marketplaceService.getTemplates(params) + .map(resources -> new ResponseDTO<>(HttpStatus.OK.value(), resources, null)); + } + + @GetMapping("/providers") + public Mono>> getAllProvidersFromMarketplace(@RequestParam MultiValueMap params) { + log.debug("Going to get all providers from Marketplace"); + return marketplaceService.getProviders(params) + .map(resources -> new ResponseDTO<>(HttpStatus.OK.value(), resources, null)); + } + + @GetMapping("/categories") + public Mono>> getAllCategoriesFromMarketplace() { + log.debug("Going to get all categories from Marketplace"); + return marketplaceService.getCategories() + .map(resources -> new ResponseDTO<>(HttpStatus.OK.value(), resources, null)); + } + } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/controllers/ProviderController.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/controllers/ProviderController.java index 999a2652bb..b4ecd55d4e 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/controllers/ProviderController.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/controllers/ProviderController.java @@ -4,6 +4,7 @@ import com.appsmith.external.models.Provider; import com.appsmith.server.constants.Url; import com.appsmith.server.dtos.ResponseDTO; import com.appsmith.server.services.ProviderService; +import lombok.extern.slf4j.Slf4j; import org.springframework.http.HttpStatus; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestMapping; @@ -14,8 +15,8 @@ import java.util.List; @RestController @RequestMapping(Url.PROVIDER_URL) +@Slf4j public class ProviderController extends BaseController { - public ProviderController(ProviderService service) { super(service); } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/exceptions/AppsmithError.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/exceptions/AppsmithError.java index f4969b4dd7..d898992001 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/exceptions/AppsmithError.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/exceptions/AppsmithError.java @@ -24,7 +24,8 @@ public enum AppsmithError { NO_CONFIGURATION_FOUND_IN_ACTION(400, 4016, "No action configuration found. Please configure it and try again."), NAME_CLASH_NOT_ALLOWED_IN_REFACTOR(400, 4017, "The new name {1} already exists in the current page. Choose another name."), PAGE_DOESNT_BELONG_TO_APPLICATION(400, 4018, "Page {0} does not belong to the application {1}"), - NO_DSL_FOUND_IN_PAGE(400, 4029, "The page {0} doesn't have a DSL. This is an unexpected state"), + PAGINATED_API_PAGE_SIZE_MISSING(400, 4019, "This is a paginated API. Page and size are mandatory paramters"), + NO_DSL_FOUND_IN_PAGE(400, 4020, "The page {0} doesn't have a DSL. This is an unexpected state"), UNAUTHORIZED_DOMAIN(401, 4019, "Invalid email domain provided. Please sign in with a valid work email ID"), INVALID_PASSWORD_RESET(400, 4020, "Unable to reset the password. Please initiate a request via 'forgot password' link to reset your password"), LOGIN_INTERNAL_ERROR(401, 4021, "Internal error while trying to login"), @@ -39,7 +40,8 @@ public enum AppsmithError { REPOSITORY_SAVE_FAILED(500, 5001, "Failed to save the repository. Try again."), PLUGIN_INSTALLATION_FAILED_DOWNLOAD_ERROR(500, 5002, "Plugin installation failed due to an error while downloading it. Check the jar location & try again."), PLUGIN_RUN_FAILED(500, 5003, "Plugin execution failed with error {0}"), - PLUGIN_EXECUTION_TIMEOUT(504, 5040, "Plugin Execution exceeded the maximum allowed time. Please increase the timeout in your action settings or check your backend action endpoint"); + PLUGIN_EXECUTION_TIMEOUT(504, 5040, "Plugin Execution exceeded the maximum allowed time. Please increase the timeout in your action settings or check your backend action endpoint"), + MARKETPLACE_TIMEOUT(504, 5041, "Marketplace is responding too slowly. Please check the internet connection"); private Integer httpErrorCode; diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ItemServiceImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ItemServiceImpl.java index 7562cfd0af..ad672f31aa 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ItemServiceImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ItemServiceImpl.java @@ -22,17 +22,17 @@ public class ItemServiceImpl implements ItemService { private final ApiTemplateService apiTemplateService; private final ActionService actionService; private final PluginService pluginService; - private final ProviderService providerService; + private final MarketplaceService marketplaceService; private static final String RAPID_API_PLUGIN = "rapidapi-plugin"; public ItemServiceImpl(ApiTemplateService apiTemplateService, ActionService actionService, PluginService pluginService, - ProviderService providerService) { + MarketplaceService marketplaceService) { this.apiTemplateService = apiTemplateService; this.actionService = actionService; this.pluginService = pluginService; - this.providerService = providerService; + this.marketplaceService = marketplaceService; } @Override @@ -78,9 +78,6 @@ public class ItemServiceImpl implements ItemService { documentation.setText(apiTemplate.getApiTemplateConfiguration().getDocumentation()); documentation.setUrl(apiTemplate.getApiTemplateConfiguration().getDocumentationUrl()); action.setDocumentation(documentation); - /** TODO - * Also hit the Marketplace to update the number of imports. - */ // Set Action Fields action.setActionConfiguration(apiTemplate.getActionConfiguration()); @@ -89,12 +86,15 @@ public class ItemServiceImpl implements ItemService { action.setCacheResponse(apiTemplate.getApiTemplateConfiguration().getSampleResponse().getBody().toString()); } - return pluginService + return marketplaceService + // First hit the marketplace to update the statistics and to subscribe to the provider in case it hasn't + .subscribeAndUpdateStatisticsOfProvider(action.getProviderId()) + // Assume that we are only adding rapid api templates right now. Set the package to rapid-api forcibly /** TODO * Scraper should set the correct package name (rapidapi-plugin) instead of restapi-plugin */ - .findByPackageName(RAPID_API_PLUGIN) + .then(pluginService.findByPackageName(RAPID_API_PLUGIN)) .map(plugin -> { //Set Datasource Datasource datasource = new Datasource(); diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/MarketplaceService.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/MarketplaceService.java new file mode 100644 index 0000000000..1e008af49c --- /dev/null +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/MarketplaceService.java @@ -0,0 +1,15 @@ +package com.appsmith.server.services; + +import com.appsmith.external.models.ApiTemplate; +import com.appsmith.external.models.Provider; +import org.springframework.util.MultiValueMap; +import reactor.core.publisher.Mono; + +import java.util.List; + +public interface MarketplaceService { + Mono> getProviders(MultiValueMap params); + Mono> getTemplates(MultiValueMap params); + Mono> getCategories(); + Mono subscribeAndUpdateStatisticsOfProvider(String providerId); +} diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/MarketplaceServiceImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/MarketplaceServiceImpl.java new file mode 100644 index 0000000000..6917a14a83 --- /dev/null +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/MarketplaceServiceImpl.java @@ -0,0 +1,164 @@ +package com.appsmith.server.services; + +import com.appsmith.external.models.ApiTemplate; +import com.appsmith.external.models.Provider; +import com.appsmith.server.configurations.MarketplaceConfig; +import com.appsmith.server.constants.FieldName; +import com.appsmith.server.exceptions.AppsmithError; +import com.appsmith.server.exceptions.AppsmithException; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; +import org.springframework.web.reactive.function.client.WebClient; +import org.springframework.web.util.UriComponentsBuilder; +import reactor.core.publisher.Mono; + +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URLEncoder; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; + +@Slf4j +@Service +public class MarketplaceServiceImpl implements MarketplaceService { + private final WebClient webClient; + + private final MarketplaceConfig marketplaceConfig; + + private static String PROVIDER_PATH = "/providers"; + + private static String TEMPLATE_PATH = "/templates"; + + private static String CATEGORIES_PATH = PROVIDER_PATH + "/categories"; + + private static String MARKETPLACE_USERNAME = "appsmith-server"; + + private static String MARKETPLACE_PASSWORD = "g:bj{64<$[k>hHBV"; + + private final ObjectMapper objectMapper; + + private final Long timeoutInMillis = Long.valueOf(10000); + + @Autowired + public MarketplaceServiceImpl(WebClient.Builder webClientBuilder, + MarketplaceConfig marketplaceConfig, ObjectMapper objectMapper) { + this.marketplaceConfig = marketplaceConfig; + this.webClient = webClientBuilder + .defaultHeaders(header -> header.setBasicAuth(MARKETPLACE_USERNAME, MARKETPLACE_PASSWORD)) + .baseUrl(marketplaceConfig.getBase_url()) + .build(); + this.objectMapper = objectMapper; + } + + @Override + public Mono> getProviders(MultiValueMap params) { + if (params.getFirst(FieldName.PAGE) == null || params.getFirst(FieldName.SIZE) == null) { + return Mono.error(new AppsmithException(AppsmithError.PAGINATED_API_PAGE_SIZE_MISSING)); + } + + URI uri = buildFullURI(params, PROVIDER_PATH); + + return webClient + .get() + .uri(uri) + .retrieve() + .bodyToMono(String.class) + .flatMap(stringBody -> { + List providerList = null; + try { + providerList = objectMapper.readValue(stringBody, ArrayList.class); + } catch (JsonProcessingException e) { + return Mono.error(new AppsmithException(AppsmithError.JSON_PROCESSING_ERROR, e)); + } + return Mono.just(providerList); + }) + .timeout(Duration.ofMillis(timeoutInMillis)) + .doOnError(error -> Mono.error(new AppsmithException(AppsmithError.MARKETPLACE_TIMEOUT))); + } + + @Override + public Mono> getTemplates(MultiValueMap params) { + URI uri = buildFullURI(params, TEMPLATE_PATH); + + return webClient + .get() + .uri(uri) + .retrieve() + .bodyToMono(String.class) + .flatMap(stringBody -> { + List templates = null; + try { + templates = objectMapper.readValue(stringBody, ArrayList.class); + } catch (JsonProcessingException e) { + return Mono.error(new AppsmithException(AppsmithError.JSON_PROCESSING_ERROR, e)); + } + return Mono.just(templates); + }) + .timeout(Duration.ofMillis(timeoutInMillis)) + .doOnError(error -> Mono.error(new AppsmithException(AppsmithError.MARKETPLACE_TIMEOUT))); + } + + @Override + public Mono> getCategories() { + URI uri = buildFullURI(null, CATEGORIES_PATH); + + return webClient + .get() + .uri(uri) + .retrieve() + .bodyToMono(String.class) + .flatMap(stringBody -> { + List categories = null; + try { + categories = objectMapper.readValue(stringBody, ArrayList.class); + } catch (JsonProcessingException e) { + return Mono.error(new AppsmithException(AppsmithError.JSON_PROCESSING_ERROR, e)); + } + return Mono.just(categories); + }) + .timeout(Duration.ofMillis(timeoutInMillis)) + .doOnError(error -> Mono.error(new AppsmithException(AppsmithError.MARKETPLACE_TIMEOUT))); + } + + @Override + public Mono subscribeAndUpdateStatisticsOfProvider(String providerId) { + MultiValueMap params = new LinkedMultiValueMap<>(); + params.add("id", providerId); + URI uri = buildFullURI(params, PROVIDER_PATH); + + if (uri == null) { + // Throw an internal server error because the URL is hard coded and must be correct + return Mono.error(new AppsmithException(AppsmithError.INTERNAL_SERVER_ERROR)); + } + + return webClient + .get() + .uri(uri) + .retrieve() + .bodyToMono(Provider.class); + } + + private URI buildFullURI(MultiValueMap params, String path) { + UriComponentsBuilder uriBuilder = UriComponentsBuilder.newInstance(); + try { + uriBuilder.uri(new URI(marketplaceConfig.getBase_url() + path)); + } catch (URISyntaxException e) { + log.error(e.getMessage()); + return null; + } + + if (params != null) { + for (String key : params.keySet()) { + uriBuilder.queryParam(key, URLEncoder.encode(params.getFirst(key))); + } + } + + return uriBuilder.build(true).toUri(); + } +} diff --git a/app/server/appsmith-server/src/main/resources/application-docker.properties b/app/server/appsmith-server/src/main/resources/application-docker.properties index 61c66e085c..de929b1b0f 100644 --- a/app/server/appsmith-server/src/main/resources/application-docker.properties +++ b/app/server/appsmith-server/src/main/resources/application-docker.properties @@ -8,6 +8,8 @@ spring.data.mongodb.port=27017 #spring.data.mongodb.username= #spring.data.mongodb.password= +marketplace.base-url = http://localhost:8081/api/v1 + # Log properties logging.level.root=info logging.level.com.appsmith=debug diff --git a/app/server/appsmith-server/src/main/resources/application-local.properties b/app/server/appsmith-server/src/main/resources/application-local.properties index 9ddf6b8227..e7f2142013 100644 --- a/app/server/appsmith-server/src/main/resources/application-local.properties +++ b/app/server/appsmith-server/src/main/resources/application-local.properties @@ -8,6 +8,8 @@ spring.data.mongodb.port=27017 #spring.data.mongodb.username= #spring.data.mongodb.password= +marketplace.base-url = http://localhost:8081/api/v1 + # Log properties logging.level.root=info logging.level.com.appsmith=debug diff --git a/app/server/appsmith-server/src/main/resources/application-staging.properties b/app/server/appsmith-server/src/main/resources/application-staging.properties index 9c6c8af343..a2bf381ef1 100644 --- a/app/server/appsmith-server/src/main/resources/application-staging.properties +++ b/app/server/appsmith-server/src/main/resources/application-staging.properties @@ -6,6 +6,8 @@ spring.data.mongodb.database=mobtools spring.data.mongodb.uri=mongodb+srv://admin:Y9PuxM52gcP3Dgfo@mobtools-test-cluster-swrsq.mongodb.net/mobtools?retryWrites=true&minPoolSize=1&maxPoolSize=10&maxIdleTimeMS=900000 spring.data.mongodb.authentication-database=admin +marketplace.base-url = http://localhost:8081/api/v1 + # Log properties logging.level.root=info logging.level.com.appsmith=debug diff --git a/app/server/appsmith-server/src/test/resources/application-test.properties b/app/server/appsmith-server/src/test/resources/application-test.properties index 0f3fde00b4..6c9209ad65 100644 --- a/app/server/appsmith-server/src/test/resources/application-test.properties +++ b/app/server/appsmith-server/src/test/resources/application-test.properties @@ -5,6 +5,8 @@ spring.data.mongodb.port=27017 #spring.data.mongodb.username= #spring.data.mongodb.password= +marketplace.base-url = http://localhost:8081/api/v1 + # Log properties logging.level.root=error logging.level.com.appsmith=debug