diff --git a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/plugins/PluginExecutor.java b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/plugins/PluginExecutor.java index f17d0ced2b..2e5e737e2a 100644 --- a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/plugins/PluginExecutor.java +++ b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/plugins/PluginExecutor.java @@ -8,10 +8,12 @@ import com.appsmith.external.models.DatasourceConfiguration; import com.appsmith.external.models.DatasourceStructure; import com.appsmith.external.models.DatasourceTestResult; import com.appsmith.external.models.Param; +import com.appsmith.external.models.Property; import org.pf4j.ExtensionPoint; import org.springframework.util.CollectionUtils; import reactor.core.publisher.Mono; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -93,6 +95,19 @@ public interface PluginExecutor extends ExtensionPoint { return Mono.empty(); } + /** + * This function executes the DB query to fetch details about the datasource when we don't want to create new action + * just to get the information about the datasource + * e.g. Get Spreadsheets from Google Drive, Get first row in datasource etc. + * + * @param pluginSpecifiedTemplates + * @param datasourceConfiguration + * @return + */ + default Mono getDatasourceMetadata(List pluginSpecifiedTemplates, DatasourceConfiguration datasourceConfiguration) { + return Mono.empty(); + } + /** * Appsmith Server calls this function for execution of the action. * Default implementation which takes the variables that need to be substituted and then calls the plugin execute function diff --git a/app/server/appsmith-plugins/googleSheetsPlugin/src/main/java/com/external/config/InfoMethod.java b/app/server/appsmith-plugins/googleSheetsPlugin/src/main/java/com/external/config/InfoMethod.java index ec85c1ebe6..24b5d13ce2 100644 --- a/app/server/appsmith-plugins/googleSheetsPlugin/src/main/java/com/external/config/InfoMethod.java +++ b/app/server/appsmith-plugins/googleSheetsPlugin/src/main/java/com/external/config/InfoMethod.java @@ -2,11 +2,22 @@ package com.external.config; import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginError; import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginException; +import com.appsmith.external.models.OAuth2; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import org.springframework.http.HttpMethod; import org.springframework.web.reactive.function.BodyInserters; import org.springframework.web.reactive.function.client.WebClient; import org.springframework.web.util.UriComponentsBuilder; +import reactor.core.Exceptions; +import reactor.core.publisher.Mono; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; /** * API reference: https://developers.google.com/drive/api/v3/reference/files/get @@ -20,6 +31,55 @@ public class InfoMethod implements Method { this.objectMapper = objectMapper; } + @Override + public Mono executePrerequisites(MethodConfig methodConfig, OAuth2 oauth2) { + WebClient client = WebClient.builder() + .exchangeStrategies(EXCHANGE_STRATEGIES) + .build(); + UriComponentsBuilder uriBuilder = getBaseUriBuilder(this.BASE_SHEETS_API_URL, + methodConfig.getSpreadsheetId()); + uriBuilder.queryParam("fields", "sheets/properties"); + return client.method(HttpMethod.GET) + .uri(uriBuilder.build(false).toUri()) + .body(BodyInserters.empty()) + .headers(headers -> headers.set( + "Authorization", + "Bearer " + oauth2.getAuthenticationResponse().getToken())) + .exchange() + .flatMap(clientResponse -> clientResponse.toEntity(byte[].class)) + .map(response -> {// Choose body depending on response status + byte[] responseBody = response.getBody(); + + if (responseBody == null || !response.getStatusCode().is2xxSuccessful()) { + throw Exceptions.propagate(new AppsmithPluginException( + AppsmithPluginError.PLUGIN_ERROR, + "Could not map request back to existing data")); + } + String jsonBody = new String(responseBody); + JsonNode sheets = null; + try { + sheets = objectMapper.readTree(jsonBody).get("sheets"); + } catch (IOException e) { + throw Exceptions.propagate(new AppsmithPluginException( + AppsmithPluginError.PLUGIN_JSON_PARSE_ERROR, + new String(responseBody), + e.getMessage() + )); + } + + assert sheets != null; + List sheetMetadata = new ArrayList<>(); + for (JsonNode sheet : sheets) { + final JsonNode properties = sheet.get("properties"); + if (!properties.get("title").asText().isEmpty()) { + sheetMetadata.add(properties); + } + } + methodConfig.setBody(sheetMetadata); + return methodConfig; + }); + } + @Override public boolean validateMethodRequest(MethodConfig methodConfig) { if (methodConfig.getSpreadsheetId() == null || methodConfig.getSpreadsheetId().isBlank()) { @@ -41,4 +101,24 @@ public class InfoMethod implements Method { .body(BodyInserters.empty()); } + @Override + public JsonNode transformResponse(JsonNode response, MethodConfig methodConfig) { + if (response == null) { + throw new AppsmithPluginException( + AppsmithPluginError.PLUGIN_ERROR, + "Missing a valid response object."); + } + + Map responseObj = new HashMap<>(); + if (methodConfig.getBody() instanceof List) { + responseObj.put("sheets", methodConfig.getBody()); + } + Iterator fieldNames = response.fieldNames(); + while(fieldNames.hasNext()) { + String fieldName = fieldNames.next(); + responseObj.put(fieldName, response.get(fieldName)); + } + return this.objectMapper.valueToTree(responseObj); + } + } diff --git a/app/server/appsmith-plugins/googleSheetsPlugin/src/main/java/com/external/plugins/GoogleSheetsPlugin.java b/app/server/appsmith-plugins/googleSheetsPlugin/src/main/java/com/external/plugins/GoogleSheetsPlugin.java index e1134ef260..e39643d0b4 100644 --- a/app/server/appsmith-plugins/googleSheetsPlugin/src/main/java/com/external/plugins/GoogleSheetsPlugin.java +++ b/app/server/appsmith-plugins/googleSheetsPlugin/src/main/java/com/external/plugins/GoogleSheetsPlugin.java @@ -183,5 +183,13 @@ public class GoogleSheetsPlugin extends BasePlugin { // This plugin would not have the option to test return Mono.just(new DatasourceTestResult()); } + + @Override + public Mono getDatasourceMetadata(List pluginSpecifiedTemplates, + DatasourceConfiguration datasourceConfiguration) { + ActionConfiguration actionConfiguration = new ActionConfiguration(); + actionConfiguration.setPluginSpecifiedTemplates(pluginSpecifiedTemplates); + return execute(null, datasourceConfiguration, actionConfiguration); + } } } \ No newline at end of file diff --git a/app/server/appsmith-plugins/googleSheetsPlugin/src/test/java/com/external/config/InfoMethodTest.java b/app/server/appsmith-plugins/googleSheetsPlugin/src/test/java/com/external/config/InfoMethodTest.java new file mode 100644 index 0000000000..5922614359 --- /dev/null +++ b/app/server/appsmith-plugins/googleSheetsPlugin/src/test/java/com/external/config/InfoMethodTest.java @@ -0,0 +1,86 @@ +package com.external.config; + +import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginException; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.Assert; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +public class InfoMethodTest { + + @Test(expected = AppsmithPluginException.class) + public void testTransformResponse_missingJSON_throwsException() { + ObjectMapper objectMapper = new ObjectMapper(); + + InfoMethod infoMethod = new InfoMethod(objectMapper); + infoMethod.transformResponse(null, null); + } + + @Test + public void testTransformResponse_missingSheets_returnsEmpty() throws JsonProcessingException { + ObjectMapper objectMapper = new ObjectMapper(); + + final String jsonString = "{\"key1\":\"value1\",\"key2\":\"value2\"}"; + + JsonNode jsonNode = objectMapper.readTree(jsonString); + Assert.assertNotNull(jsonNode); + + MethodConfig methodConfig = new MethodConfig(new ArrayList<>()); + JsonNode sheetNode = objectMapper.readTree(""); + methodConfig.setBody(sheetNode); + + InfoMethod infoMethod = new InfoMethod(objectMapper); + JsonNode result = infoMethod.transformResponse(jsonNode, methodConfig); + + Assert.assertNotNull(result); + Assert.assertTrue(result.isObject()); + Assert.assertEquals(null, result.get("sheets")); + } + + @Test + public void testTransformResponse_emptySheets_returnsEmpty() throws JsonProcessingException { + ObjectMapper objectMapper = new ObjectMapper(); + + final String jsonString = "{\"key1\":\"value1\",\"key2\":\"value2\"}"; + + JsonNode jsonNode = objectMapper.readTree(jsonString); + Assert.assertNotNull(jsonNode); + + MethodConfig methodConfig = new MethodConfig(new ArrayList<>()); + methodConfig.setBody(new ArrayList<>()); + + InfoMethod infoMethod = new InfoMethod(objectMapper); + JsonNode result = infoMethod.transformResponse(jsonNode, methodConfig); + + Assert.assertNotNull(result); + Assert.assertTrue(result.isObject()); + Assert.assertEquals(0, result.get("sheets").size()); + } + + @Test + public void testTransformResponse_validSheets_toListOfSheets() throws JsonProcessingException { + ObjectMapper objectMapper = new ObjectMapper(); + + final String jsonString = "{\"key1\":\"value1\",\"key2\":\"value2\"}"; + final String sheetMetadataString = "{\"sheetId\":\"1\", \"title\":\"test\", \"sheetType\":\"GRID\", \"index\":0}"; + + JsonNode jsonNode = objectMapper.readTree(jsonString); + Assert.assertNotNull(jsonNode); + + MethodConfig methodConfig = new MethodConfig(new ArrayList<>()); + JsonNode sheetNode = objectMapper.readTree(sheetMetadataString); + methodConfig.setBody(List.of(sheetNode)); + + InfoMethod infoMethod = new InfoMethod(objectMapper); + JsonNode result = infoMethod.transformResponse(jsonNode, methodConfig); + + Assert.assertNotNull(result); + Assert.assertTrue(result.isObject()); + Assert.assertEquals(1, result.get("sheets").size()); + Assert.assertTrue("test".equalsIgnoreCase(result.get("sheets").get(0).get("title").asText())); + } +} diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/controllers/DatasourceController.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/controllers/DatasourceController.java index 1c8c526387..6c6f86c5b1 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/controllers/DatasourceController.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/controllers/DatasourceController.java @@ -1,7 +1,9 @@ package com.appsmith.server.controllers; +import com.appsmith.external.models.ActionExecutionResult; import com.appsmith.external.models.DatasourceStructure; import com.appsmith.external.models.DatasourceTestResult; +import com.appsmith.external.models.Property; import com.appsmith.server.constants.Url; import com.appsmith.server.domains.Datasource; import com.appsmith.server.dtos.AuthorizationCodeCallbackDTO; @@ -20,6 +22,7 @@ import org.springframework.http.HttpStatus; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.PutMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; @@ -27,6 +30,7 @@ import org.springframework.web.bind.annotation.RestController; import org.springframework.web.server.ServerWebExchange; import reactor.core.publisher.Mono; +import javax.validation.Valid; import java.net.URI; import java.util.List; @@ -103,4 +107,12 @@ public class DatasourceController extends BaseController new ResponseDTO<>(HttpStatus.OK.value(), datasource, null)); } + @PutMapping("/datasource-query/{datasourceId}") + public Mono> runQueryOnDatasource(@PathVariable String datasourceId, + @Valid @RequestBody List pluginSpecifiedTemplates) { + log.debug("Getting datasource metadata"); + return datasourceStructureSolution.getDatasourceMetadata(datasourceId, pluginSpecifiedTemplates) + .map(metadata -> new ResponseDTO<>(HttpStatus.OK.value(), metadata, null)); + } + } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/solutions/DatasourceStructureSolution.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/solutions/DatasourceStructureSolution.java index 3f9f3f7801..353559e0c3 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/solutions/DatasourceStructureSolution.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/solutions/DatasourceStructureSolution.java @@ -3,9 +3,13 @@ package com.appsmith.server.solutions; import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginError; import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginException; import com.appsmith.external.exceptions.pluginExceptions.StaleConnectionException; +import com.appsmith.external.models.ActionExecutionResult; +import com.appsmith.external.models.AuthenticationDTO; import com.appsmith.external.models.DatasourceStructure; +import com.appsmith.external.models.Property; import com.appsmith.external.plugins.PluginExecutor; import com.appsmith.external.services.EncryptionService; +import com.appsmith.server.acl.AclPermission; import com.appsmith.server.constants.FieldName; import com.appsmith.server.domains.Datasource; import com.appsmith.server.exceptions.AppsmithError; @@ -22,8 +26,11 @@ import org.springframework.util.CollectionUtils; import reactor.core.publisher.Mono; import java.time.Duration; +import java.util.List; import java.util.concurrent.TimeoutException; +import static com.appsmith.external.models.AuthenticationDTO.AuthenticationStatus.SUCCESS; + @Component @RequiredArgsConstructor @Slf4j @@ -125,4 +132,47 @@ public class DatasourceStructureSolution { : datasourceRepository.saveStructure(datasource.getId(), structure).thenReturn(structure) ); } + + /** + * This function will be used to execute queries on datasource without creating the new action + * e.g. get all spreadsheets from google drive, fetch 1st row from the table etc + * @param datasourceId + * @param pluginSpecifiedTemplates + * @return + */ + public Mono getDatasourceMetadata(String datasourceId, List pluginSpecifiedTemplates) { + + /* + 1. Check if the datasource is present + 2. Check plugin is present + 3. Execute DB query from the information provided present in pluginSpecifiedTemplates + */ + Mono datasourceMono = datasourceService.findById(datasourceId, AclPermission.MANAGE_DATASOURCES) + .switchIfEmpty(Mono.error(new AppsmithException( + AppsmithError.ACL_NO_RESOURCE_FOUND, FieldName.DATASOURCE, datasourceId + ))); + + return datasourceMono.flatMap(datasource -> { + + AuthenticationDTO auth = datasource.getDatasourceConfiguration() == null + || datasource.getDatasourceConfiguration().getAuthentication() == null ? + null : datasource.getDatasourceConfiguration().getAuthentication(); + if (auth == null || !SUCCESS.equals(auth.getAuthenticationStatus())) { + // Don't attempt to run query for invalid datasources. + return Mono.error(new AppsmithException( + AppsmithError.INVALID_DATASOURCE, + datasource.getName(), + "Authentication failure for datasource, please re-authenticate and try again!" + )); + } + // check if the plugin is present and call method from plugin executor + return pluginExecutorHelper + .getPluginExecutor(pluginService.findById(datasource.getPluginId())) + .switchIfEmpty(Mono.error(new AppsmithException(AppsmithError.NO_RESOURCE_FOUND, FieldName.PLUGIN, datasource.getPluginId()))) + .flatMap(pluginExecutor -> + pluginExecutor.getDatasourceMetadata(pluginSpecifiedTemplates, datasource.getDatasourceConfiguration()) + ) + .timeout(Duration.ofSeconds(GET_STRUCTURE_TIMEOUT_SECONDS)); + }); + } }