Protei/PromucFlow_constructor
2
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Google sheet APIs for getting all the spreadsheet, getting all sheets within spreadsheet and column headers (#5875)

Browse Source 
* API to run DB query using plugin specified templates

* Included get spreadsheet metadata in get info method

* Added TCs for checking Spreadsheet info response

* Added error message for invalid datasources

* Authentication check for datasource modified to AuthenticationStatus field
This commit is contained in:
Abhijeet 2021-07-16 16:15:29 +05:30 committed by GitHub
parent 3a4de769ee
commit c5a909ddc8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 251 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
app/server/appsmith-interfaces/src/main/java/com/appsmith/external/plugins/PluginExecutor.java vendored
View File

@ -8,10 +8,12 @@ import com.appsmith.external.models.DatasourceConfiguration;
import com.appsmith.external.models.DatasourceStructure;
import com.appsmith.external.models.DatasourceTestResult;
import com.appsmith.external.models.Param;
import com.appsmith.external.models.Property;
import org.pf4j.ExtensionPoint;
import org.springframework.util.CollectionUtils;
import reactor.core.publisher.Mono;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
@ -93,6 +95,19 @@ public interface PluginExecutor<C> extends ExtensionPoint {
return Mono.empty();
}
/**
* This function executes the DB query to fetch details about the datasource when we don't want to create new action
* just to get the information about the datasource
* e.g. Get Spreadsheets from Google Drive, Get first row in datasource etc.
*
* @param pluginSpecifiedTemplates
* @param datasourceConfiguration
* @return
*/
default Mono<ActionExecutionResult> getDatasourceMetadata(List<Property> pluginSpecifiedTemplates, DatasourceConfiguration datasourceConfiguration) {
return Mono.empty();
}
/**
* Appsmith Server calls this function for execution of the action.
* Default implementation which takes the variables that need to be substituted and then calls the plugin execute function

80
app/server/appsmith-plugins/googleSheetsPlugin/src/main/java/com/external/config/InfoMethod.java vendored
View File

@ -2,11 +2,22 @@ package com.external.config;
import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginError;
import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginException;
import com.appsmith.external.models.OAuth2;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.springframework.http.HttpMethod;
import org.springframework.web.reactive.function.BodyInserters;
import org.springframework.web.reactive.function.client.WebClient;
import org.springframework.web.util.UriComponentsBuilder;
import reactor.core.Exceptions;
import reactor.core.publisher.Mono;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
/**
* API reference: https://developers.google.com/drive/api/v3/reference/files/get
@ -20,6 +31,55 @@ public class InfoMethod implements Method {
this.objectMapper = objectMapper;
}
@Override
public Mono<Object> executePrerequisites(MethodConfig methodConfig, OAuth2 oauth2) {
WebClient client = WebClient.builder()
.exchangeStrategies(EXCHANGE_STRATEGIES)
.build();
UriComponentsBuilder uriBuilder = getBaseUriBuilder(this.BASE_SHEETS_API_URL,
methodConfig.getSpreadsheetId());
uriBuilder.queryParam("fields", "sheets/properties");
return client.method(HttpMethod.GET)
.uri(uriBuilder.build(false).toUri())
.body(BodyInserters.empty())
.headers(headers -> headers.set(
"Authorization",
"Bearer " + oauth2.getAuthenticationResponse().getToken()))
.exchange()
.flatMap(clientResponse -> clientResponse.toEntity(byte[].class))
.map(response -> {// Choose body depending on response status
byte[] responseBody = response.getBody();
if (responseBody == null || !response.getStatusCode().is2xxSuccessful()) {
throw Exceptions.propagate(new AppsmithPluginException(
AppsmithPluginError.PLUGIN_ERROR,
"Could not map request back to existing data"));
}
String jsonBody = new String(responseBody);
JsonNode sheets = null;
try {
sheets = objectMapper.readTree(jsonBody).get("sheets");
} catch (IOException e) {
throw Exceptions.propagate(new AppsmithPluginException(
AppsmithPluginError.PLUGIN_JSON_PARSE_ERROR,
new String(responseBody),
e.getMessage()
));
}
assert sheets != null;
List<JsonNode> sheetMetadata = new ArrayList<>();
for (JsonNode sheet : sheets) {
final JsonNode properties = sheet.get("properties");
if (!properties.get("title").asText().isEmpty()) {
sheetMetadata.add(properties);
}
}
methodConfig.setBody(sheetMetadata);
return methodConfig;
});
}
@Override
public boolean validateMethodRequest(MethodConfig methodConfig) {
if (methodConfig.getSpreadsheetId() == null || methodConfig.getSpreadsheetId().isBlank()) {
@ -41,4 +101,24 @@ public class InfoMethod implements Method {
.body(BodyInserters.empty());
}
@Override
public JsonNode transformResponse(JsonNode response, MethodConfig methodConfig) {
if (response == null) {
throw new AppsmithPluginException(
AppsmithPluginError.PLUGIN_ERROR,
"Missing a valid response object.");
}
Map<String, Object> responseObj = new HashMap<>();
if (methodConfig.getBody() instanceof List) {
responseObj.put("sheets", methodConfig.getBody());
}
Iterator<String> fieldNames = response.fieldNames();
while(fieldNames.hasNext()) {
String fieldName = fieldNames.next();
responseObj.put(fieldName, response.get(fieldName));
}
return this.objectMapper.valueToTree(responseObj);
}
}

8
app/server/appsmith-plugins/googleSheetsPlugin/src/main/java/com/external/plugins/GoogleSheetsPlugin.java vendored
View File

@ -183,5 +183,13 @@ public class GoogleSheetsPlugin extends BasePlugin {
// This plugin would not have the option to test
return Mono.just(new DatasourceTestResult());
}
@Override
public Mono<ActionExecutionResult> getDatasourceMetadata(List<Property> pluginSpecifiedTemplates,
DatasourceConfiguration datasourceConfiguration) {
ActionConfiguration actionConfiguration = new ActionConfiguration();
actionConfiguration.setPluginSpecifiedTemplates(pluginSpecifiedTemplates);
return execute(null, datasourceConfiguration, actionConfiguration);
}
}
}

86
app/server/appsmith-plugins/googleSheetsPlugin/src/test/java/com/external/config/InfoMethodTest.java vendored Normal file
View File

@ -0,0 +1,86 @@
package com.external.config;
import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginException;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.junit.Assert;
import org.junit.Test;
import java.util.ArrayList;
import java.util.List;
public class InfoMethodTest {
@Test(expected = AppsmithPluginException.class)
public void testTransformResponse_missingJSON_throwsException() {
ObjectMapper objectMapper = new ObjectMapper();
InfoMethod infoMethod = new InfoMethod(objectMapper);
infoMethod.transformResponse(null, null);
}
@Test
public void testTransformResponse_missingSheets_returnsEmpty() throws JsonProcessingException {
ObjectMapper objectMapper = new ObjectMapper();
final String jsonString = "{\"key1\":\"value1\",\"key2\":\"value2\"}";
JsonNode jsonNode = objectMapper.readTree(jsonString);
Assert.assertNotNull(jsonNode);
MethodConfig methodConfig = new MethodConfig(new ArrayList<>());
JsonNode sheetNode = objectMapper.readTree("");
methodConfig.setBody(sheetNode);
InfoMethod infoMethod = new InfoMethod(objectMapper);
JsonNode result = infoMethod.transformResponse(jsonNode, methodConfig);
Assert.assertNotNull(result);
Assert.assertTrue(result.isObject());
Assert.assertEquals(null, result.get("sheets"));
}
@Test
public void testTransformResponse_emptySheets_returnsEmpty() throws JsonProcessingException {
ObjectMapper objectMapper = new ObjectMapper();
final String jsonString = "{\"key1\":\"value1\",\"key2\":\"value2\"}";
JsonNode jsonNode = objectMapper.readTree(jsonString);
Assert.assertNotNull(jsonNode);
MethodConfig methodConfig = new MethodConfig(new ArrayList<>());
methodConfig.setBody(new ArrayList<>());
InfoMethod infoMethod = new InfoMethod(objectMapper);
JsonNode result = infoMethod.transformResponse(jsonNode, methodConfig);
Assert.assertNotNull(result);
Assert.assertTrue(result.isObject());
Assert.assertEquals(0, result.get("sheets").size());
}
@Test
public void testTransformResponse_validSheets_toListOfSheets() throws JsonProcessingException {
ObjectMapper objectMapper = new ObjectMapper();
final String jsonString = "{\"key1\":\"value1\",\"key2\":\"value2\"}";
final String sheetMetadataString = "{\"sheetId\":\"1\", \"title\":\"test\", \"sheetType\":\"GRID\", \"index\":0}";
JsonNode jsonNode = objectMapper.readTree(jsonString);
Assert.assertNotNull(jsonNode);
MethodConfig methodConfig = new MethodConfig(new ArrayList<>());
JsonNode sheetNode = objectMapper.readTree(sheetMetadataString);
methodConfig.setBody(List.of(sheetNode));
InfoMethod infoMethod = new InfoMethod(objectMapper);
JsonNode result = infoMethod.transformResponse(jsonNode, methodConfig);
Assert.assertNotNull(result);
Assert.assertTrue(result.isObject());
Assert.assertEquals(1, result.get("sheets").size());
Assert.assertTrue("test".equalsIgnoreCase(result.get("sheets").get(0).get("title").asText()));
}
}

12
app/server/appsmith-server/src/main/java/com/appsmith/server/controllers/DatasourceController.java
View File

@ -1,7 +1,9 @@
package com.appsmith.server.controllers;
import com.appsmith.external.models.ActionExecutionResult;
import com.appsmith.external.models.DatasourceStructure;
import com.appsmith.external.models.DatasourceTestResult;
import com.appsmith.external.models.Property;
import com.appsmith.server.constants.Url;
import com.appsmith.server.domains.Datasource;
import com.appsmith.server.dtos.AuthorizationCodeCallbackDTO;
@ -20,6 +22,7 @@ import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.PutMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
@ -27,6 +30,7 @@ import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.server.ServerWebExchange;
import reactor.core.publisher.Mono;
import javax.validation.Valid;
import java.net.URI;
import java.util.List;
@ -103,4 +107,12 @@ public class DatasourceController extends BaseController<DatasourceService, Data
.map(datasource -> new ResponseDTO<>(HttpStatus.OK.value(), datasource, null));
}
@PutMapping("/datasource-query/{datasourceId}")
public Mono<ResponseDTO<ActionExecutionResult>> runQueryOnDatasource(@PathVariable String datasourceId,
@Valid @RequestBody List<Property> pluginSpecifiedTemplates) {
log.debug("Getting datasource metadata");
return datasourceStructureSolution.getDatasourceMetadata(datasourceId, pluginSpecifiedTemplates)
.map(metadata -> new ResponseDTO<>(HttpStatus.OK.value(), metadata, null));
}
}

50
app/server/appsmith-server/src/main/java/com/appsmith/server/solutions/DatasourceStructureSolution.java
View File

@ -3,9 +3,13 @@ package com.appsmith.server.solutions;
import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginError;
import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginException;
import com.appsmith.external.exceptions.pluginExceptions.StaleConnectionException;
import com.appsmith.external.models.ActionExecutionResult;
import com.appsmith.external.models.AuthenticationDTO;
import com.appsmith.external.models.DatasourceStructure;
import com.appsmith.external.models.Property;
import com.appsmith.external.plugins.PluginExecutor;
import com.appsmith.external.services.EncryptionService;
import com.appsmith.server.acl.AclPermission;
import com.appsmith.server.constants.FieldName;
import com.appsmith.server.domains.Datasource;
import com.appsmith.server.exceptions.AppsmithError;
@ -22,8 +26,11 @@ import org.springframework.util.CollectionUtils;
import reactor.core.publisher.Mono;
import java.time.Duration;
import java.util.List;
import java.util.concurrent.TimeoutException;
import static com.appsmith.external.models.AuthenticationDTO.AuthenticationStatus.SUCCESS;
@Component
@RequiredArgsConstructor
@Slf4j
@ -125,4 +132,47 @@ public class DatasourceStructureSolution {
: datasourceRepository.saveStructure(datasource.getId(), structure).thenReturn(structure)
);
}
/**
* This function will be used to execute queries on datasource without creating the new action
* e.g. get all spreadsheets from google drive, fetch 1st row from the table etc
* @param datasourceId
* @param pluginSpecifiedTemplates
* @return
*/
public Mono<ActionExecutionResult> getDatasourceMetadata(String datasourceId, List<Property> pluginSpecifiedTemplates) {
/*
1. Check if the datasource is present
2. Check plugin is present
3. Execute DB query from the information provided present in pluginSpecifiedTemplates
*/
Mono<Datasource> datasourceMono = datasourceService.findById(datasourceId, AclPermission.MANAGE_DATASOURCES)
.switchIfEmpty(Mono.error(new AppsmithException(
AppsmithError.ACL_NO_RESOURCE_FOUND, FieldName.DATASOURCE, datasourceId
)));
return datasourceMono.flatMap(datasource -> {
AuthenticationDTO auth = datasource.getDatasourceConfiguration() == null
|| datasource.getDatasourceConfiguration().getAuthentication() == null ?
null : datasource.getDatasourceConfiguration().getAuthentication();
if (auth == null || !SUCCESS.equals(auth.getAuthenticationStatus())) {
// Don't attempt to run query for invalid datasources.
return Mono.error(new AppsmithException(
AppsmithError.INVALID_DATASOURCE,
datasource.getName(),
"Authentication failure for datasource, please re-authenticate and try again!"
));
}
// check if the plugin is present and call method from plugin executor
return pluginExecutorHelper
.getPluginExecutor(pluginService.findById(datasource.getPluginId()))
.switchIfEmpty(Mono.error(new AppsmithException(AppsmithError.NO_RESOURCE_FOUND, FieldName.PLUGIN, datasource.getPluginId())))
.flatMap(pluginExecutor ->
pluginExecutor.getDatasourceMetadata(pluginSpecifiedTemplates, datasource.getDatasourceConfiguration())
)
.timeout(Duration.ofSeconds(GET_STRUCTURE_TIMEOUT_SECONDS));
});
}
}