Protei/PromucFlow_constructor
2
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Support server-side pagination for Firestore (#3128)

Browse Source 
* POC for Firestore server-side pagination

* Load pagination information values from action configuration

* Get execution information for Firestore plugin

* Working implementation of pagination for Firestore

* Add tests for next and previou page navigations

* Require ordering to be set when paginating

* Remove commented code

* Don't report error on bad configuration

* Error out on usupported operation

Co-authored-by: Trisha Anand <trisha@appsmith.com>

* Move constant indices to constant fields

* Use executeParameterized instead of execute

Co-authored-by: Trisha Anand <trisha@appsmith.com>
This commit is contained in:
Shrikant Sharat Kandula 2021-02-24 10:20:08 +05:30 committed by GitHub
parent 6c80f23201
commit 68bbc4fb28
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 300 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
app/server/appsmith-interfaces/src/main/java/com/appsmith/external/plugins/PluginExecutor.java vendored
View File

@ -129,7 +129,6 @@ public interface PluginExecutor<C> extends ExtensionPoint {
variableSubstitution(actionConfiguration, datasourceConfiguration, executeActionDTO);
return;
}
/**
@ -140,7 +139,7 @@ public interface PluginExecutor<C> extends ExtensionPoint {
ExecuteActionDTO executeActionDTO) {
//Do variable substitution
//Do this only if params have been provided in the execute command
if (executeActionDTO.getParams() != null && !executeActionDTO.getParams().isEmpty()) {
if (executeActionDTO != null && !CollectionUtils.isEmpty(executeActionDTO.getParams())) {
Map<String, String> replaceParamsMap = executeActionDTO
.getParams()
.stream()

126
app/server/appsmith-plugins/firestorePlugin/src/main/java/com/external/plugins/FirestorePlugin.java vendored
View File

@ -1,5 +1,6 @@
package com.external.plugins;
import com.appsmith.external.dtos.ExecuteActionDTO;
import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginError;
import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginException;
import com.appsmith.external.models.ActionConfiguration;
@ -8,6 +9,7 @@ import com.appsmith.external.models.DBAuth;
import com.appsmith.external.models.DatasourceConfiguration;
import com.appsmith.external.models.DatasourceStructure;
import com.appsmith.external.models.DatasourceTestResult;
import com.appsmith.external.models.PaginationField;
import com.appsmith.external.models.Property;
import com.appsmith.external.plugins.BasePlugin;
import com.appsmith.external.plugins.PluginExecutor;
@ -59,6 +61,14 @@ import java.util.stream.StreamSupport;
*/
public class FirestorePlugin extends BasePlugin {
private static final int ORDER_PROPERTY_INDEX = 1;
private static final int LIMIT_PROPERTY_INDEX = 2;
private static final int QUERY_PROPERTY_INDEX = 3;
private static final int OPERATOR_PROPERTY_INDEX = 4;
private static final int QUERY_VALUE_PROPERTY_INDEX = 5;
private static final int START_AFTER_PROPERTY_INDEX = 6;
private static final int END_BEFORE_PROPERTY_INDEX = 7;
public FirestorePlugin(PluginWrapper wrapper) {
super(wrapper);
}
@ -70,9 +80,22 @@ public class FirestorePlugin extends BasePlugin {
private final Scheduler scheduler = Schedulers.elastic();
@Override
@Deprecated
public Mono<ActionExecutionResult> execute(Firestore connection,
DatasourceConfiguration datasourceConfiguration,
ActionConfiguration actionConfiguration) {
return Mono.error(new AppsmithPluginException(AppsmithPluginError.PLUGIN_ERROR, "Unsupported Operation"));
}
@Override
public Mono<ActionExecutionResult> executeParameterized(
Firestore connection,
ExecuteActionDTO executeActionDTO,
DatasourceConfiguration datasourceConfiguration,
ActionConfiguration actionConfiguration) {
// Do the template substitutions.
prepareConfigurationsForExecution(executeActionDTO, actionConfiguration, datasourceConfiguration);
final String path = actionConfiguration.getPath();
@ -102,6 +125,8 @@ public class FirestorePlugin extends BasePlugin {
));
}
final PaginationField paginationField = executeActionDTO == null ? null : executeActionDTO.getPaginationField();
return Mono
.justOrEmpty(actionConfiguration.getBody())
.defaultIfEmpty("")
@ -132,7 +157,7 @@ public class FirestorePlugin extends BasePlugin {
if (method.isDocumentLevel()) {
return handleDocumentLevelMethod(connection, path, method, mapBody);
} else {
return handleCollectionLevelMethod(connection, path, method, properties, mapBody);
return handleCollectionLevelMethod(connection, path, method, properties, mapBody, paginationField);
}
})
.subscribeOn(scheduler);
@ -220,14 +245,15 @@ public class FirestorePlugin extends BasePlugin {
public Mono<ActionExecutionResult> handleCollectionLevelMethod(
Firestore connection,
String path,
com.external.plugins.Method method,
Method method,
List<Property> properties,
Map<String, Object> mapBody
) {
Map<String, Object> mapBody,
PaginationField paginationField) {
final CollectionReference collection = connection.collection(path);
if (method == Method.GET_COLLECTION) {
return methodGetCollection(collection, properties);
return methodGetCollection(collection, properties, paginationField);
} else if (method == Method.ADD_TO_COLLECTION) {
return methodAddToCollection(collection, mapBody);
@ -240,16 +266,92 @@ public class FirestorePlugin extends BasePlugin {
));
}
private Mono<ActionExecutionResult> methodGetCollection(CollectionReference query, List<Property> properties) {
final String orderBy = properties.size() > 1 && properties.get(1) != null ? properties.get(1).getValue() : null;
final int limit = properties.size() > 2 && properties.get(2) != null ? Integer.parseInt(properties.get(2).getValue()) : 10;
final String queryFieldPath = properties.size() > 3 && properties.get(3) != null ? properties.get(3).getValue() : null;
final Op operator = properties.size() > 4 && properties.get(4) != null ? Op.valueOf(properties.get(4).getValue()) : null;
final String queryValue = properties.size() > 5 && properties.get(5) != null ? properties.get(5).getValue() : null;
private String getPropertyAt(List<Property> properties, int index, String defaultValue) {
if (properties.size() <= index) {
return defaultValue;
}
final Property property = properties.get(index);
if (property == null) {
return defaultValue;
}
final String value = property.getValue();
return value != null ? value : defaultValue;
}
private Mono<ActionExecutionResult> methodGetCollection(CollectionReference query, List<Property> properties, PaginationField paginationField) {
final String limitString = getPropertyAt(properties, LIMIT_PROPERTY_INDEX, "10");
final int limit = StringUtils.isEmpty(limitString) ? 10 : Integer.parseInt(limitString);
final String queryFieldPath = getPropertyAt(properties, QUERY_PROPERTY_INDEX, null);
final String operatorString = getPropertyAt(properties, OPERATOR_PROPERTY_INDEX, null);
final Op operator = StringUtils.isEmpty(operatorString) ? null : Op.valueOf(operatorString);
final String queryValue = getPropertyAt(properties, QUERY_VALUE_PROPERTY_INDEX, null);
final String orderByString = getPropertyAt(properties, ORDER_PROPERTY_INDEX, "");
final List<String> orderings;
try {
orderings = StringUtils.isEmpty(orderByString) ? Collections.emptyList() : objectMapper.readValue(orderByString, List.class);
} catch (IOException e) {
// TODO: Investigate how many actions are using this today on prod.
return Mono.error(new AppsmithPluginException(AppsmithPluginError.PLUGIN_JSON_PARSE_ERROR, orderByString, e));
}
Map<String, Object> startAfterTemp = null;
if (PaginationField.NEXT.equals(paginationField)) {
final String startAfterJson = getPropertyAt(properties, START_AFTER_PROPERTY_INDEX, "{}");
try {
startAfterTemp = StringUtils.isEmpty(startAfterJson) ? Collections.emptyMap() : objectMapper.readValue(startAfterJson, Map.class);
} catch (IOException e) {
return Mono.error(new AppsmithPluginException(AppsmithPluginError.PLUGIN_JSON_PARSE_ERROR, startAfterJson, e));
}
}
Map<String, Object> endBeforeTemp = null;
if (PaginationField.PREV.equals(paginationField)) {
final String endBeforeJson = getPropertyAt(properties, END_BEFORE_PROPERTY_INDEX, "{}");
try {
endBeforeTemp = StringUtils.isEmpty(endBeforeJson) ? Collections.emptyMap() : objectMapper.readValue(endBeforeJson, Map.class);
} catch (IOException e) {
return Mono.error(new AppsmithPluginException(AppsmithPluginError.PLUGIN_JSON_PARSE_ERROR, endBeforeJson, e));
}
}
final Map<String, Object> startAfter = startAfterTemp;
final Map<String, Object> endBefore = endBeforeTemp;
if (paginationField != null && CollectionUtils.isEmpty(orderings)) {
return Mono.error(new AppsmithPluginException(
AppsmithPluginError.PLUGIN_EXECUTE_ARGUMENT_ERROR,
"Cannot do pagination without specifying an ordering."
));
}
return Mono.just(query)
// Apply ordering, if provided.
.map(query1 -> StringUtils.isEmpty(orderBy) ? query1 : query1.orderBy(orderBy))
.map(query1 -> {
Query q = query1;
final List<Object> startAfterValues = new ArrayList<>();
final List<Object> endBeforeValues = new ArrayList<>();
for (final String field : orderings) {
q = q.orderBy(field);
if (startAfter != null) {
startAfterValues.add(startAfter.get(field));
}
if (endBefore != null) {
endBeforeValues.add(endBefore.get(field));
}
}
if (PaginationField.NEXT.equals(paginationField) && !CollectionUtils.isEmpty(startAfter)) {
q = q.startAfter(startAfterValues.toArray());
} else if (PaginationField.PREV.equals(paginationField) && !CollectionUtils.isEmpty(endBefore)) {
q = q.endBefore(endBeforeValues.toArray());
}
return q;
})
// Apply where condition, if provided.
.flatMap(query1 -> {
if (StringUtils.isEmpty(queryFieldPath) || operator == null || queryValue == null) {

38
app/server/appsmith-plugins/firestorePlugin/src/main/resources/editor.json
View File

@ -63,7 +63,7 @@
"initialValue": "orderBy"
},
{
"label": "Order By",
"label": "Order By (JSON array of field names to order by)",
"configProperty": "actionConfiguration.pluginSpecifiedTemplates[1].value",
"controlType": "INPUT_TEXT",
"hidden": {
@ -73,6 +73,42 @@
},
"initialValue": ""
},
{
"label": "Start After Key",
"configProperty": "actionConfiguration.pluginSpecifiedTemplates[6].key",
"controlType": "INPUT_TEXT",
"hidden": true,
"initialValue": "limit"
},
{
"label": "Start After",
"configProperty": "actionConfiguration.pluginSpecifiedTemplates[6].value",
"controlType": "INPUT_TEXT",
"hidden": {
"path": "actionConfiguration.pluginSpecifiedTemplates[0].value",
"comparison": "NOT_EQUALS",
"value": "GET_COLLECTION"
},
"initialValue": ""
},
{
"label": "End Before Key",
"configProperty": "actionConfiguration.pluginSpecifiedTemplates[7].key",
"controlType": "INPUT_TEXT",
"hidden": true,
"initialValue": "limit"
},
{
"label": "End Before",
"configProperty": "actionConfiguration.pluginSpecifiedTemplates[7].value",
"controlType": "INPUT_TEXT",
"hidden": {
"path": "actionConfiguration.pluginSpecifiedTemplates[0].value",
"comparison": "NOT_EQUALS",
"value": "GET_COLLECTION"
},
"initialValue": ""
},
{
"label": "Limit Documents Key",
"configProperty": "actionConfiguration.pluginSpecifiedTemplates[2].key",

157
app/server/appsmith-plugins/firestorePlugin/src/test/java/com/external/plugins/FirestorePluginTest.java vendored
View File

@ -1,13 +1,18 @@
package com.external.plugins;
import com.appsmith.external.dtos.ExecuteActionDTO;
import com.appsmith.external.models.ActionConfiguration;
import com.appsmith.external.models.ActionExecutionResult;
import com.appsmith.external.models.DBAuth;
import com.appsmith.external.models.DatasourceConfiguration;
import com.appsmith.external.models.PaginationField;
import com.appsmith.external.models.Property;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.cloud.NoCredentials;
import com.google.cloud.ServiceOptions;
import com.google.cloud.firestore.Blob;
import com.google.cloud.firestore.CollectionReference;
import com.google.cloud.firestore.DocumentSnapshot;
import com.google.cloud.firestore.FieldValue;
import com.google.cloud.firestore.Firestore;
@ -21,6 +26,7 @@ import org.testcontainers.containers.FirestoreEmulatorContainer;
import org.testcontainers.utility.DockerImageName;
import reactor.core.publisher.Mono;
import reactor.test.StepVerifier;
import reactor.util.function.Tuple3;
import java.nio.charset.StandardCharsets;
import java.util.Collections;
@ -28,6 +34,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import java.util.stream.Collectors;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@ -86,9 +93,27 @@ public class FirestorePluginTest {
firestoreConnection.document("initial/two")
)
)).get();
firestoreConnection.document("changing/to-update").set(Map.of("value", 1)).get();
firestoreConnection.document("changing/to-delete").set(Map.of("value", 1)).get();
final CollectionReference paginationCol = firestoreConnection.collection("pagination");
paginationCol.add(Map.of("n", 1, "name", "Michele Cole", "firm", "Appsmith")).get();
paginationCol.add(Map.of("n", 2, "name", "Meghan Steele", "firm", "Google")).get();
paginationCol.add(Map.of("n", 3, "name", "Della Moore", "firm", "Facebook")).get();
paginationCol.add(Map.of("n", 4, "name", "Eunice Hines", "firm", "Microsoft")).get();
paginationCol.add(Map.of("n", 5, "name", "Harriet Myers", "firm", "Netflix")).get();
paginationCol.add(Map.of("n", 6, "name", "Lowell Reese", "firm", "Apple")).get();
paginationCol.add(Map.of("n", 7, "name", "Gerard Neal", "firm", "Oracle")).get();
paginationCol.add(Map.of("n", 8, "name", "Allen Arnold", "firm", "IBM")).get();
paginationCol.add(Map.of("n", 9, "name", "Josefina Perkins", "firm", "Google")).get();
paginationCol.add(Map.of("n", 10, "name", "Alvin Zimmerman", "firm", "Facebook")).get();
paginationCol.add(Map.of("n", 11, "name", "Israel Broc", "firm", "Microsoft")).get();
paginationCol.add(Map.of("n", 12, "name", "Larry Frazie", "firm", "Netflix")).get();
paginationCol.add(Map.of("n", 13, "name", "Rufus Green", "firm", "Apple")).get();
paginationCol.add(Map.of("n", 14, "name", "Marco Murra", "firm", "Oracle")).get();
paginationCol.add(Map.of("n", 15, "name", "Jeremy Mille", "firm", "IBM")).get();
dsConfig.setUrl(emulator.getEmulatorEndpoint());
DBAuth auth = new DBAuth();
auth.setUsername("test-project");
@ -103,7 +128,7 @@ public class FirestorePluginTest {
actionConfiguration.setPluginSpecifiedTemplates(List.of(new Property("method", "GET_DOCUMENT")));
Mono<ActionExecutionResult> resultMono = pluginExecutor
.execute(firestoreConnection, dsConfig, actionConfiguration);
.executeParameterized(firestoreConnection, null, dsConfig, actionConfiguration);
StepVerifier.create(resultMono)
.assertNext(result -> {
@ -125,7 +150,7 @@ public class FirestorePluginTest {
actionConfiguration.setPluginSpecifiedTemplates(List.of(new Property("method", "GET_DOCUMENT")));
Mono<ActionExecutionResult> resultMono = pluginExecutor
.execute(firestoreConnection, dsConfig, actionConfiguration);
.executeParameterized(firestoreConnection, null, dsConfig, actionConfiguration);
StepVerifier.create(resultMono)
.assertNext(result -> {
@ -152,7 +177,7 @@ public class FirestorePluginTest {
actionConfiguration.setPluginSpecifiedTemplates(List.of(new Property("method", "GET_DOCUMENT")));
Mono<ActionExecutionResult> resultMono = pluginExecutor
.execute(firestoreConnection, dsConfig, actionConfiguration);
.executeParameterized(firestoreConnection, null, dsConfig, actionConfiguration);
StepVerifier.create(resultMono)
.assertNext(result -> {
@ -181,7 +206,7 @@ public class FirestorePluginTest {
actionConfiguration.setPluginSpecifiedTemplates(List.of(new Property("method", "GET_COLLECTION")));
Mono<ActionExecutionResult> resultMono = pluginExecutor
.execute(firestoreConnection, dsConfig, actionConfiguration);
.executeParameterized(firestoreConnection, null, dsConfig, actionConfiguration);
StepVerifier.create(resultMono)
.assertNext(result -> {
@ -241,7 +266,7 @@ public class FirestorePluginTest {
actionConfiguration.setPluginSpecifiedTemplates(List.of(new Property("method", "SET_DOCUMENT")));
Mono<ActionExecutionResult> resultMono = pluginExecutor
.execute(firestoreConnection, dsConfig, actionConfiguration);
.executeParameterized(firestoreConnection, null, dsConfig, actionConfiguration);
StepVerifier.create(resultMono)
.assertNext(result -> {
@ -262,7 +287,7 @@ public class FirestorePluginTest {
actionConfiguration.setPluginSpecifiedTemplates(List.of(new Property("method", "CREATE_DOCUMENT")));
Mono<ActionExecutionResult> resultMono = pluginExecutor
.execute(firestoreConnection, dsConfig, actionConfiguration);
.executeParameterized(firestoreConnection, null, dsConfig, actionConfiguration);
StepVerifier.create(resultMono)
.assertNext(result -> {
@ -282,7 +307,7 @@ public class FirestorePluginTest {
actionConfiguration.setPluginSpecifiedTemplates(List.of(new Property("method", "UPDATE_DOCUMENT")));
Mono<ActionExecutionResult> resultMono = pluginExecutor
.execute(firestoreConnection, dsConfig, actionConfiguration);
.executeParameterized(firestoreConnection, null, dsConfig, actionConfiguration);
StepVerifier.create(resultMono)
.assertNext(result -> {
@ -306,7 +331,7 @@ public class FirestorePluginTest {
actionConfiguration.setPluginSpecifiedTemplates(List.of(new Property("method", "DELETE_DOCUMENT")));
Mono<ActionExecutionResult> resultMono = pluginExecutor
.execute(firestoreConnection, dsConfig, actionConfiguration);
.executeParameterized(firestoreConnection, null, dsConfig, actionConfiguration);
StepVerifier.create(resultMono)
.assertNext(result -> {
@ -334,7 +359,7 @@ public class FirestorePluginTest {
"}");
Mono<ActionExecutionResult> resultMono = pluginExecutor
.execute(firestoreConnection, dsConfig, actionConfiguration);
.executeParameterized(firestoreConnection, null, dsConfig, actionConfiguration);
StepVerifier.create(resultMono)
.assertNext(result -> {
@ -344,4 +369,118 @@ public class FirestorePluginTest {
.verifyComplete();
}
@Test
public void testPagination() {
final ActionConfiguration actionConfiguration = new ActionConfiguration();
actionConfiguration.setPath("pagination");
actionConfiguration.setPluginSpecifiedTemplates(List.of(
new Property("method", "GET_COLLECTION"),
new Property("order", "[\"n\"]"),
new Property("limit", "5")
));
final ObjectMapper objectMapper = new ObjectMapper();
Mono<Tuple3<ActionExecutionResult, ActionExecutionResult, ActionExecutionResult>> pagesMono = pluginExecutor
.executeParameterized(firestoreConnection, null, dsConfig, actionConfiguration)
.flatMap(result -> {
List<Map<String, Object>> results = (List) result.getBody();
final Map<String, Object> first = results.get(0);
final Map<String, Object> last = results.get(results.size() - 1);
final ExecuteActionDTO execDetails = new ExecuteActionDTO();
execDetails.setPaginationField(PaginationField.NEXT);
final ActionConfiguration actionConfiguration1 = new ActionConfiguration();
actionConfiguration1.setPath(actionConfiguration.getPath());
try {
actionConfiguration1.setPluginSpecifiedTemplates(List.of(
new Property("method", "GET_COLLECTION"),
new Property("order", "[\"n\"]"),
new Property("limit", "5"),
new Property(),
new Property(),
new Property(),
new Property("startAfter", objectMapper.writeValueAsString(last)),
new Property("endBefore", objectMapper.writeValueAsString(first))
));
} catch (JsonProcessingException e) {
e.printStackTrace();
return Mono.error(e);
}
return Mono.zip(
Mono.just(result),
pluginExecutor.executeParameterized(firestoreConnection, execDetails, dsConfig, actionConfiguration1)
);
})
.flatMap(twoPagesMono -> {
final ActionExecutionResult page1 = twoPagesMono.getT1();
final ActionExecutionResult page2 = twoPagesMono.getT2();
List<Map<String, Object>> results = (List) page2.getBody();
final Map<String, Object> first = results.get(0);
final Map<String, Object> last = results.get(results.size() - 1);
final ExecuteActionDTO execDetails = new ExecuteActionDTO();
execDetails.setPaginationField(PaginationField.PREV);
final ActionConfiguration actionConfiguration1 = new ActionConfiguration();
actionConfiguration1.setPath(actionConfiguration.getPath());
try {
actionConfiguration1.setPluginSpecifiedTemplates(List.of(
new Property("method", "GET_COLLECTION"),
new Property("order", "[\"n\"]"),
new Property("limit", "5"),
new Property(),
new Property(),
new Property(),
new Property("startAfter", objectMapper.writeValueAsString(last)),
new Property("endBefore", objectMapper.writeValueAsString(first))
));
} catch (JsonProcessingException e) {
e.printStackTrace();
return Mono.error(e);
}
return Mono.zip(
Mono.just(page1),
Mono.just(page2),
pluginExecutor.executeParameterized(firestoreConnection, execDetails, dsConfig, actionConfiguration1)
);
});
StepVerifier.create(pagesMono)
.assertNext(resultPages -> {
final ActionExecutionResult firstPageResult = resultPages.getT1();
final ActionExecutionResult secondPageResult = resultPages.getT2();
final ActionExecutionResult firstPageResultAgain = resultPages.getT3();
assertTrue(firstPageResult.getIsExecutionSuccess());
List<Map<String, Object>> firstResults = (List) firstPageResult.getBody();
assertEquals(5, firstResults.size());
assertEquals(
"[1, 2, 3, 4, 5]",
firstResults.stream().map(m -> m.get("n").toString()).collect(Collectors.toList()).toString()
);
List<Map<String, Object>> secondResults = (List) secondPageResult.getBody();
assertEquals(5, secondResults.size());
assertEquals(
"[6, 7, 8, 9, 10]",
secondResults.stream().map(m -> m.get("n").toString()).collect(Collectors.toList()).toString()
);
List<Map<String, Object>> firstResultsAgain = (List) firstPageResultAgain.getBody();
assertEquals(5, firstResultsAgain.size());
assertEquals(
"[1, 2, 3, 4, 5]",
firstResultsAgain.stream().map(m -> m.get("n").toString()).collect(Collectors.toList()).toString()
);
})
.verifyComplete();
}
}