diff --git a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/plugins/PluginExecutor.java b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/plugins/PluginExecutor.java index ea5024c003..f17d0ced2b 100644 --- a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/plugins/PluginExecutor.java +++ b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/plugins/PluginExecutor.java @@ -129,7 +129,6 @@ public interface PluginExecutor extends ExtensionPoint { variableSubstitution(actionConfiguration, datasourceConfiguration, executeActionDTO); - return; } /** @@ -140,7 +139,7 @@ public interface PluginExecutor extends ExtensionPoint { ExecuteActionDTO executeActionDTO) { //Do variable substitution //Do this only if params have been provided in the execute command - if (executeActionDTO.getParams() != null && !executeActionDTO.getParams().isEmpty()) { + if (executeActionDTO != null && !CollectionUtils.isEmpty(executeActionDTO.getParams())) { Map replaceParamsMap = executeActionDTO .getParams() .stream() diff --git a/app/server/appsmith-plugins/firestorePlugin/src/main/java/com/external/plugins/FirestorePlugin.java b/app/server/appsmith-plugins/firestorePlugin/src/main/java/com/external/plugins/FirestorePlugin.java index a7b68272d4..7419a92882 100644 --- a/app/server/appsmith-plugins/firestorePlugin/src/main/java/com/external/plugins/FirestorePlugin.java +++ b/app/server/appsmith-plugins/firestorePlugin/src/main/java/com/external/plugins/FirestorePlugin.java @@ -1,5 +1,6 @@ package com.external.plugins; +import com.appsmith.external.dtos.ExecuteActionDTO; import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginError; import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginException; import com.appsmith.external.models.ActionConfiguration; @@ -8,6 +9,7 @@ import com.appsmith.external.models.DBAuth; import com.appsmith.external.models.DatasourceConfiguration; import com.appsmith.external.models.DatasourceStructure; import com.appsmith.external.models.DatasourceTestResult; +import com.appsmith.external.models.PaginationField; import com.appsmith.external.models.Property; import com.appsmith.external.plugins.BasePlugin; import com.appsmith.external.plugins.PluginExecutor; @@ -59,6 +61,14 @@ import java.util.stream.StreamSupport; */ public class FirestorePlugin extends BasePlugin { + private static final int ORDER_PROPERTY_INDEX = 1; + private static final int LIMIT_PROPERTY_INDEX = 2; + private static final int QUERY_PROPERTY_INDEX = 3; + private static final int OPERATOR_PROPERTY_INDEX = 4; + private static final int QUERY_VALUE_PROPERTY_INDEX = 5; + private static final int START_AFTER_PROPERTY_INDEX = 6; + private static final int END_BEFORE_PROPERTY_INDEX = 7; + public FirestorePlugin(PluginWrapper wrapper) { super(wrapper); } @@ -70,9 +80,22 @@ public class FirestorePlugin extends BasePlugin { private final Scheduler scheduler = Schedulers.elastic(); @Override + @Deprecated public Mono execute(Firestore connection, DatasourceConfiguration datasourceConfiguration, ActionConfiguration actionConfiguration) { + return Mono.error(new AppsmithPluginException(AppsmithPluginError.PLUGIN_ERROR, "Unsupported Operation")); + } + + @Override + public Mono executeParameterized( + Firestore connection, + ExecuteActionDTO executeActionDTO, + DatasourceConfiguration datasourceConfiguration, + ActionConfiguration actionConfiguration) { + + // Do the template substitutions. + prepareConfigurationsForExecution(executeActionDTO, actionConfiguration, datasourceConfiguration); final String path = actionConfiguration.getPath(); @@ -102,6 +125,8 @@ public class FirestorePlugin extends BasePlugin { )); } + final PaginationField paginationField = executeActionDTO == null ? null : executeActionDTO.getPaginationField(); + return Mono .justOrEmpty(actionConfiguration.getBody()) .defaultIfEmpty("") @@ -132,7 +157,7 @@ public class FirestorePlugin extends BasePlugin { if (method.isDocumentLevel()) { return handleDocumentLevelMethod(connection, path, method, mapBody); } else { - return handleCollectionLevelMethod(connection, path, method, properties, mapBody); + return handleCollectionLevelMethod(connection, path, method, properties, mapBody, paginationField); } }) .subscribeOn(scheduler); @@ -220,14 +245,15 @@ public class FirestorePlugin extends BasePlugin { public Mono handleCollectionLevelMethod( Firestore connection, String path, - com.external.plugins.Method method, + Method method, List properties, - Map mapBody - ) { + Map mapBody, + PaginationField paginationField) { + final CollectionReference collection = connection.collection(path); if (method == Method.GET_COLLECTION) { - return methodGetCollection(collection, properties); + return methodGetCollection(collection, properties, paginationField); } else if (method == Method.ADD_TO_COLLECTION) { return methodAddToCollection(collection, mapBody); @@ -240,16 +266,92 @@ public class FirestorePlugin extends BasePlugin { )); } - private Mono methodGetCollection(CollectionReference query, List properties) { - final String orderBy = properties.size() > 1 && properties.get(1) != null ? properties.get(1).getValue() : null; - final int limit = properties.size() > 2 && properties.get(2) != null ? Integer.parseInt(properties.get(2).getValue()) : 10; - final String queryFieldPath = properties.size() > 3 && properties.get(3) != null ? properties.get(3).getValue() : null; - final Op operator = properties.size() > 4 && properties.get(4) != null ? Op.valueOf(properties.get(4).getValue()) : null; - final String queryValue = properties.size() > 5 && properties.get(5) != null ? properties.get(5).getValue() : null; + private String getPropertyAt(List properties, int index, String defaultValue) { + if (properties.size() <= index) { + return defaultValue; + } + + final Property property = properties.get(index); + if (property == null) { + return defaultValue; + } + + final String value = property.getValue(); + return value != null ? value : defaultValue; + } + + private Mono methodGetCollection(CollectionReference query, List properties, PaginationField paginationField) { + final String limitString = getPropertyAt(properties, LIMIT_PROPERTY_INDEX, "10"); + final int limit = StringUtils.isEmpty(limitString) ? 10 : Integer.parseInt(limitString); + + final String queryFieldPath = getPropertyAt(properties, QUERY_PROPERTY_INDEX, null); + + final String operatorString = getPropertyAt(properties, OPERATOR_PROPERTY_INDEX, null); + final Op operator = StringUtils.isEmpty(operatorString) ? null : Op.valueOf(operatorString); + + final String queryValue = getPropertyAt(properties, QUERY_VALUE_PROPERTY_INDEX, null); + + final String orderByString = getPropertyAt(properties, ORDER_PROPERTY_INDEX, ""); + final List orderings; + try { + orderings = StringUtils.isEmpty(orderByString) ? Collections.emptyList() : objectMapper.readValue(orderByString, List.class); + } catch (IOException e) { + // TODO: Investigate how many actions are using this today on prod. + return Mono.error(new AppsmithPluginException(AppsmithPluginError.PLUGIN_JSON_PARSE_ERROR, orderByString, e)); + } + + Map startAfterTemp = null; + if (PaginationField.NEXT.equals(paginationField)) { + final String startAfterJson = getPropertyAt(properties, START_AFTER_PROPERTY_INDEX, "{}"); + try { + startAfterTemp = StringUtils.isEmpty(startAfterJson) ? Collections.emptyMap() : objectMapper.readValue(startAfterJson, Map.class); + } catch (IOException e) { + return Mono.error(new AppsmithPluginException(AppsmithPluginError.PLUGIN_JSON_PARSE_ERROR, startAfterJson, e)); + } + } + + Map endBeforeTemp = null; + if (PaginationField.PREV.equals(paginationField)) { + final String endBeforeJson = getPropertyAt(properties, END_BEFORE_PROPERTY_INDEX, "{}"); + try { + endBeforeTemp = StringUtils.isEmpty(endBeforeJson) ? Collections.emptyMap() : objectMapper.readValue(endBeforeJson, Map.class); + } catch (IOException e) { + return Mono.error(new AppsmithPluginException(AppsmithPluginError.PLUGIN_JSON_PARSE_ERROR, endBeforeJson, e)); + } + } + + final Map startAfter = startAfterTemp; + final Map endBefore = endBeforeTemp; + + if (paginationField != null && CollectionUtils.isEmpty(orderings)) { + return Mono.error(new AppsmithPluginException( + AppsmithPluginError.PLUGIN_EXECUTE_ARGUMENT_ERROR, + "Cannot do pagination without specifying an ordering." + )); + } return Mono.just(query) // Apply ordering, if provided. - .map(query1 -> StringUtils.isEmpty(orderBy) ? query1 : query1.orderBy(orderBy)) + .map(query1 -> { + Query q = query1; + final List startAfterValues = new ArrayList<>(); + final List endBeforeValues = new ArrayList<>(); + for (final String field : orderings) { + q = q.orderBy(field); + if (startAfter != null) { + startAfterValues.add(startAfter.get(field)); + } + if (endBefore != null) { + endBeforeValues.add(endBefore.get(field)); + } + } + if (PaginationField.NEXT.equals(paginationField) && !CollectionUtils.isEmpty(startAfter)) { + q = q.startAfter(startAfterValues.toArray()); + } else if (PaginationField.PREV.equals(paginationField) && !CollectionUtils.isEmpty(endBefore)) { + q = q.endBefore(endBeforeValues.toArray()); + } + return q; + }) // Apply where condition, if provided. .flatMap(query1 -> { if (StringUtils.isEmpty(queryFieldPath) || operator == null || queryValue == null) { diff --git a/app/server/appsmith-plugins/firestorePlugin/src/main/resources/editor.json b/app/server/appsmith-plugins/firestorePlugin/src/main/resources/editor.json index e592ec6a3a..787d155b59 100644 --- a/app/server/appsmith-plugins/firestorePlugin/src/main/resources/editor.json +++ b/app/server/appsmith-plugins/firestorePlugin/src/main/resources/editor.json @@ -63,7 +63,7 @@ "initialValue": "orderBy" }, { - "label": "Order By", + "label": "Order By (JSON array of field names to order by)", "configProperty": "actionConfiguration.pluginSpecifiedTemplates[1].value", "controlType": "INPUT_TEXT", "hidden": { @@ -73,6 +73,42 @@ }, "initialValue": "" }, + { + "label": "Start After Key", + "configProperty": "actionConfiguration.pluginSpecifiedTemplates[6].key", + "controlType": "INPUT_TEXT", + "hidden": true, + "initialValue": "limit" + }, + { + "label": "Start After", + "configProperty": "actionConfiguration.pluginSpecifiedTemplates[6].value", + "controlType": "INPUT_TEXT", + "hidden": { + "path": "actionConfiguration.pluginSpecifiedTemplates[0].value", + "comparison": "NOT_EQUALS", + "value": "GET_COLLECTION" + }, + "initialValue": "" + }, + { + "label": "End Before Key", + "configProperty": "actionConfiguration.pluginSpecifiedTemplates[7].key", + "controlType": "INPUT_TEXT", + "hidden": true, + "initialValue": "limit" + }, + { + "label": "End Before", + "configProperty": "actionConfiguration.pluginSpecifiedTemplates[7].value", + "controlType": "INPUT_TEXT", + "hidden": { + "path": "actionConfiguration.pluginSpecifiedTemplates[0].value", + "comparison": "NOT_EQUALS", + "value": "GET_COLLECTION" + }, + "initialValue": "" + }, { "label": "Limit Documents Key", "configProperty": "actionConfiguration.pluginSpecifiedTemplates[2].key", diff --git a/app/server/appsmith-plugins/firestorePlugin/src/test/java/com/external/plugins/FirestorePluginTest.java b/app/server/appsmith-plugins/firestorePlugin/src/test/java/com/external/plugins/FirestorePluginTest.java index ef72ac41d6..6dceb9a2f8 100644 --- a/app/server/appsmith-plugins/firestorePlugin/src/test/java/com/external/plugins/FirestorePluginTest.java +++ b/app/server/appsmith-plugins/firestorePlugin/src/test/java/com/external/plugins/FirestorePluginTest.java @@ -1,13 +1,18 @@ package com.external.plugins; +import com.appsmith.external.dtos.ExecuteActionDTO; import com.appsmith.external.models.ActionConfiguration; import com.appsmith.external.models.ActionExecutionResult; import com.appsmith.external.models.DBAuth; import com.appsmith.external.models.DatasourceConfiguration; +import com.appsmith.external.models.PaginationField; import com.appsmith.external.models.Property; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.cloud.NoCredentials; import com.google.cloud.ServiceOptions; import com.google.cloud.firestore.Blob; +import com.google.cloud.firestore.CollectionReference; import com.google.cloud.firestore.DocumentSnapshot; import com.google.cloud.firestore.FieldValue; import com.google.cloud.firestore.Firestore; @@ -21,6 +26,7 @@ import org.testcontainers.containers.FirestoreEmulatorContainer; import org.testcontainers.utility.DockerImageName; import reactor.core.publisher.Mono; import reactor.test.StepVerifier; +import reactor.util.function.Tuple3; import java.nio.charset.StandardCharsets; import java.util.Collections; @@ -28,6 +34,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -86,9 +93,27 @@ public class FirestorePluginTest { firestoreConnection.document("initial/two") ) )).get(); + firestoreConnection.document("changing/to-update").set(Map.of("value", 1)).get(); firestoreConnection.document("changing/to-delete").set(Map.of("value", 1)).get(); + final CollectionReference paginationCol = firestoreConnection.collection("pagination"); + paginationCol.add(Map.of("n", 1, "name", "Michele Cole", "firm", "Appsmith")).get(); + paginationCol.add(Map.of("n", 2, "name", "Meghan Steele", "firm", "Google")).get(); + paginationCol.add(Map.of("n", 3, "name", "Della Moore", "firm", "Facebook")).get(); + paginationCol.add(Map.of("n", 4, "name", "Eunice Hines", "firm", "Microsoft")).get(); + paginationCol.add(Map.of("n", 5, "name", "Harriet Myers", "firm", "Netflix")).get(); + paginationCol.add(Map.of("n", 6, "name", "Lowell Reese", "firm", "Apple")).get(); + paginationCol.add(Map.of("n", 7, "name", "Gerard Neal", "firm", "Oracle")).get(); + paginationCol.add(Map.of("n", 8, "name", "Allen Arnold", "firm", "IBM")).get(); + paginationCol.add(Map.of("n", 9, "name", "Josefina Perkins", "firm", "Google")).get(); + paginationCol.add(Map.of("n", 10, "name", "Alvin Zimmerman", "firm", "Facebook")).get(); + paginationCol.add(Map.of("n", 11, "name", "Israel Broc", "firm", "Microsoft")).get(); + paginationCol.add(Map.of("n", 12, "name", "Larry Frazie", "firm", "Netflix")).get(); + paginationCol.add(Map.of("n", 13, "name", "Rufus Green", "firm", "Apple")).get(); + paginationCol.add(Map.of("n", 14, "name", "Marco Murra", "firm", "Oracle")).get(); + paginationCol.add(Map.of("n", 15, "name", "Jeremy Mille", "firm", "IBM")).get(); + dsConfig.setUrl(emulator.getEmulatorEndpoint()); DBAuth auth = new DBAuth(); auth.setUsername("test-project"); @@ -103,7 +128,7 @@ public class FirestorePluginTest { actionConfiguration.setPluginSpecifiedTemplates(List.of(new Property("method", "GET_DOCUMENT"))); Mono resultMono = pluginExecutor - .execute(firestoreConnection, dsConfig, actionConfiguration); + .executeParameterized(firestoreConnection, null, dsConfig, actionConfiguration); StepVerifier.create(resultMono) .assertNext(result -> { @@ -125,7 +150,7 @@ public class FirestorePluginTest { actionConfiguration.setPluginSpecifiedTemplates(List.of(new Property("method", "GET_DOCUMENT"))); Mono resultMono = pluginExecutor - .execute(firestoreConnection, dsConfig, actionConfiguration); + .executeParameterized(firestoreConnection, null, dsConfig, actionConfiguration); StepVerifier.create(resultMono) .assertNext(result -> { @@ -152,7 +177,7 @@ public class FirestorePluginTest { actionConfiguration.setPluginSpecifiedTemplates(List.of(new Property("method", "GET_DOCUMENT"))); Mono resultMono = pluginExecutor - .execute(firestoreConnection, dsConfig, actionConfiguration); + .executeParameterized(firestoreConnection, null, dsConfig, actionConfiguration); StepVerifier.create(resultMono) .assertNext(result -> { @@ -181,7 +206,7 @@ public class FirestorePluginTest { actionConfiguration.setPluginSpecifiedTemplates(List.of(new Property("method", "GET_COLLECTION"))); Mono resultMono = pluginExecutor - .execute(firestoreConnection, dsConfig, actionConfiguration); + .executeParameterized(firestoreConnection, null, dsConfig, actionConfiguration); StepVerifier.create(resultMono) .assertNext(result -> { @@ -241,7 +266,7 @@ public class FirestorePluginTest { actionConfiguration.setPluginSpecifiedTemplates(List.of(new Property("method", "SET_DOCUMENT"))); Mono resultMono = pluginExecutor - .execute(firestoreConnection, dsConfig, actionConfiguration); + .executeParameterized(firestoreConnection, null, dsConfig, actionConfiguration); StepVerifier.create(resultMono) .assertNext(result -> { @@ -262,7 +287,7 @@ public class FirestorePluginTest { actionConfiguration.setPluginSpecifiedTemplates(List.of(new Property("method", "CREATE_DOCUMENT"))); Mono resultMono = pluginExecutor - .execute(firestoreConnection, dsConfig, actionConfiguration); + .executeParameterized(firestoreConnection, null, dsConfig, actionConfiguration); StepVerifier.create(resultMono) .assertNext(result -> { @@ -282,7 +307,7 @@ public class FirestorePluginTest { actionConfiguration.setPluginSpecifiedTemplates(List.of(new Property("method", "UPDATE_DOCUMENT"))); Mono resultMono = pluginExecutor - .execute(firestoreConnection, dsConfig, actionConfiguration); + .executeParameterized(firestoreConnection, null, dsConfig, actionConfiguration); StepVerifier.create(resultMono) .assertNext(result -> { @@ -306,7 +331,7 @@ public class FirestorePluginTest { actionConfiguration.setPluginSpecifiedTemplates(List.of(new Property("method", "DELETE_DOCUMENT"))); Mono resultMono = pluginExecutor - .execute(firestoreConnection, dsConfig, actionConfiguration); + .executeParameterized(firestoreConnection, null, dsConfig, actionConfiguration); StepVerifier.create(resultMono) .assertNext(result -> { @@ -334,7 +359,7 @@ public class FirestorePluginTest { "}"); Mono resultMono = pluginExecutor - .execute(firestoreConnection, dsConfig, actionConfiguration); + .executeParameterized(firestoreConnection, null, dsConfig, actionConfiguration); StepVerifier.create(resultMono) .assertNext(result -> { @@ -344,4 +369,118 @@ public class FirestorePluginTest { .verifyComplete(); } + @Test + public void testPagination() { + final ActionConfiguration actionConfiguration = new ActionConfiguration(); + actionConfiguration.setPath("pagination"); + actionConfiguration.setPluginSpecifiedTemplates(List.of( + new Property("method", "GET_COLLECTION"), + new Property("order", "[\"n\"]"), + new Property("limit", "5") + )); + + final ObjectMapper objectMapper = new ObjectMapper(); + + Mono> pagesMono = pluginExecutor + .executeParameterized(firestoreConnection, null, dsConfig, actionConfiguration) + .flatMap(result -> { + List> results = (List) result.getBody(); + final Map first = results.get(0); + final Map last = results.get(results.size() - 1); + + final ExecuteActionDTO execDetails = new ExecuteActionDTO(); + execDetails.setPaginationField(PaginationField.NEXT); + + final ActionConfiguration actionConfiguration1 = new ActionConfiguration(); + actionConfiguration1.setPath(actionConfiguration.getPath()); + try { + actionConfiguration1.setPluginSpecifiedTemplates(List.of( + new Property("method", "GET_COLLECTION"), + new Property("order", "[\"n\"]"), + new Property("limit", "5"), + new Property(), + new Property(), + new Property(), + new Property("startAfter", objectMapper.writeValueAsString(last)), + new Property("endBefore", objectMapper.writeValueAsString(first)) + )); + } catch (JsonProcessingException e) { + e.printStackTrace(); + return Mono.error(e); + } + + return Mono.zip( + Mono.just(result), + pluginExecutor.executeParameterized(firestoreConnection, execDetails, dsConfig, actionConfiguration1) + ); + }) + .flatMap(twoPagesMono -> { + final ActionExecutionResult page1 = twoPagesMono.getT1(); + final ActionExecutionResult page2 = twoPagesMono.getT2(); + + List> results = (List) page2.getBody(); + final Map first = results.get(0); + final Map last = results.get(results.size() - 1); + + final ExecuteActionDTO execDetails = new ExecuteActionDTO(); + execDetails.setPaginationField(PaginationField.PREV); + + final ActionConfiguration actionConfiguration1 = new ActionConfiguration(); + actionConfiguration1.setPath(actionConfiguration.getPath()); + try { + actionConfiguration1.setPluginSpecifiedTemplates(List.of( + new Property("method", "GET_COLLECTION"), + new Property("order", "[\"n\"]"), + new Property("limit", "5"), + new Property(), + new Property(), + new Property(), + new Property("startAfter", objectMapper.writeValueAsString(last)), + new Property("endBefore", objectMapper.writeValueAsString(first)) + )); + } catch (JsonProcessingException e) { + e.printStackTrace(); + return Mono.error(e); + } + + return Mono.zip( + Mono.just(page1), + Mono.just(page2), + pluginExecutor.executeParameterized(firestoreConnection, execDetails, dsConfig, actionConfiguration1) + ); + }); + + StepVerifier.create(pagesMono) + .assertNext(resultPages -> { + final ActionExecutionResult firstPageResult = resultPages.getT1(); + final ActionExecutionResult secondPageResult = resultPages.getT2(); + final ActionExecutionResult firstPageResultAgain = resultPages.getT3(); + + assertTrue(firstPageResult.getIsExecutionSuccess()); + + List> firstResults = (List) firstPageResult.getBody(); + assertEquals(5, firstResults.size()); + assertEquals( + "[1, 2, 3, 4, 5]", + firstResults.stream().map(m -> m.get("n").toString()).collect(Collectors.toList()).toString() + ); + + List> secondResults = (List) secondPageResult.getBody(); + assertEquals(5, secondResults.size()); + assertEquals( + "[6, 7, 8, 9, 10]", + secondResults.stream().map(m -> m.get("n").toString()).collect(Collectors.toList()).toString() + ); + + List> firstResultsAgain = (List) firstPageResultAgain.getBody(); + assertEquals(5, firstResultsAgain.size()); + assertEquals( + "[1, 2, 3, 4, 5]", + firstResultsAgain.stream().map(m -> m.get("n").toString()).collect(Collectors.toList()).toString() + ); + + }) + .verifyComplete(); + } + }