From 74cd3620573ee8c389f964f520a2522b5cb19432 Mon Sep 17 00:00:00 2001 From: Shrikant Sharat Kandula Date: Mon, 23 Nov 2020 18:42:33 +0530 Subject: [PATCH] Firestore Integration (#1799) * Adding the skeleton for Firestore integration * Adding the datasource & query editor forms Also adding the database changelog for the firestore plugin Commenting out the firestore.close() connection because that causes issues with multiple Firestore tenants running in the same JVM. * Adding the code for fetching the structure of collections from Firestore * Use single document path field for Firestore * Fix potential NPE when datasource destroy timeouts * Work in progress on collection level ops for Firestore * Get documents in a collection now works * Add collection level querying support * Mild refactoring * Fix NPE when some fields are missing * Hide clientJSON as a password field for Firestore * Make collection level querying reactive * Make reactive * Validate before connecting * Add tests for all supported methods in Firestore * Fix forms for Firestore with hidden fields * Hide limit and order by fields when not needed * Restore log entry deleted by mistake * Use S3 URL for Firestore/Firebase logo * Add comments detailing why some code is commented * Make parsing JSON reactive and fix subscribe calls * Fix reactive scheduler Co-authored-by: Arpit Mohan --- .../external/plugins/PluginExecutor.java | 36 +- .../firestorePlugin/plugin.properties | 5 + .../appsmith-plugins/firestorePlugin/pom.xml | 152 ++++++ .../com/external/plugins/FirestorePlugin.java | 475 ++++++++++++++++++ .../java/com/external/plugins/Method.java | 27 + .../main/java/com/external/plugins/Op.java | 14 + .../src/main/resources/editor.json | 192 +++++++ .../src/main/resources/form.json | 32 ++ .../external/plugins/FirestorePluginTest.java | 206 ++++++++ app/server/appsmith-plugins/pom.xml | 1 + .../server/migrations/DatabaseChangelog.java | 20 + .../DatasourceContextServiceImpl.java | 6 +- 12 files changed, 1164 insertions(+), 2 deletions(-) create mode 100644 app/server/appsmith-plugins/firestorePlugin/plugin.properties create mode 100644 app/server/appsmith-plugins/firestorePlugin/pom.xml create mode 100644 app/server/appsmith-plugins/firestorePlugin/src/main/java/com/external/plugins/FirestorePlugin.java create mode 100644 app/server/appsmith-plugins/firestorePlugin/src/main/java/com/external/plugins/Method.java create mode 100644 app/server/appsmith-plugins/firestorePlugin/src/main/java/com/external/plugins/Op.java create mode 100644 app/server/appsmith-plugins/firestorePlugin/src/main/resources/editor.json create mode 100644 app/server/appsmith-plugins/firestorePlugin/src/main/resources/form.json create mode 100644 app/server/appsmith-plugins/firestorePlugin/src/test/java/com/external/plugins/FirestorePluginTest.java diff --git a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/plugins/PluginExecutor.java b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/plugins/PluginExecutor.java index 3ab030101d..3eb48ace37 100644 --- a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/plugins/PluginExecutor.java +++ b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/plugins/PluginExecutor.java @@ -20,7 +20,7 @@ public interface PluginExecutor extends ExtensionPoint { * to the parameters in Datasource Configuration * @param datasourceConfiguration : These are the configurations which have been used to create a Datasource from a Plugin * @param actionConfiguration : These are the configurations which have been used to create an Action from a Datasource. - * @return ActionExecutionResult : This object is returned to the user which contains the result values from the execution. + * @return ActionExecutionResult : This object is returned to the user which contains the result values from the execution. */ Mono execute(C connection, DatasourceConfiguration datasourceConfiguration, ActionConfiguration actionConfiguration); @@ -40,14 +40,48 @@ public interface PluginExecutor extends ExtensionPoint { */ void datasourceDestroy(C connection); + /** + * This function tells the platform if datasource is valid by checking the set of invalid strings. + * If empty, the datasource is valid. This set of invalid strings is populated by + * {@link #validateDatasource(DatasourceConfiguration)} + * + * @param datasourceConfiguration + * @return boolean + */ default boolean isDatasourceValid(DatasourceConfiguration datasourceConfiguration) { return CollectionUtils.isEmpty(validateDatasource(datasourceConfiguration)); } + /** + * This function checks if the datasource is valid. It should only check if all the mandatory fields are filled and + * if the values are of the right format. It does NOT check the validity of those fields. + * Please use {@link #testDatasource(DatasourceConfiguration)} to establish the correctness of those fields. + * + * If the datasource configuration is valid, it should return an empty set of invalid strings. + * If not, it should return the list of invalid messages as a set. + * + * @param datasourceConfiguration : The datasource to be validated + * @return Set : The set of invalid strings informing the user of all the invalid fields + */ Set validateDatasource(DatasourceConfiguration datasourceConfiguration); + /** + * This function tests the datasource by executing a test query or hitting the endpoint to check the correctness + * of the values provided in the datasource configuration + * + * @param datasourceConfiguration + * @return + */ Mono testDatasource(DatasourceConfiguration datasourceConfiguration); + /** + * This function fetches the structure of the tables/collections in the datasource. It's used to make query creation + * easier for the user. + * + * @param connection + * @param datasourceConfiguration + * @return + */ default Mono getStructure(C connection, DatasourceConfiguration datasourceConfiguration) { return Mono.empty(); } diff --git a/app/server/appsmith-plugins/firestorePlugin/plugin.properties b/app/server/appsmith-plugins/firestorePlugin/plugin.properties new file mode 100644 index 0000000000..0ee54de337 --- /dev/null +++ b/app/server/appsmith-plugins/firestorePlugin/plugin.properties @@ -0,0 +1,5 @@ +plugin.id=firestore-plugin +plugin.class=com.external.plugins.FirestorePlugin +plugin.version=1.0-SNAPSHOT +plugin.provider=tech@appsmith.com +plugin.dependencies= diff --git a/app/server/appsmith-plugins/firestorePlugin/pom.xml b/app/server/appsmith-plugins/firestorePlugin/pom.xml new file mode 100644 index 0000000000..44304a65a8 --- /dev/null +++ b/app/server/appsmith-plugins/firestorePlugin/pom.xml @@ -0,0 +1,152 @@ + + + + 4.0.0 + + com.external.plugins + firestorePlugin + 1.0-SNAPSHOT + + firestorePlugin + + + UTF-8 + 11 + ${java.version} + ${java.version} + firestore-plugin + com.external.plugins.FirestorePlugin + 1.0-SNAPSHOT + tech@appsmith.com + + + + + + + org.pf4j + pf4j-spring + 0.6.0 + provided + + + + com.appsmith + interfaces + 1.0-SNAPSHOT + provided + + + + org.projectlombok + lombok + 1.18.8 + provided + + + com.google.firebase + firebase-admin + 7.0.1 + + + org.slf4j + slf4j-api + + + + + + + + junit + junit + 4.13.1 + test + + + + org.testcontainers + testcontainers + 1.15.0-rc2 + test + + + org.testcontainers + gcloud + 1.15.0 + test + + + + io.projectreactor + reactor-test + 3.2.11.RELEASE + test + + + org.mockito + mockito-core + 3.1.0 + test + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.2.4 + + false + + + + ${plugin.id} + ${plugin.class} + ${plugin.version} + ${plugin.provider} + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + package + + shade + + + + + + maven-dependency-plugin + + + copy-dependencies + package + + copy-dependencies + + + runtime + ${project.build.directory}/lib + + + + + + + + diff --git a/app/server/appsmith-plugins/firestorePlugin/src/main/java/com/external/plugins/FirestorePlugin.java b/app/server/appsmith-plugins/firestorePlugin/src/main/java/com/external/plugins/FirestorePlugin.java new file mode 100644 index 0000000000..c40524364d --- /dev/null +++ b/app/server/appsmith-plugins/firestorePlugin/src/main/java/com/external/plugins/FirestorePlugin.java @@ -0,0 +1,475 @@ +package com.external.plugins; + +import com.appsmith.external.models.ActionConfiguration; +import com.appsmith.external.models.ActionExecutionResult; +import com.appsmith.external.models.AuthenticationDTO; +import com.appsmith.external.models.DatasourceConfiguration; +import com.appsmith.external.models.DatasourceStructure; +import com.appsmith.external.models.DatasourceTestResult; +import com.appsmith.external.models.Property; +import com.appsmith.external.pluginExceptions.AppsmithPluginError; +import com.appsmith.external.pluginExceptions.AppsmithPluginException; +import com.appsmith.external.plugins.BasePlugin; +import com.appsmith.external.plugins.PluginExecutor; +import com.google.api.core.ApiFuture; +import com.google.auth.oauth2.GoogleCredentials; +import com.google.cloud.firestore.CollectionReference; +import com.google.cloud.firestore.DocumentReference; +import com.google.cloud.firestore.DocumentSnapshot; +import com.google.cloud.firestore.Firestore; +import com.google.cloud.firestore.QueryDocumentSnapshot; +import com.google.cloud.firestore.QuerySnapshot; +import com.google.cloud.firestore.WriteResult; +import com.google.firebase.FirebaseApp; +import com.google.firebase.FirebaseOptions; +import com.google.firebase.cloud.FirestoreClient; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.pf4j.Extension; +import org.pf4j.PluginWrapper; +import org.springframework.util.CollectionUtils; +import reactor.core.Exceptions; +import reactor.core.publisher.Mono; +import reactor.core.scheduler.Scheduler; +import reactor.core.scheduler.Schedulers; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.lang.reflect.InvocationTargetException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; + +/** + * Datasource properties: + * 1. Client JSON + *

+ * Query Action properties: + * 1. method: Dropdown + * 2. collection: String + * 3. documentKey: String + */ +public class FirestorePlugin extends BasePlugin { + + public FirestorePlugin(PluginWrapper wrapper) { + super(wrapper); + } + + @Slf4j + @Extension + public static class FirestorePluginExecutor implements PluginExecutor { + + private final Scheduler scheduler = Schedulers.elastic(); + + @Override + public Mono execute(Firestore connection, + DatasourceConfiguration datasourceConfiguration, + ActionConfiguration actionConfiguration) { + + final String path = actionConfiguration.getPath(); + + if (StringUtils.isBlank(path)) { + return Mono.error(new AppsmithPluginException( + AppsmithPluginError.PLUGIN_ERROR, + "Document/Collection path cannot be empty" + )); + } + + if (path.startsWith("/") || path.endsWith("/")) { + return Mono.error(new AppsmithPluginException( + AppsmithPluginError.PLUGIN_ERROR, + "Firestore paths should not begin or end with `/` character." + )); + } + + final List properties = actionConfiguration.getPluginSpecifiedTemplates(); + final com.external.plugins.Method method = CollectionUtils.isEmpty(properties) + ? null + : com.external.plugins.Method.valueOf(properties.get(0).getValue()); + + if (method == null) { + return Mono.error(new AppsmithPluginException( + AppsmithPluginError.PLUGIN_ERROR, + "Missing Firestore method." + )); + } + + return Mono + .justOrEmpty(actionConfiguration.getBody()) + .defaultIfEmpty("") + .flatMap(strBody -> { + if (StringUtils.isBlank(strBody)) { + return Mono.just(Collections.emptyMap()); + } + + try { + return Mono.just(objectMapper.readValue(strBody, HashMap.class)); + } catch (IOException e) { + return Mono.error(new AppsmithPluginException( + AppsmithPluginError.PLUGIN_ERROR, + e.getMessage() + )); + } + }) + .flatMap(mapBody -> { + if (mapBody.isEmpty() && method.isBodyNeeded()) { + return Mono.error(new AppsmithPluginException( + AppsmithPluginError.PLUGIN_ERROR, + "The method " + method.toString() + " needs a non-empty body to work." + )); + } + return Mono.just((Map) mapBody); + }) + .flatMap(mapBody -> { + if (method.isDocumentLevel()) { + return handleDocumentLevelMethod(connection, path, method, mapBody); + } else { + return handleCollectionLevelMethod(connection, path, method, properties); + } + }) + .subscribeOn(scheduler); + } + + public Mono handleDocumentLevelMethod( + Firestore connection, + String path, + com.external.plugins.Method method, + Map mapBody + ) { + return Mono.just(method) + // Get the actual Java method to be called. + .flatMap(method1 -> { + final String methodName = method1.toString().split("_")[0].toLowerCase(); + + try { + switch (method1) { + case GET_DOCUMENT: + case DELETE_DOCUMENT: + return Mono.justOrEmpty(DocumentReference.class.getMethod(methodName)); + case SET_DOCUMENT: + case CREATE_DOCUMENT: + case UPDATE_DOCUMENT: + return Mono.justOrEmpty(DocumentReference.class.getMethod(methodName, Map.class)); + default: + return Mono.error(new AppsmithPluginException( + AppsmithPluginError.PLUGIN_ERROR, + "Invalid document-level method " + method1.toString() + )); + } + + } catch (NoSuchMethodException e) { + return Mono.error(new AppsmithPluginException( + AppsmithPluginError.PLUGIN_ERROR, + "Error getting actual method for operation " + method1.toString() + )); + + } + }) + // Call that method and get a Future of the result. + .flatMap(operationMethod -> { + DocumentReference document = connection.document(path); + Object objFuture; + + try { + if (operationMethod.getParameterCount() == 1) { + objFuture = operationMethod.invoke(document, mapBody); + } else { + objFuture = operationMethod.invoke(document); + } + + } catch (IllegalAccessException | InvocationTargetException e) { + return Mono.error(new AppsmithPluginException(AppsmithPluginError.PLUGIN_ERROR, e.getMessage())); + + } + + return Mono.just((ApiFuture) objFuture); + }) + // Consume the Future to get the actual result object. + .flatMap(resultFuture -> { + try { + + return Mono.just(resultFuture.get()); + } catch (InterruptedException | ExecutionException e) { + return Mono.error(new AppsmithPluginException(AppsmithPluginError.PLUGIN_ERROR, e.getMessage())); + } + }) + // Build a response object with the result. + .flatMap(objResult1 -> { + ActionExecutionResult result = new ActionExecutionResult(); + try { + result.setBody(resultToMap(objResult1)); + } catch (AppsmithPluginException e) { + return Mono.error(e); + } + result.setIsExecutionSuccess(true); + System.out.println( + Thread.currentThread().getName() + + ": In the Firestore Plugin, got action execution result: " + + result.toString() + ); + return Mono.just(result); + }); + } + + public Mono handleCollectionLevelMethod( + Firestore connection, + String path, + com.external.plugins.Method method, + List properties + ) { + final String orderBy = properties.size() > 1 && properties.get(1) != null ? properties.get(1).getValue() : null; + final int limit = properties.size() > 2 && properties.get(2) != null ? Integer.parseInt(properties.get(2).getValue()) : 10; + final String queryFieldPath = properties.size() > 3 && properties.get(3) != null ? properties.get(3).getValue() : null; + final Op operator = properties.size() > 4 && properties.get(4) != null ? Op.valueOf(properties.get(4).getValue()) : null; + final String queryValue = properties.size() > 5 && properties.get(5) != null ? properties.get(5).getValue() : null; + + return Mono.just(connection.collection(path)) + // Apply ordering, if provided. + .map(query1 -> StringUtils.isEmpty(orderBy) ? query1 : query1.orderBy(orderBy)) + // Apply where condition, if provided. + .flatMap(query1 -> { + if (StringUtils.isEmpty(queryFieldPath) || operator == null || queryValue == null) { + return Mono.just(query1); + } + + switch (operator) { + case LT: + return Mono.just(query1.whereLessThan(queryFieldPath, queryValue)); + case LTE: + return Mono.just(query1.whereLessThanOrEqualTo(queryFieldPath, queryValue)); + case EQ: + return Mono.just(query1.whereEqualTo(queryFieldPath, queryValue)); + // TODO: NOT_EQ operator support is awaited in the next version of Firestore driver. + // case NOT_EQ: + // return Mono.just(query1.whereNotEqualTo(queryFieldPath, queryValue)); + case GT: + return Mono.just(query1.whereGreaterThan(queryFieldPath, queryValue)); + case GTE: + return Mono.just(query1.whereGreaterThanOrEqualTo(queryFieldPath, queryValue)); + case ARRAY_CONTAINS: + return Mono.just(query1.whereArrayContains(queryFieldPath, queryValue)); + case ARRAY_CONTAINS_ANY: + try { + return Mono.just(query1.whereArrayContainsAny(queryFieldPath, parseList(queryValue))); + } catch (IOException e) { + return Mono.error(new AppsmithPluginException( + AppsmithPluginError.PLUGIN_ERROR, + "Unable to parse condition value as a JSON list." + )); + } + case IN: + try { + return Mono.just(query1.whereIn(queryFieldPath, parseList(queryValue))); + } catch (IOException e) { + return Mono.error(new AppsmithPluginException( + AppsmithPluginError.PLUGIN_ERROR, + "Unable to parse condition value as a JSON list." + )); + } + // TODO: NOT_IN operator support is awaited in the next version of Firestore driver. + // case NOT_IN: + // return Mono.just(query1.whereNotIn(queryFieldPath, queryValue)); + default: + return Mono.error(new AppsmithPluginException( + AppsmithPluginError.PLUGIN_ERROR, + "Unsupported operator for `where` condition " + operator.toString() + "." + )); + } + }) + // Apply limit, always provided, since without it we can inadvertently end up processing too much data. + .map(query1 -> query1.limit(limit)) + // Run the Firestore query to get a Future of the results. + .flatMap(query1 -> { + switch (method) { + case GET_COLLECTION: + return Mono.just(query1.get()); + default: + return Mono.error(new AppsmithPluginException( + AppsmithPluginError.PLUGIN_ERROR, + "Unknown collection method: " + method.toString() + "." + )); + } + }) + // Consume the future to get the actual results. + .flatMap(resultFuture -> { + try { + return Mono.just(resultFuture.get()); + } catch (InterruptedException | ExecutionException e) { + return Mono.error(new AppsmithPluginException(AppsmithPluginError.PLUGIN_ERROR, e.getMessage())); + } + }) + // Build response object with the results from the Future. + .flatMap(objResult1 -> { + ActionExecutionResult result = new ActionExecutionResult(); + try { + result.setBody(resultToMap(objResult1)); + } catch (AppsmithPluginException e) { + return Mono.error(e); + } + result.setIsExecutionSuccess(true); + System.out.println( + Thread.currentThread().getName() + + ": In the Firestore Plugin, got action execution result: " + + result.toString() + ); + return Mono.just(result); + }); + } + + private Object resultToMap(Object objResult) throws AppsmithPluginException { + if (objResult instanceof WriteResult) { + WriteResult writeResult = (WriteResult) objResult; + Map resultMap = new HashMap<>(); + resultMap.put("lastUpdateTime", writeResult.getUpdateTime()); + return resultMap; + + } else if (objResult instanceof DocumentSnapshot) { + DocumentSnapshot documentSnapshot = (DocumentSnapshot) objResult; + Map resultMap = new HashMap<>(); + if (documentSnapshot.getData() != null) { + resultMap = documentSnapshot.getData(); + } + return resultMap; + + } else if (objResult instanceof QuerySnapshot) { + QuerySnapshot querySnapshot = (QuerySnapshot) objResult; + List> documents = new ArrayList<>(); + for (QueryDocumentSnapshot documentSnapshot : querySnapshot.getDocuments()) { + documents.add(documentSnapshot.getData()); + } + return documents; + + } else { + throw new AppsmithPluginException( + AppsmithPluginError.PLUGIN_ERROR, + "Unable to serialize object of type " + objResult.getClass().getName() + "." + ); + + } + } + + @Override + public Mono datasourceCreate(DatasourceConfiguration datasourceConfiguration) { + final AuthenticationDTO authentication = datasourceConfiguration.getAuthentication(); + + final Set errors = validateDatasource(datasourceConfiguration); + if (!CollectionUtils.isEmpty(errors)) { + return Mono.error(new AppsmithPluginException(AppsmithPluginError.PLUGIN_ERROR, errors.iterator().next())); + } + + final String projectId = authentication.getUsername(); + final String clientJson = authentication.getPassword(); + + InputStream serviceAccount = new ByteArrayInputStream(clientJson.getBytes()); + + return Mono + .fromSupplier(() -> { + GoogleCredentials credentials; + try { + credentials = GoogleCredentials.fromStream(serviceAccount); + } catch (IOException e) { + throw Exceptions.propagate(new AppsmithPluginException( + AppsmithPluginError.PLUGIN_ERROR, + e.getMessage() + )); + } + + return FirebaseOptions.builder() + .setDatabaseUrl(datasourceConfiguration.getUrl()) + .setProjectId(projectId) + .setCredentials(credentials) + .build(); + }) + .onErrorMap(Exceptions::unwrap) + .map(options -> { + try { + return FirebaseApp.getInstance(projectId); + } catch (IllegalStateException e) { + return FirebaseApp.initializeApp(options, projectId); + } + }) + .map(FirestoreClient::getFirestore) + .subscribeOn(scheduler); + } + + @Override + public void datasourceDestroy(Firestore connection) { + // This is empty because there's no concept of destroying a Firestore connection here. + // When the datasource is updated, the FirebaseApp instance will delete & re-create the app + } + + @Override + public Set validateDatasource(DatasourceConfiguration datasourceConfiguration) { + final AuthenticationDTO authentication = datasourceConfiguration.getAuthentication(); + + Set invalids = new HashSet<>(); + + if (authentication == null) { + invalids.add("Missing ProjectID and ClientJSON in datasource."); + + } else { + if (StringUtils.isEmpty(authentication.getUsername())) { + invalids.add("Missing ProjectID in datasource."); + } + + if (StringUtils.isEmpty(authentication.getPassword())) { + invalids.add("Missing ClientJSON in datasource."); + } + + } + + if (StringUtils.isBlank(datasourceConfiguration.getUrl())) { + invalids.add("Missing Firestore URL."); + } + + return invalids; + } + + private List parseList(String arrayJson) throws IOException { + return (List) objectMapper.readValue(arrayJson, ArrayList.class); + } + + @Override + public Mono testDatasource(DatasourceConfiguration datasourceConfiguration) { + return datasourceCreate(datasourceConfiguration) + .map(connection -> new DatasourceTestResult()) + .onErrorResume(error -> Mono.just(new DatasourceTestResult(error.getMessage()))); + } + + @Override + public Mono getStructure(Firestore connection, DatasourceConfiguration datasourceConfiguration) { + return Mono + .fromSupplier(() -> { + Iterable collectionReferences = connection.listCollections(); + + List tables = StreamSupport.stream(collectionReferences.spliterator(), false) + .map(collectionReference -> { + String id = collectionReference.getId(); + final ArrayList templates = new ArrayList<>(); + return new DatasourceStructure.Table( + DatasourceStructure.TableType.COLLECTION, + id, + new ArrayList<>(), + new ArrayList<>(), + templates + ); + }) + .collect(Collectors.toList()); + + DatasourceStructure structure = new DatasourceStructure(); + structure.setTables(tables); + + return structure; + }) + .subscribeOn(scheduler); + } + } +} diff --git a/app/server/appsmith-plugins/firestorePlugin/src/main/java/com/external/plugins/Method.java b/app/server/appsmith-plugins/firestorePlugin/src/main/java/com/external/plugins/Method.java new file mode 100644 index 0000000000..086bc8953f --- /dev/null +++ b/app/server/appsmith-plugins/firestorePlugin/src/main/java/com/external/plugins/Method.java @@ -0,0 +1,27 @@ +package com.external.plugins; + +public enum Method { + GET_DOCUMENT(true, false), + GET_COLLECTION(false, false), + SET_DOCUMENT(true, true), + CREATE_DOCUMENT(true, true), + UPDATE_DOCUMENT(true, true), + DELETE_DOCUMENT(true, false), + ; + + private final boolean isDocumentLevel; + private final boolean isBodyNeeded; + + Method(boolean isDocumentLevel, boolean isBodyNeeded) { + this.isDocumentLevel = isDocumentLevel; + this.isBodyNeeded = isBodyNeeded; + } + + public boolean isDocumentLevel() { + return isDocumentLevel; + } + + public boolean isBodyNeeded() { + return isBodyNeeded; + } +} diff --git a/app/server/appsmith-plugins/firestorePlugin/src/main/java/com/external/plugins/Op.java b/app/server/appsmith-plugins/firestorePlugin/src/main/java/com/external/plugins/Op.java new file mode 100644 index 0000000000..ac177c01d0 --- /dev/null +++ b/app/server/appsmith-plugins/firestorePlugin/src/main/java/com/external/plugins/Op.java @@ -0,0 +1,14 @@ +package com.external.plugins; + +public enum Op { + LT, + LTE, + EQ, + NOT_EQ, + GT, + GTE, + ARRAY_CONTAINS, + IN, + ARRAY_CONTAINS_ANY, + NOT_IN, +} diff --git a/app/server/appsmith-plugins/firestorePlugin/src/main/resources/editor.json b/app/server/appsmith-plugins/firestorePlugin/src/main/resources/editor.json new file mode 100644 index 0000000000..38e261a5b0 --- /dev/null +++ b/app/server/appsmith-plugins/firestorePlugin/src/main/resources/editor.json @@ -0,0 +1,192 @@ +{ + "editor": [ + { + "sectionName": "", + "id": 1, + "children": [ + { + "label": "Method Key", + "configProperty": "actionConfiguration.pluginSpecifiedTemplates[0].key", + "controlType": "INPUT_TEXT", + "hidden": true, + "initialValue": "method" + }, + { + "label": "Method", + "configProperty": "actionConfiguration.pluginSpecifiedTemplates[0].value", + "controlType": "DROP_DOWN", + "isRequired": true, + "initialValue": "GET_DOCUMENT", + "options": [ + { + "label": "Get Single Document", + "value": "GET_DOCUMENT" + }, + { + "label": "Get Documents in Collection", + "value": "GET_COLLECTION" + }, + { + "label": "Set Document", + "value": "SET_DOCUMENT" + }, + { + "label": "Create Document", + "value": "CREATE_DOCUMENT" + }, + { + "label": "Update Document", + "value": "UPDATE_DOCUMENT" + }, + { + "label": "Delete Document", + "value": "DELETE_DOCUMENT" + } + ] + }, + { + "label": "Document/Collection Path", + "configProperty": "actionConfiguration.path", + "controlType": "INPUT_TEXT", + "isRequired": true, + "initialValue": "" + }, + { + "label": "Order By Key", + "configProperty": "actionConfiguration.pluginSpecifiedTemplates[1].key", + "controlType": "INPUT_TEXT", + "hidden": true, + "initialValue": "orderBy" + }, + { + "label": "Order By", + "configProperty": "actionConfiguration.pluginSpecifiedTemplates[1].value", + "controlType": "INPUT_TEXT", + "hidden": { + "path": "actionConfiguration.pluginSpecifiedTemplates[0].value", + "comparison": "NOT_EQUALS", + "value": "GET_COLLECTION" + }, + "initialValue": "" + }, + { + "label": "Limit Documents Key", + "configProperty": "actionConfiguration.pluginSpecifiedTemplates[2].key", + "controlType": "INPUT_TEXT", + "hidden": true, + "initialValue": "limit" + }, + { + "label": "Limit Documents", + "configProperty": "actionConfiguration.pluginSpecifiedTemplates[2].value", + "controlType": "INPUT_TEXT", + "hidden": { + "path": "actionConfiguration.pluginSpecifiedTemplates[0].value", + "comparison": "NOT_EQUALS", + "value": "GET_COLLECTION" + }, + "initialValue": "10" + }, + { + "sectionName": "Query", + "id": 2, + "children": [ + { + "label": "Field Path Key", + "configProperty": "actionConfiguration.pluginSpecifiedTemplates[3].key", + "controlType": "INPUT_TEXT", + "hidden": true, + "initialValue": "fieldPath" + }, + { + "label": "Where Condition: Field Path (leave emtpy to not apply any conditions)", + "configProperty": "actionConfiguration.pluginSpecifiedTemplates[3].value", + "controlType": "INPUT_TEXT", + "hidden": { + "path": "actionConfiguration.pluginSpecifiedTemplates[0].value", + "comparison": "NOT_EQUALS", + "value": "GET_COLLECTION" + }, + "initialValue": "" + }, + { + "label": "Operator Key", + "configProperty": "actionConfiguration.pluginSpecifiedTemplates[4].key", + "controlType": "INPUT_TEXT", + "hidden": true, + "initialValue": "operator" + }, + { + "label": "Where Condition: Operator", + "configProperty": "actionConfiguration.pluginSpecifiedTemplates[4].value", + "controlType": "DROP_DOWN", + "hidden": { + "path": "actionConfiguration.pluginSpecifiedTemplates[0].value", + "comparison": "NOT_EQUALS", + "value": "GET_COLLECTION" + }, + "initialValue": "EQ", + "options": [ + { + "label": "<", + "value": "LT" + }, + { + "label": "<=", + "value": "LTE" + }, + { + "label": "==", + "value": "EQ" + }, + { + "label": ">=", + "value": "GTE" + }, + { + "label": ">", + "value": "GT" + }, + { + "label": "array-contains", + "value": "ARRAY_CONTAINS" + }, + { + "label": "in", + "value": "IN" + }, + { + "label": "array-contains-any", + "value": "ARRAY_CONTAINS_ANY" + } + ] + }, + { + "label": "Value Key", + "configProperty": "actionConfiguration.pluginSpecifiedTemplates[5].key", + "controlType": "INPUT_TEXT", + "hidden": true, + "initialValue": "fieldValue" + }, + { + "label": "Where Condition: Value", + "configProperty": "actionConfiguration.pluginSpecifiedTemplates[5].value", + "controlType": "INPUT_TEXT", + "hidden": { + "path": "actionConfiguration.pluginSpecifiedTemplates[0].value", + "comparison": "NOT_EQUALS", + "value": "GET_COLLECTION" + }, + "initialValue": "" + } + ] + }, + { + "label": "Body", + "configProperty": "actionConfiguration.body", + "controlType": "QUERY_DYNAMIC_TEXT" + } + ] + } + ] +} diff --git a/app/server/appsmith-plugins/firestorePlugin/src/main/resources/form.json b/app/server/appsmith-plugins/firestorePlugin/src/main/resources/form.json new file mode 100644 index 0000000000..cc55b206b7 --- /dev/null +++ b/app/server/appsmith-plugins/firestorePlugin/src/main/resources/form.json @@ -0,0 +1,32 @@ +{ + "form": [ + { + "sectionName": "Details", + "id": 1, + "children": [ + { + "label": "Database URL", + "configProperty": "datasourceConfiguration.url", + "controlType": "INPUT_TEXT", + "isRequired": true, + "initialValue": "something.firebase.io" + }, + { + "label": "Project Id", + "configProperty": "datasourceConfiguration.authentication.username", + "controlType": "INPUT_TEXT", + "isRequired": true, + "initialValue": "" + }, + { + "label": "Service Account Credentials", + "configProperty": "datasourceConfiguration.authentication.password", + "controlType": "INPUT_TEXT", + "dataType": "PASSWORD", + "isRequired": true, + "initialValue": "" + } + ] + } + ] +} diff --git a/app/server/appsmith-plugins/firestorePlugin/src/test/java/com/external/plugins/FirestorePluginTest.java b/app/server/appsmith-plugins/firestorePlugin/src/test/java/com/external/plugins/FirestorePluginTest.java new file mode 100644 index 0000000000..fa2cc2c56d --- /dev/null +++ b/app/server/appsmith-plugins/firestorePlugin/src/test/java/com/external/plugins/FirestorePluginTest.java @@ -0,0 +1,206 @@ +package com.external.plugins; + +import com.appsmith.external.models.ActionConfiguration; +import com.appsmith.external.models.ActionExecutionResult; +import com.appsmith.external.models.AuthenticationDTO; +import com.appsmith.external.models.DatasourceConfiguration; +import com.appsmith.external.models.Property; +import com.google.cloud.NoCredentials; +import com.google.cloud.ServiceOptions; +import com.google.cloud.firestore.DocumentSnapshot; +import com.google.cloud.firestore.Firestore; +import com.google.cloud.firestore.FirestoreOptions; +import lombok.extern.slf4j.Slf4j; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.testcontainers.containers.FirestoreEmulatorContainer; +import org.testcontainers.utility.DockerImageName; +import reactor.core.publisher.Mono; +import reactor.test.StepVerifier; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * Unit tests for the FirestorePlugin + */ +@Slf4j +public class FirestorePluginTest { + + static FirestorePlugin.FirestorePluginExecutor pluginExecutor = new FirestorePlugin.FirestorePluginExecutor(); + + @ClassRule + public static final FirestoreEmulatorContainer emulator = new FirestoreEmulatorContainer( + DockerImageName.parse("gcr.io/google.com/cloudsdktool/cloud-sdk:316.0.0-emulators") + ); + + static Firestore firestoreConnection; + + static DatasourceConfiguration dsConfig = new DatasourceConfiguration(); + + @BeforeClass + public static void setUp() throws ExecutionException, InterruptedException { + firestoreConnection = FirestoreOptions.newBuilder() + .setHost(emulator.getEmulatorEndpoint()) + .setCredentials(NoCredentials.getInstance()) + .setRetrySettings(ServiceOptions.getNoRetrySettings()) + .setProjectId("test-project") + .build() + .getService(); + + firestoreConnection.document("initial/one").set(Map.of("value", 1, "name", "one", "isPlural", false)).get(); + firestoreConnection.document("initial/two").set(Map.of("value", 2, "name", "two", "isPlural", true)).get(); + firestoreConnection.document("changing/to-update").set(Map.of("value", 1)).get(); + firestoreConnection.document("changing/to-delete").set(Map.of("value", 1)).get(); + + dsConfig.setUrl(emulator.getEmulatorEndpoint()); + dsConfig.setAuthentication(new AuthenticationDTO()); + dsConfig.getAuthentication().setUsername("test-project"); + dsConfig.getAuthentication().setPassword(""); + } + + @Test + public void testGetSingleDocument() { + ActionConfiguration actionConfiguration = new ActionConfiguration(); + actionConfiguration.setPath("initial/one"); + actionConfiguration.setPluginSpecifiedTemplates(List.of(new Property("method", "GET_DOCUMENT"))); + + Mono resultMono = pluginExecutor + .execute(firestoreConnection, dsConfig, actionConfiguration); + + StepVerifier.create(resultMono) + .assertNext(result -> { + assertTrue(result.getIsExecutionSuccess()); + assertTrue(((Map) result.getBody()).entrySet().stream().allMatch(entry -> { + Object value = entry.getValue(); + switch (entry.getKey()) { + case "name": + return "one".equals(value); + case "isPlural": + return Boolean.FALSE.equals(value); + case "value": + return value.equals(1L); + default: + return false; + } + })); + }) + .verifyComplete(); + } + + @Test + public void testGetDocumentsInCollection() { + ActionConfiguration actionConfiguration = new ActionConfiguration(); + actionConfiguration.setPath("initial"); + actionConfiguration.setPluginSpecifiedTemplates(List.of(new Property("method", "GET_COLLECTION"))); + + Mono resultMono = pluginExecutor + .execute(firestoreConnection, dsConfig, actionConfiguration); + + StepVerifier.create(resultMono) + .assertNext(result -> { + assertTrue(result.getIsExecutionSuccess()); + assertEquals(2, ((List) result.getBody()).size()); + }) + .verifyComplete(); + } + + @Test + public void testSetNewDocument() { + ActionConfiguration actionConfiguration = new ActionConfiguration(); + actionConfiguration.setPath("test/new_with_set"); + actionConfiguration.setBody("{\n" + + " \"firstName\": \"test\",\n" + + " \"lastName\":\"lastTest\"\n" + + "}"); + + actionConfiguration.setPluginSpecifiedTemplates(List.of(new Property("method", "SET_DOCUMENT"))); + + Mono resultMono = pluginExecutor + .execute(firestoreConnection, dsConfig, actionConfiguration); + + StepVerifier.create(resultMono) + .assertNext(result -> { + assertTrue(result.getIsExecutionSuccess()); + }) + .verifyComplete(); + } + + @Test + public void testCreateDocument() { + ActionConfiguration actionConfiguration = new ActionConfiguration(); + actionConfiguration.setPath("test/new_with_create"); + actionConfiguration.setBody("{\n" + + " \"firstName\": \"test\",\n" + + " \"lastName\":\"lastTest\"\n" + + "}"); + + actionConfiguration.setPluginSpecifiedTemplates(List.of(new Property("method", "CREATE_DOCUMENT"))); + + Mono resultMono = pluginExecutor + .execute(firestoreConnection, dsConfig, actionConfiguration); + + StepVerifier.create(resultMono) + .assertNext(result -> { + assertTrue(result.getIsExecutionSuccess()); + }) + .verifyComplete(); + } + + @Test + public void testUpdateDocument() { + ActionConfiguration actionConfiguration = new ActionConfiguration(); + actionConfiguration.setPath("changing/to-update"); + actionConfiguration.setBody("{\n" + + " \"value\": 2\n" + + "}"); + + actionConfiguration.setPluginSpecifiedTemplates(List.of(new Property("method", "UPDATE_DOCUMENT"))); + + Mono resultMono = pluginExecutor + .execute(firestoreConnection, dsConfig, actionConfiguration); + + StepVerifier.create(resultMono) + .assertNext(result -> { + assertTrue(result.getIsExecutionSuccess()); + try { + final DocumentSnapshot documentSnapshot = firestoreConnection.document("changing/to-update").get().get(); + assertTrue(documentSnapshot.exists()); + assertEquals(2L, documentSnapshot.getLong("value").longValue()); + } catch (NullPointerException | InterruptedException | ExecutionException e) { + e.printStackTrace(); + } + }) + .verifyComplete(); + } + + @Test + public void testDeleteDocument() { + ActionConfiguration actionConfiguration = new ActionConfiguration(); + actionConfiguration.setPath("changing/to-delete"); + + actionConfiguration.setPluginSpecifiedTemplates(List.of(new Property("method", "DELETE_DOCUMENT"))); + + Mono resultMono = pluginExecutor + .execute(firestoreConnection, dsConfig, actionConfiguration); + + StepVerifier.create(resultMono) + .assertNext(result -> { + assertTrue(result.getIsExecutionSuccess()); + try { + final DocumentSnapshot documentSnapshot = firestoreConnection.document("changing/to-delete").get().get(); + assertFalse(documentSnapshot.exists()); + } catch (InterruptedException | ExecutionException e) { + e.printStackTrace(); + } + }) + .verifyComplete(); + } + +} diff --git a/app/server/appsmith-plugins/pom.xml b/app/server/appsmith-plugins/pom.xml index 3e2cedb946..608773285d 100644 --- a/app/server/appsmith-plugins/pom.xml +++ b/app/server/appsmith-plugins/pom.xml @@ -23,5 +23,6 @@ dynamoPlugin redisPlugin mssqlPlugin + firestorePlugin \ No newline at end of file diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/DatabaseChangelog.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/DatabaseChangelog.java index 27d743175a..c971c43481 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/DatabaseChangelog.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/DatabaseChangelog.java @@ -1247,4 +1247,24 @@ public class DatabaseChangelog { ); } + @ChangeSet(order = "043", id = "add-firestore-plugin", author = "") + public void addFirestorePlugin(MongoTemplate mongoTemplate) { + Plugin plugin = new Plugin(); + plugin.setName("Firestore"); + plugin.setType(PluginType.DB); + plugin.setPackageName("firestore-plugin"); + plugin.setUiComponent("DbEditorForm"); + plugin.setResponseType(Plugin.ResponseType.JSON); + plugin.setIconLocation("https://s3.us-east-2.amazonaws.com/assets.appsmith.com/Firestore.png"); + plugin.setDocumentationLink("https://docs.appsmith.com/core-concepts/connecting-to-databases/querying-firestore"); + plugin.setDefaultInstall(true); + try { + mongoTemplate.insert(plugin); + } catch (DuplicateKeyException e) { + log.warn(plugin.getPackageName() + " already present in database."); + } + + installPluginToAllOrganizations(mongoTemplate, plugin.getId()); + } + } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/DatasourceContextServiceImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/DatasourceContextServiceImpl.java index 281781bb42..566d56bd53 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/DatasourceContextServiceImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/DatasourceContextServiceImpl.java @@ -54,7 +54,11 @@ public class DatasourceContextServiceImpl implements DatasourceContextService { if (datasourceId == null) { log.debug("This is a dry run or an embedded datasource. The datasource context would not exist in this scenario"); - } else if (datasourceContextMap.get(datasourceId) != null && !isStale) { + } else if (datasourceContextMap.get(datasourceId) != null + // The following condition happens when there's a timout in the middle of destroying a connection and + // the reactive flow interrupts, resulting in the destroy operation not completing. + && datasourceContextMap.get(datasourceId).getConnection() != null + && !isStale) { log.debug("resource context exists. Returning the same."); return Mono.just(datasourceContextMap.get(datasourceId)); }