Firestore Integration (#1799)

* Adding the skeleton for Firestore integration

* Adding the datasource & query editor forms

Also adding the database changelog for the firestore plugin

Commenting out the firestore.close() connection because that causes issues with multiple Firestore tenants running in the same JVM.

* Adding the code for fetching the structure of collections from Firestore

* Use single document path field for Firestore

* Fix potential NPE when datasource destroy timeouts

* Work in progress on collection level ops for Firestore

* Get documents in a collection now works

* Add collection level querying support

* Mild refactoring

* Fix NPE when some fields are missing

* Hide clientJSON as a password field for Firestore

* Make collection level querying reactive

* Make reactive

* Validate before connecting

* Add tests for all supported methods in Firestore

* Fix forms for Firestore with hidden fields

* Hide limit and order by fields when not needed

* Restore log entry deleted by mistake

* Use S3 URL for Firestore/Firebase logo

* Add comments detailing why some code is commented

* Make parsing JSON reactive and fix subscribe calls

* Fix reactive scheduler

Co-authored-by: Arpit Mohan <arpit@appsmith.com>
This commit is contained in:
Shrikant Sharat Kandula 2020-11-23 18:42:33 +05:30 committed by GitHub
parent c07264f1c1
commit 74cd362057
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 1164 additions and 2 deletions

View File

@ -20,7 +20,7 @@ public interface PluginExecutor<C> extends ExtensionPoint {
* to the parameters in Datasource Configuration
* @param datasourceConfiguration : These are the configurations which have been used to create a Datasource from a Plugin
* @param actionConfiguration : These are the configurations which have been used to create an Action from a Datasource.
* @return ActionExecutionResult : This object is returned to the user which contains the result values from the execution.
* @return ActionExecutionResult : This object is returned to the user which contains the result values from the execution.
*/
Mono<ActionExecutionResult> execute(C connection, DatasourceConfiguration datasourceConfiguration, ActionConfiguration actionConfiguration);
@ -40,14 +40,48 @@ public interface PluginExecutor<C> extends ExtensionPoint {
*/
void datasourceDestroy(C connection);
/**
* This function tells the platform if datasource is valid by checking the set of invalid strings.
* If empty, the datasource is valid. This set of invalid strings is populated by
* {@link #validateDatasource(DatasourceConfiguration)}
*
* @param datasourceConfiguration
* @return boolean
*/
default boolean isDatasourceValid(DatasourceConfiguration datasourceConfiguration) {
return CollectionUtils.isEmpty(validateDatasource(datasourceConfiguration));
}
/**
* This function checks if the datasource is valid. It should only check if all the mandatory fields are filled and
* if the values are of the right format. It does NOT check the validity of those fields.
* Please use {@link #testDatasource(DatasourceConfiguration)} to establish the correctness of those fields.
*
* If the datasource configuration is valid, it should return an empty set of invalid strings.
* If not, it should return the list of invalid messages as a set.
*
* @param datasourceConfiguration : The datasource to be validated
* @return Set : The set of invalid strings informing the user of all the invalid fields
*/
Set<String> validateDatasource(DatasourceConfiguration datasourceConfiguration);
/**
* This function tests the datasource by executing a test query or hitting the endpoint to check the correctness
* of the values provided in the datasource configuration
*
* @param datasourceConfiguration
* @return
*/
Mono<DatasourceTestResult> testDatasource(DatasourceConfiguration datasourceConfiguration);
/**
* This function fetches the structure of the tables/collections in the datasource. It's used to make query creation
* easier for the user.
*
* @param connection
* @param datasourceConfiguration
* @return
*/
default Mono<DatasourceStructure> getStructure(C connection, DatasourceConfiguration datasourceConfiguration) {
return Mono.empty();
}

View File

@ -0,0 +1,5 @@
plugin.id=firestore-plugin
plugin.class=com.external.plugins.FirestorePlugin
plugin.version=1.0-SNAPSHOT
plugin.provider=tech@appsmith.com
plugin.dependencies=

View File

@ -0,0 +1,152 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.external.plugins</groupId>
<artifactId>firestorePlugin</artifactId>
<version>1.0-SNAPSHOT</version>
<name>firestorePlugin</name>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<java.version>11</java.version>
<maven.compiler.source>${java.version}</maven.compiler.source>
<maven.compiler.target>${java.version}</maven.compiler.target>
<plugin.id>firestore-plugin</plugin.id>
<plugin.class>com.external.plugins.FirestorePlugin</plugin.class>
<plugin.version>1.0-SNAPSHOT</plugin.version>
<plugin.provider>tech@appsmith.com</plugin.provider>
<plugin.dependencies/>
</properties>
<dependencies>
<dependency>
<groupId>org.pf4j</groupId>
<artifactId>pf4j-spring</artifactId>
<version>0.6.0</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.appsmith</groupId>
<artifactId>interfaces</artifactId>
<version>1.0-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>1.18.8</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.google.firebase</groupId>
<artifactId>firebase-admin</artifactId>
<version>7.0.1</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</exclusion>
</exclusions>
</dependency>
<!-- Test Dependencies -->
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.13.1</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>testcontainers</artifactId>
<version>1.15.0-rc2</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>gcloud</artifactId>
<version>1.15.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.projectreactor</groupId>
<artifactId>reactor-test</artifactId>
<version>3.2.11.RELEASE</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<version>3.1.0</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>3.2.4</version>
<configuration>
<minimizeJar>false</minimizeJar>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<manifestEntries>
<Plugin-Id>${plugin.id}</Plugin-Id>
<Plugin-Class>${plugin.class}</Plugin-Class>
<Plugin-Version>${plugin.version}</Plugin-Version>
<Plugin-Provider>${plugin.provider}</Plugin-Provider>
</manifestEntries>
</transformer>
</transformers>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
</excludes>
</filter>
</filters>
</configuration>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<includeScope>runtime</includeScope>
<outputDirectory>${project.build.directory}/lib</outputDirectory>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

View File

@ -0,0 +1,475 @@
package com.external.plugins;
import com.appsmith.external.models.ActionConfiguration;
import com.appsmith.external.models.ActionExecutionResult;
import com.appsmith.external.models.AuthenticationDTO;
import com.appsmith.external.models.DatasourceConfiguration;
import com.appsmith.external.models.DatasourceStructure;
import com.appsmith.external.models.DatasourceTestResult;
import com.appsmith.external.models.Property;
import com.appsmith.external.pluginExceptions.AppsmithPluginError;
import com.appsmith.external.pluginExceptions.AppsmithPluginException;
import com.appsmith.external.plugins.BasePlugin;
import com.appsmith.external.plugins.PluginExecutor;
import com.google.api.core.ApiFuture;
import com.google.auth.oauth2.GoogleCredentials;
import com.google.cloud.firestore.CollectionReference;
import com.google.cloud.firestore.DocumentReference;
import com.google.cloud.firestore.DocumentSnapshot;
import com.google.cloud.firestore.Firestore;
import com.google.cloud.firestore.QueryDocumentSnapshot;
import com.google.cloud.firestore.QuerySnapshot;
import com.google.cloud.firestore.WriteResult;
import com.google.firebase.FirebaseApp;
import com.google.firebase.FirebaseOptions;
import com.google.firebase.cloud.FirestoreClient;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.pf4j.Extension;
import org.pf4j.PluginWrapper;
import org.springframework.util.CollectionUtils;
import reactor.core.Exceptions;
import reactor.core.publisher.Mono;
import reactor.core.scheduler.Scheduler;
import reactor.core.scheduler.Schedulers;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.lang.reflect.InvocationTargetException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
/**
* Datasource properties:
* 1. Client JSON
* <p>
* Query Action properties:
* 1. method: Dropdown
* 2. collection: String
* 3. documentKey: String
*/
public class FirestorePlugin extends BasePlugin {
public FirestorePlugin(PluginWrapper wrapper) {
super(wrapper);
}
@Slf4j
@Extension
public static class FirestorePluginExecutor implements PluginExecutor<Firestore> {
private final Scheduler scheduler = Schedulers.elastic();
@Override
public Mono<ActionExecutionResult> execute(Firestore connection,
DatasourceConfiguration datasourceConfiguration,
ActionConfiguration actionConfiguration) {
final String path = actionConfiguration.getPath();
if (StringUtils.isBlank(path)) {
return Mono.error(new AppsmithPluginException(
AppsmithPluginError.PLUGIN_ERROR,
"Document/Collection path cannot be empty"
));
}
if (path.startsWith("/") || path.endsWith("/")) {
return Mono.error(new AppsmithPluginException(
AppsmithPluginError.PLUGIN_ERROR,
"Firestore paths should not begin or end with `/` character."
));
}
final List<Property> properties = actionConfiguration.getPluginSpecifiedTemplates();
final com.external.plugins.Method method = CollectionUtils.isEmpty(properties)
? null
: com.external.plugins.Method.valueOf(properties.get(0).getValue());
if (method == null) {
return Mono.error(new AppsmithPluginException(
AppsmithPluginError.PLUGIN_ERROR,
"Missing Firestore method."
));
}
return Mono
.justOrEmpty(actionConfiguration.getBody())
.defaultIfEmpty("")
.flatMap(strBody -> {
if (StringUtils.isBlank(strBody)) {
return Mono.just(Collections.emptyMap());
}
try {
return Mono.just(objectMapper.readValue(strBody, HashMap.class));
} catch (IOException e) {
return Mono.error(new AppsmithPluginException(
AppsmithPluginError.PLUGIN_ERROR,
e.getMessage()
));
}
})
.flatMap(mapBody -> {
if (mapBody.isEmpty() && method.isBodyNeeded()) {
return Mono.error(new AppsmithPluginException(
AppsmithPluginError.PLUGIN_ERROR,
"The method " + method.toString() + " needs a non-empty body to work."
));
}
return Mono.just((Map<String, Object>) mapBody);
})
.flatMap(mapBody -> {
if (method.isDocumentLevel()) {
return handleDocumentLevelMethod(connection, path, method, mapBody);
} else {
return handleCollectionLevelMethod(connection, path, method, properties);
}
})
.subscribeOn(scheduler);
}
public Mono<ActionExecutionResult> handleDocumentLevelMethod(
Firestore connection,
String path,
com.external.plugins.Method method,
Map<String, Object> mapBody
) {
return Mono.just(method)
// Get the actual Java method to be called.
.flatMap(method1 -> {
final String methodName = method1.toString().split("_")[0].toLowerCase();
try {
switch (method1) {
case GET_DOCUMENT:
case DELETE_DOCUMENT:
return Mono.justOrEmpty(DocumentReference.class.getMethod(methodName));
case SET_DOCUMENT:
case CREATE_DOCUMENT:
case UPDATE_DOCUMENT:
return Mono.justOrEmpty(DocumentReference.class.getMethod(methodName, Map.class));
default:
return Mono.error(new AppsmithPluginException(
AppsmithPluginError.PLUGIN_ERROR,
"Invalid document-level method " + method1.toString()
));
}
} catch (NoSuchMethodException e) {
return Mono.error(new AppsmithPluginException(
AppsmithPluginError.PLUGIN_ERROR,
"Error getting actual method for operation " + method1.toString()
));
}
})
// Call that method and get a Future of the result.
.flatMap(operationMethod -> {
DocumentReference document = connection.document(path);
Object objFuture;
try {
if (operationMethod.getParameterCount() == 1) {
objFuture = operationMethod.invoke(document, mapBody);
} else {
objFuture = operationMethod.invoke(document);
}
} catch (IllegalAccessException | InvocationTargetException e) {
return Mono.error(new AppsmithPluginException(AppsmithPluginError.PLUGIN_ERROR, e.getMessage()));
}
return Mono.just((ApiFuture<Object>) objFuture);
})
// Consume the Future to get the actual result object.
.flatMap(resultFuture -> {
try {
return Mono.just(resultFuture.get());
} catch (InterruptedException | ExecutionException e) {
return Mono.error(new AppsmithPluginException(AppsmithPluginError.PLUGIN_ERROR, e.getMessage()));
}
})
// Build a response object with the result.
.flatMap(objResult1 -> {
ActionExecutionResult result = new ActionExecutionResult();
try {
result.setBody(resultToMap(objResult1));
} catch (AppsmithPluginException e) {
return Mono.error(e);
}
result.setIsExecutionSuccess(true);
System.out.println(
Thread.currentThread().getName()
+ ": In the Firestore Plugin, got action execution result: "
+ result.toString()
);
return Mono.just(result);
});
}
public Mono<ActionExecutionResult> handleCollectionLevelMethod(
Firestore connection,
String path,
com.external.plugins.Method method,
List<Property> properties
) {
final String orderBy = properties.size() > 1 && properties.get(1) != null ? properties.get(1).getValue() : null;
final int limit = properties.size() > 2 && properties.get(2) != null ? Integer.parseInt(properties.get(2).getValue()) : 10;
final String queryFieldPath = properties.size() > 3 && properties.get(3) != null ? properties.get(3).getValue() : null;
final Op operator = properties.size() > 4 && properties.get(4) != null ? Op.valueOf(properties.get(4).getValue()) : null;
final String queryValue = properties.size() > 5 && properties.get(5) != null ? properties.get(5).getValue() : null;
return Mono.just(connection.collection(path))
// Apply ordering, if provided.
.map(query1 -> StringUtils.isEmpty(orderBy) ? query1 : query1.orderBy(orderBy))
// Apply where condition, if provided.
.flatMap(query1 -> {
if (StringUtils.isEmpty(queryFieldPath) || operator == null || queryValue == null) {
return Mono.just(query1);
}
switch (operator) {
case LT:
return Mono.just(query1.whereLessThan(queryFieldPath, queryValue));
case LTE:
return Mono.just(query1.whereLessThanOrEqualTo(queryFieldPath, queryValue));
case EQ:
return Mono.just(query1.whereEqualTo(queryFieldPath, queryValue));
// TODO: NOT_EQ operator support is awaited in the next version of Firestore driver.
// case NOT_EQ:
// return Mono.just(query1.whereNotEqualTo(queryFieldPath, queryValue));
case GT:
return Mono.just(query1.whereGreaterThan(queryFieldPath, queryValue));
case GTE:
return Mono.just(query1.whereGreaterThanOrEqualTo(queryFieldPath, queryValue));
case ARRAY_CONTAINS:
return Mono.just(query1.whereArrayContains(queryFieldPath, queryValue));
case ARRAY_CONTAINS_ANY:
try {
return Mono.just(query1.whereArrayContainsAny(queryFieldPath, parseList(queryValue)));
} catch (IOException e) {
return Mono.error(new AppsmithPluginException(
AppsmithPluginError.PLUGIN_ERROR,
"Unable to parse condition value as a JSON list."
));
}
case IN:
try {
return Mono.just(query1.whereIn(queryFieldPath, parseList(queryValue)));
} catch (IOException e) {
return Mono.error(new AppsmithPluginException(
AppsmithPluginError.PLUGIN_ERROR,
"Unable to parse condition value as a JSON list."
));
}
// TODO: NOT_IN operator support is awaited in the next version of Firestore driver.
// case NOT_IN:
// return Mono.just(query1.whereNotIn(queryFieldPath, queryValue));
default:
return Mono.error(new AppsmithPluginException(
AppsmithPluginError.PLUGIN_ERROR,
"Unsupported operator for `where` condition " + operator.toString() + "."
));
}
})
// Apply limit, always provided, since without it we can inadvertently end up processing too much data.
.map(query1 -> query1.limit(limit))
// Run the Firestore query to get a Future of the results.
.flatMap(query1 -> {
switch (method) {
case GET_COLLECTION:
return Mono.just(query1.get());
default:
return Mono.error(new AppsmithPluginException(
AppsmithPluginError.PLUGIN_ERROR,
"Unknown collection method: " + method.toString() + "."
));
}
})
// Consume the future to get the actual results.
.flatMap(resultFuture -> {
try {
return Mono.just(resultFuture.get());
} catch (InterruptedException | ExecutionException e) {
return Mono.error(new AppsmithPluginException(AppsmithPluginError.PLUGIN_ERROR, e.getMessage()));
}
})
// Build response object with the results from the Future.
.flatMap(objResult1 -> {
ActionExecutionResult result = new ActionExecutionResult();
try {
result.setBody(resultToMap(objResult1));
} catch (AppsmithPluginException e) {
return Mono.error(e);
}
result.setIsExecutionSuccess(true);
System.out.println(
Thread.currentThread().getName()
+ ": In the Firestore Plugin, got action execution result: "
+ result.toString()
);
return Mono.just(result);
});
}
private Object resultToMap(Object objResult) throws AppsmithPluginException {
if (objResult instanceof WriteResult) {
WriteResult writeResult = (WriteResult) objResult;
Map<String, Object> resultMap = new HashMap<>();
resultMap.put("lastUpdateTime", writeResult.getUpdateTime());
return resultMap;
} else if (objResult instanceof DocumentSnapshot) {
DocumentSnapshot documentSnapshot = (DocumentSnapshot) objResult;
Map<String, Object> resultMap = new HashMap<>();
if (documentSnapshot.getData() != null) {
resultMap = documentSnapshot.getData();
}
return resultMap;
} else if (objResult instanceof QuerySnapshot) {
QuerySnapshot querySnapshot = (QuerySnapshot) objResult;
List<Map<String, Object>> documents = new ArrayList<>();
for (QueryDocumentSnapshot documentSnapshot : querySnapshot.getDocuments()) {
documents.add(documentSnapshot.getData());
}
return documents;
} else {
throw new AppsmithPluginException(
AppsmithPluginError.PLUGIN_ERROR,
"Unable to serialize object of type " + objResult.getClass().getName() + "."
);
}
}
@Override
public Mono<Firestore> datasourceCreate(DatasourceConfiguration datasourceConfiguration) {
final AuthenticationDTO authentication = datasourceConfiguration.getAuthentication();
final Set<String> errors = validateDatasource(datasourceConfiguration);
if (!CollectionUtils.isEmpty(errors)) {
return Mono.error(new AppsmithPluginException(AppsmithPluginError.PLUGIN_ERROR, errors.iterator().next()));
}
final String projectId = authentication.getUsername();
final String clientJson = authentication.getPassword();
InputStream serviceAccount = new ByteArrayInputStream(clientJson.getBytes());
return Mono
.fromSupplier(() -> {
GoogleCredentials credentials;
try {
credentials = GoogleCredentials.fromStream(serviceAccount);
} catch (IOException e) {
throw Exceptions.propagate(new AppsmithPluginException(
AppsmithPluginError.PLUGIN_ERROR,
e.getMessage()
));
}
return FirebaseOptions.builder()
.setDatabaseUrl(datasourceConfiguration.getUrl())
.setProjectId(projectId)
.setCredentials(credentials)
.build();
})
.onErrorMap(Exceptions::unwrap)
.map(options -> {
try {
return FirebaseApp.getInstance(projectId);
} catch (IllegalStateException e) {
return FirebaseApp.initializeApp(options, projectId);
}
})
.map(FirestoreClient::getFirestore)
.subscribeOn(scheduler);
}
@Override
public void datasourceDestroy(Firestore connection) {
// This is empty because there's no concept of destroying a Firestore connection here.
// When the datasource is updated, the FirebaseApp instance will delete & re-create the app
}
@Override
public Set<String> validateDatasource(DatasourceConfiguration datasourceConfiguration) {
final AuthenticationDTO authentication = datasourceConfiguration.getAuthentication();
Set<String> invalids = new HashSet<>();
if (authentication == null) {
invalids.add("Missing ProjectID and ClientJSON in datasource.");
} else {
if (StringUtils.isEmpty(authentication.getUsername())) {
invalids.add("Missing ProjectID in datasource.");
}
if (StringUtils.isEmpty(authentication.getPassword())) {
invalids.add("Missing ClientJSON in datasource.");
}
}
if (StringUtils.isBlank(datasourceConfiguration.getUrl())) {
invalids.add("Missing Firestore URL.");
}
return invalids;
}
private <T> List<T> parseList(String arrayJson) throws IOException {
return (List<T>) objectMapper.readValue(arrayJson, ArrayList.class);
}
@Override
public Mono<DatasourceTestResult> testDatasource(DatasourceConfiguration datasourceConfiguration) {
return datasourceCreate(datasourceConfiguration)
.map(connection -> new DatasourceTestResult())
.onErrorResume(error -> Mono.just(new DatasourceTestResult(error.getMessage())));
}
@Override
public Mono<DatasourceStructure> getStructure(Firestore connection, DatasourceConfiguration datasourceConfiguration) {
return Mono
.fromSupplier(() -> {
Iterable<CollectionReference> collectionReferences = connection.listCollections();
List<DatasourceStructure.Table> tables = StreamSupport.stream(collectionReferences.spliterator(), false)
.map(collectionReference -> {
String id = collectionReference.getId();
final ArrayList<DatasourceStructure.Template> templates = new ArrayList<>();
return new DatasourceStructure.Table(
DatasourceStructure.TableType.COLLECTION,
id,
new ArrayList<>(),
new ArrayList<>(),
templates
);
})
.collect(Collectors.toList());
DatasourceStructure structure = new DatasourceStructure();
structure.setTables(tables);
return structure;
})
.subscribeOn(scheduler);
}
}
}

View File

@ -0,0 +1,27 @@
package com.external.plugins;
public enum Method {
GET_DOCUMENT(true, false),
GET_COLLECTION(false, false),
SET_DOCUMENT(true, true),
CREATE_DOCUMENT(true, true),
UPDATE_DOCUMENT(true, true),
DELETE_DOCUMENT(true, false),
;
private final boolean isDocumentLevel;
private final boolean isBodyNeeded;
Method(boolean isDocumentLevel, boolean isBodyNeeded) {
this.isDocumentLevel = isDocumentLevel;
this.isBodyNeeded = isBodyNeeded;
}
public boolean isDocumentLevel() {
return isDocumentLevel;
}
public boolean isBodyNeeded() {
return isBodyNeeded;
}
}

View File

@ -0,0 +1,14 @@
package com.external.plugins;
public enum Op {
LT,
LTE,
EQ,
NOT_EQ,
GT,
GTE,
ARRAY_CONTAINS,
IN,
ARRAY_CONTAINS_ANY,
NOT_IN,
}

View File

@ -0,0 +1,192 @@
{
"editor": [
{
"sectionName": "",
"id": 1,
"children": [
{
"label": "Method Key",
"configProperty": "actionConfiguration.pluginSpecifiedTemplates[0].key",
"controlType": "INPUT_TEXT",
"hidden": true,
"initialValue": "method"
},
{
"label": "Method",
"configProperty": "actionConfiguration.pluginSpecifiedTemplates[0].value",
"controlType": "DROP_DOWN",
"isRequired": true,
"initialValue": "GET_DOCUMENT",
"options": [
{
"label": "Get Single Document",
"value": "GET_DOCUMENT"
},
{
"label": "Get Documents in Collection",
"value": "GET_COLLECTION"
},
{
"label": "Set Document",
"value": "SET_DOCUMENT"
},
{
"label": "Create Document",
"value": "CREATE_DOCUMENT"
},
{
"label": "Update Document",
"value": "UPDATE_DOCUMENT"
},
{
"label": "Delete Document",
"value": "DELETE_DOCUMENT"
}
]
},
{
"label": "Document/Collection Path",
"configProperty": "actionConfiguration.path",
"controlType": "INPUT_TEXT",
"isRequired": true,
"initialValue": ""
},
{
"label": "Order By Key",
"configProperty": "actionConfiguration.pluginSpecifiedTemplates[1].key",
"controlType": "INPUT_TEXT",
"hidden": true,
"initialValue": "orderBy"
},
{
"label": "Order By",
"configProperty": "actionConfiguration.pluginSpecifiedTemplates[1].value",
"controlType": "INPUT_TEXT",
"hidden": {
"path": "actionConfiguration.pluginSpecifiedTemplates[0].value",
"comparison": "NOT_EQUALS",
"value": "GET_COLLECTION"
},
"initialValue": ""
},
{
"label": "Limit Documents Key",
"configProperty": "actionConfiguration.pluginSpecifiedTemplates[2].key",
"controlType": "INPUT_TEXT",
"hidden": true,
"initialValue": "limit"
},
{
"label": "Limit Documents",
"configProperty": "actionConfiguration.pluginSpecifiedTemplates[2].value",
"controlType": "INPUT_TEXT",
"hidden": {
"path": "actionConfiguration.pluginSpecifiedTemplates[0].value",
"comparison": "NOT_EQUALS",
"value": "GET_COLLECTION"
},
"initialValue": "10"
},
{
"sectionName": "Query",
"id": 2,
"children": [
{
"label": "Field Path Key",
"configProperty": "actionConfiguration.pluginSpecifiedTemplates[3].key",
"controlType": "INPUT_TEXT",
"hidden": true,
"initialValue": "fieldPath"
},
{
"label": "Where Condition: Field Path (leave emtpy to not apply any conditions)",
"configProperty": "actionConfiguration.pluginSpecifiedTemplates[3].value",
"controlType": "INPUT_TEXT",
"hidden": {
"path": "actionConfiguration.pluginSpecifiedTemplates[0].value",
"comparison": "NOT_EQUALS",
"value": "GET_COLLECTION"
},
"initialValue": ""
},
{
"label": "Operator Key",
"configProperty": "actionConfiguration.pluginSpecifiedTemplates[4].key",
"controlType": "INPUT_TEXT",
"hidden": true,
"initialValue": "operator"
},
{
"label": "Where Condition: Operator",
"configProperty": "actionConfiguration.pluginSpecifiedTemplates[4].value",
"controlType": "DROP_DOWN",
"hidden": {
"path": "actionConfiguration.pluginSpecifiedTemplates[0].value",
"comparison": "NOT_EQUALS",
"value": "GET_COLLECTION"
},
"initialValue": "EQ",
"options": [
{
"label": "<",
"value": "LT"
},
{
"label": "<=",
"value": "LTE"
},
{
"label": "==",
"value": "EQ"
},
{
"label": ">=",
"value": "GTE"
},
{
"label": ">",
"value": "GT"
},
{
"label": "array-contains",
"value": "ARRAY_CONTAINS"
},
{
"label": "in",
"value": "IN"
},
{
"label": "array-contains-any",
"value": "ARRAY_CONTAINS_ANY"
}
]
},
{
"label": "Value Key",
"configProperty": "actionConfiguration.pluginSpecifiedTemplates[5].key",
"controlType": "INPUT_TEXT",
"hidden": true,
"initialValue": "fieldValue"
},
{
"label": "Where Condition: Value",
"configProperty": "actionConfiguration.pluginSpecifiedTemplates[5].value",
"controlType": "INPUT_TEXT",
"hidden": {
"path": "actionConfiguration.pluginSpecifiedTemplates[0].value",
"comparison": "NOT_EQUALS",
"value": "GET_COLLECTION"
},
"initialValue": ""
}
]
},
{
"label": "Body",
"configProperty": "actionConfiguration.body",
"controlType": "QUERY_DYNAMIC_TEXT"
}
]
}
]
}

View File

@ -0,0 +1,32 @@
{
"form": [
{
"sectionName": "Details",
"id": 1,
"children": [
{
"label": "Database URL",
"configProperty": "datasourceConfiguration.url",
"controlType": "INPUT_TEXT",
"isRequired": true,
"initialValue": "something.firebase.io"
},
{
"label": "Project Id",
"configProperty": "datasourceConfiguration.authentication.username",
"controlType": "INPUT_TEXT",
"isRequired": true,
"initialValue": ""
},
{
"label": "Service Account Credentials",
"configProperty": "datasourceConfiguration.authentication.password",
"controlType": "INPUT_TEXT",
"dataType": "PASSWORD",
"isRequired": true,
"initialValue": ""
}
]
}
]
}

View File

@ -0,0 +1,206 @@
package com.external.plugins;
import com.appsmith.external.models.ActionConfiguration;
import com.appsmith.external.models.ActionExecutionResult;
import com.appsmith.external.models.AuthenticationDTO;
import com.appsmith.external.models.DatasourceConfiguration;
import com.appsmith.external.models.Property;
import com.google.cloud.NoCredentials;
import com.google.cloud.ServiceOptions;
import com.google.cloud.firestore.DocumentSnapshot;
import com.google.cloud.firestore.Firestore;
import com.google.cloud.firestore.FirestoreOptions;
import lombok.extern.slf4j.Slf4j;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.Test;
import org.testcontainers.containers.FirestoreEmulatorContainer;
import org.testcontainers.utility.DockerImageName;
import reactor.core.publisher.Mono;
import reactor.test.StepVerifier;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
/**
* Unit tests for the FirestorePlugin
*/
@Slf4j
public class FirestorePluginTest {
static FirestorePlugin.FirestorePluginExecutor pluginExecutor = new FirestorePlugin.FirestorePluginExecutor();
@ClassRule
public static final FirestoreEmulatorContainer emulator = new FirestoreEmulatorContainer(
DockerImageName.parse("gcr.io/google.com/cloudsdktool/cloud-sdk:316.0.0-emulators")
);
static Firestore firestoreConnection;
static DatasourceConfiguration dsConfig = new DatasourceConfiguration();
@BeforeClass
public static void setUp() throws ExecutionException, InterruptedException {
firestoreConnection = FirestoreOptions.newBuilder()
.setHost(emulator.getEmulatorEndpoint())
.setCredentials(NoCredentials.getInstance())
.setRetrySettings(ServiceOptions.getNoRetrySettings())
.setProjectId("test-project")
.build()
.getService();
firestoreConnection.document("initial/one").set(Map.of("value", 1, "name", "one", "isPlural", false)).get();
firestoreConnection.document("initial/two").set(Map.of("value", 2, "name", "two", "isPlural", true)).get();
firestoreConnection.document("changing/to-update").set(Map.of("value", 1)).get();
firestoreConnection.document("changing/to-delete").set(Map.of("value", 1)).get();
dsConfig.setUrl(emulator.getEmulatorEndpoint());
dsConfig.setAuthentication(new AuthenticationDTO());
dsConfig.getAuthentication().setUsername("test-project");
dsConfig.getAuthentication().setPassword("");
}
@Test
public void testGetSingleDocument() {
ActionConfiguration actionConfiguration = new ActionConfiguration();
actionConfiguration.setPath("initial/one");
actionConfiguration.setPluginSpecifiedTemplates(List.of(new Property("method", "GET_DOCUMENT")));
Mono<ActionExecutionResult> resultMono = pluginExecutor
.execute(firestoreConnection, dsConfig, actionConfiguration);
StepVerifier.create(resultMono)
.assertNext(result -> {
assertTrue(result.getIsExecutionSuccess());
assertTrue(((Map<String, Object>) result.getBody()).entrySet().stream().allMatch(entry -> {
Object value = entry.getValue();
switch (entry.getKey()) {
case "name":
return "one".equals(value);
case "isPlural":
return Boolean.FALSE.equals(value);
case "value":
return value.equals(1L);
default:
return false;
}
}));
})
.verifyComplete();
}
@Test
public void testGetDocumentsInCollection() {
ActionConfiguration actionConfiguration = new ActionConfiguration();
actionConfiguration.setPath("initial");
actionConfiguration.setPluginSpecifiedTemplates(List.of(new Property("method", "GET_COLLECTION")));
Mono<ActionExecutionResult> resultMono = pluginExecutor
.execute(firestoreConnection, dsConfig, actionConfiguration);
StepVerifier.create(resultMono)
.assertNext(result -> {
assertTrue(result.getIsExecutionSuccess());
assertEquals(2, ((List) result.getBody()).size());
})
.verifyComplete();
}
@Test
public void testSetNewDocument() {
ActionConfiguration actionConfiguration = new ActionConfiguration();
actionConfiguration.setPath("test/new_with_set");
actionConfiguration.setBody("{\n" +
" \"firstName\": \"test\",\n" +
" \"lastName\":\"lastTest\"\n" +
"}");
actionConfiguration.setPluginSpecifiedTemplates(List.of(new Property("method", "SET_DOCUMENT")));
Mono<ActionExecutionResult> resultMono = pluginExecutor
.execute(firestoreConnection, dsConfig, actionConfiguration);
StepVerifier.create(resultMono)
.assertNext(result -> {
assertTrue(result.getIsExecutionSuccess());
})
.verifyComplete();
}
@Test
public void testCreateDocument() {
ActionConfiguration actionConfiguration = new ActionConfiguration();
actionConfiguration.setPath("test/new_with_create");
actionConfiguration.setBody("{\n" +
" \"firstName\": \"test\",\n" +
" \"lastName\":\"lastTest\"\n" +
"}");
actionConfiguration.setPluginSpecifiedTemplates(List.of(new Property("method", "CREATE_DOCUMENT")));
Mono<ActionExecutionResult> resultMono = pluginExecutor
.execute(firestoreConnection, dsConfig, actionConfiguration);
StepVerifier.create(resultMono)
.assertNext(result -> {
assertTrue(result.getIsExecutionSuccess());
})
.verifyComplete();
}
@Test
public void testUpdateDocument() {
ActionConfiguration actionConfiguration = new ActionConfiguration();
actionConfiguration.setPath("changing/to-update");
actionConfiguration.setBody("{\n" +
" \"value\": 2\n" +
"}");
actionConfiguration.setPluginSpecifiedTemplates(List.of(new Property("method", "UPDATE_DOCUMENT")));
Mono<ActionExecutionResult> resultMono = pluginExecutor
.execute(firestoreConnection, dsConfig, actionConfiguration);
StepVerifier.create(resultMono)
.assertNext(result -> {
assertTrue(result.getIsExecutionSuccess());
try {
final DocumentSnapshot documentSnapshot = firestoreConnection.document("changing/to-update").get().get();
assertTrue(documentSnapshot.exists());
assertEquals(2L, documentSnapshot.getLong("value").longValue());
} catch (NullPointerException | InterruptedException | ExecutionException e) {
e.printStackTrace();
}
})
.verifyComplete();
}
@Test
public void testDeleteDocument() {
ActionConfiguration actionConfiguration = new ActionConfiguration();
actionConfiguration.setPath("changing/to-delete");
actionConfiguration.setPluginSpecifiedTemplates(List.of(new Property("method", "DELETE_DOCUMENT")));
Mono<ActionExecutionResult> resultMono = pluginExecutor
.execute(firestoreConnection, dsConfig, actionConfiguration);
StepVerifier.create(resultMono)
.assertNext(result -> {
assertTrue(result.getIsExecutionSuccess());
try {
final DocumentSnapshot documentSnapshot = firestoreConnection.document("changing/to-delete").get().get();
assertFalse(documentSnapshot.exists());
} catch (InterruptedException | ExecutionException e) {
e.printStackTrace();
}
})
.verifyComplete();
}
}

View File

@ -23,5 +23,6 @@
<module>dynamoPlugin</module>
<module>redisPlugin</module>
<module>mssqlPlugin</module>
<module>firestorePlugin</module>
</modules>
</project>

View File

@ -1247,4 +1247,24 @@ public class DatabaseChangelog {
);
}
@ChangeSet(order = "043", id = "add-firestore-plugin", author = "")
public void addFirestorePlugin(MongoTemplate mongoTemplate) {
Plugin plugin = new Plugin();
plugin.setName("Firestore");
plugin.setType(PluginType.DB);
plugin.setPackageName("firestore-plugin");
plugin.setUiComponent("DbEditorForm");
plugin.setResponseType(Plugin.ResponseType.JSON);
plugin.setIconLocation("https://s3.us-east-2.amazonaws.com/assets.appsmith.com/Firestore.png");
plugin.setDocumentationLink("https://docs.appsmith.com/core-concepts/connecting-to-databases/querying-firestore");
plugin.setDefaultInstall(true);
try {
mongoTemplate.insert(plugin);
} catch (DuplicateKeyException e) {
log.warn(plugin.getPackageName() + " already present in database.");
}
installPluginToAllOrganizations(mongoTemplate, plugin.getId());
}
}

View File

@ -54,7 +54,11 @@ public class DatasourceContextServiceImpl implements DatasourceContextService {
if (datasourceId == null) {
log.debug("This is a dry run or an embedded datasource. The datasource context would not exist in this scenario");
} else if (datasourceContextMap.get(datasourceId) != null && !isStale) {
} else if (datasourceContextMap.get(datasourceId) != null
// The following condition happens when there's a timout in the middle of destroying a connection and
// the reactive flow interrupts, resulting in the destroy operation not completing.
&& datasourceContextMap.get(datasourceId).getConnection() != null
&& !isStale) {
log.debug("resource context exists. Returning the same.");
return Mono.just(datasourceContextMap.get(datasourceId));
}