From eee2cfcaff944d14ccaa2ffe530424386b2f2a32 Mon Sep 17 00:00:00 2001 From: Shrikant Kandula Date: Wed, 24 Jun 2020 11:08:25 +0000 Subject: [PATCH] Add server-side templates support for plugins --- .../src/main/resources/templates/CREATE.json | 12 ++ .../src/main/resources/templates/DELETE.json | 10 ++ .../src/main/resources/templates/READ.json | 12 ++ .../src/main/resources/templates/UPDATE.json | 16 +++ .../src/main/resources/form.json | 12 +- .../src/main/resources/templates/CREATE.sql | 4 + .../src/main/resources/templates/DELETE.sql | 1 + .../src/main/resources/templates/SELECT.sql | 1 + .../src/main/resources/templates/UPDATE.sql | 3 + .../com/appsmith/server/domains/Plugin.java | 5 + .../server/exceptions/AppsmithError.java | 1 + .../server/services/PluginService.java | 5 +- .../server/services/PluginServiceImpl.java | 126 ++++++++++++++---- 13 files changed, 175 insertions(+), 33 deletions(-) create mode 100644 app/server/appsmith-plugins/mongoPlugin/src/main/resources/templates/CREATE.json create mode 100644 app/server/appsmith-plugins/mongoPlugin/src/main/resources/templates/DELETE.json create mode 100644 app/server/appsmith-plugins/mongoPlugin/src/main/resources/templates/READ.json create mode 100644 app/server/appsmith-plugins/mongoPlugin/src/main/resources/templates/UPDATE.json create mode 100644 app/server/appsmith-plugins/postgresPlugin/src/main/resources/templates/CREATE.sql create mode 100644 app/server/appsmith-plugins/postgresPlugin/src/main/resources/templates/DELETE.sql create mode 100644 app/server/appsmith-plugins/postgresPlugin/src/main/resources/templates/SELECT.sql create mode 100644 app/server/appsmith-plugins/postgresPlugin/src/main/resources/templates/UPDATE.sql diff --git a/app/server/appsmith-plugins/mongoPlugin/src/main/resources/templates/CREATE.json b/app/server/appsmith-plugins/mongoPlugin/src/main/resources/templates/CREATE.json new file mode 100644 index 0000000000..481c0c6323 --- /dev/null +++ b/app/server/appsmith-plugins/mongoPlugin/src/main/resources/templates/CREATE.json @@ -0,0 +1,12 @@ +{ + "insert": "users", + "documents": [ + { + "name": "John Smith", + "email": [ + "john@appsmith.com" + ], + "gender": "M" + } + ] +} diff --git a/app/server/appsmith-plugins/mongoPlugin/src/main/resources/templates/DELETE.json b/app/server/appsmith-plugins/mongoPlugin/src/main/resources/templates/DELETE.json new file mode 100644 index 0000000000..842160cf90 --- /dev/null +++ b/app/server/appsmith-plugins/mongoPlugin/src/main/resources/templates/DELETE.json @@ -0,0 +1,10 @@ +{ + "delete": "users", + "deletes": [ + { + "q": { + "id": 10 + } + } + ] +} diff --git a/app/server/appsmith-plugins/mongoPlugin/src/main/resources/templates/READ.json b/app/server/appsmith-plugins/mongoPlugin/src/main/resources/templates/READ.json new file mode 100644 index 0000000000..2ecc90b82c --- /dev/null +++ b/app/server/appsmith-plugins/mongoPlugin/src/main/resources/templates/READ.json @@ -0,0 +1,12 @@ +{ + "find": "users", + "filter": { + "id": { + "$gte": 10 + } + }, + "sort": { + "id": 1 + }, + "limit": 10 +} diff --git a/app/server/appsmith-plugins/mongoPlugin/src/main/resources/templates/UPDATE.json b/app/server/appsmith-plugins/mongoPlugin/src/main/resources/templates/UPDATE.json new file mode 100644 index 0000000000..231d713ff9 --- /dev/null +++ b/app/server/appsmith-plugins/mongoPlugin/src/main/resources/templates/UPDATE.json @@ -0,0 +1,16 @@ +{ + "update": "users", + "updates": [ + { + "q": { + "id": 10 + }, + "u": { + "name": "Updated Sam", + "email": [ + "updates@appsmith.com" + ] + } + } + ] +} diff --git a/app/server/appsmith-plugins/postgresPlugin/src/main/resources/form.json b/app/server/appsmith-plugins/postgresPlugin/src/main/resources/form.json index a1637b83dd..65fe25e6a4 100644 --- a/app/server/appsmith-plugins/postgresPlugin/src/main/resources/form.json +++ b/app/server/appsmith-plugins/postgresPlugin/src/main/resources/form.json @@ -119,8 +119,7 @@ }, { "label": "Certificate", - "configProperty": - "datasourceConfiguration.connection.ssl.certificateFile", + "configProperty": "datasourceConfiguration.connection.ssl.certificateFile", "controlType": "FILE_PICKER" } ] @@ -130,20 +129,17 @@ "children": [ { "label": "CA Certificate", - "configProperty": - "datasourceConfiguration.connection.ssl.caCertificateFile", + "configProperty": "datasourceConfiguration.connection.ssl.caCertificateFile", "controlType": "FILE_PICKER" }, { "label": "PEM Certificate", - "configProperty": - "datasourceConfiguration.connection.ssl.pemCertificate.file", + "configProperty": "datasourceConfiguration.connection.ssl.pemCertificate.file", "controlType": "FILE_PICKER" }, { "label": "PEM Passphrase", - "configProperty": - "datasourceConfiguration.connection.ssl.pemCertificate.password", + "configProperty": "datasourceConfiguration.connection.ssl.pemCertificate.password", "dataType": "PASSWORD", "controlType": "INPUT_TEXT", "placeholderText": "PEM Passphrase" diff --git a/app/server/appsmith-plugins/postgresPlugin/src/main/resources/templates/CREATE.sql b/app/server/appsmith-plugins/postgresPlugin/src/main/resources/templates/CREATE.sql new file mode 100644 index 0000000000..37d328b85c --- /dev/null +++ b/app/server/appsmith-plugins/postgresPlugin/src/main/resources/templates/CREATE.sql @@ -0,0 +1,4 @@ +INSERT INTO users + (id, name, gender, avatar, email, address, role) +VALUES + (?, ?, ?, ?, ?, ?, ?); diff --git a/app/server/appsmith-plugins/postgresPlugin/src/main/resources/templates/DELETE.sql b/app/server/appsmith-plugins/postgresPlugin/src/main/resources/templates/DELETE.sql new file mode 100644 index 0000000000..c487578842 --- /dev/null +++ b/app/server/appsmith-plugins/postgresPlugin/src/main/resources/templates/DELETE.sql @@ -0,0 +1 @@ +DELETE FROM users WHERE id = ?; diff --git a/app/server/appsmith-plugins/postgresPlugin/src/main/resources/templates/SELECT.sql b/app/server/appsmith-plugins/postgresPlugin/src/main/resources/templates/SELECT.sql new file mode 100644 index 0000000000..0a8208db0d --- /dev/null +++ b/app/server/appsmith-plugins/postgresPlugin/src/main/resources/templates/SELECT.sql @@ -0,0 +1 @@ +SELECT * FROM users ORDER BY id LIMIT 10; diff --git a/app/server/appsmith-plugins/postgresPlugin/src/main/resources/templates/UPDATE.sql b/app/server/appsmith-plugins/postgresPlugin/src/main/resources/templates/UPDATE.sql new file mode 100644 index 0000000000..18445fd6c3 --- /dev/null +++ b/app/server/appsmith-plugins/postgresPlugin/src/main/resources/templates/UPDATE.sql @@ -0,0 +1,3 @@ +UPDATE users + SET status = 'APPROVED' + WHERE id = 1; diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/domains/Plugin.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/domains/Plugin.java index 8c6a7d36e0..d797addacb 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/domains/Plugin.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/domains/Plugin.java @@ -6,9 +6,11 @@ import lombok.NoArgsConstructor; import lombok.Setter; import lombok.ToString; import net.minidev.json.annotate.JsonIgnore; +import org.springframework.data.annotation.Transient; import org.springframework.data.mongodb.core.mapping.Document; import java.util.List; +import java.util.Map; @Getter @Setter @@ -44,4 +46,7 @@ public class Plugin extends BaseDomain { Boolean allowUserDatasources = true; + @Transient + Map templates; + } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/exceptions/AppsmithError.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/exceptions/AppsmithError.java index df0282c3f0..13dc856e52 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/exceptions/AppsmithError.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/exceptions/AppsmithError.java @@ -44,6 +44,7 @@ public enum AppsmithError { PLUGIN_RUN_FAILED(500, 5003, "Plugin execution failed with error {0}"), PLUGIN_EXECUTION_TIMEOUT(504, 5040, "Plugin Execution exceeded the maximum allowed time. Please increase the timeout in your action settings or check your backend action endpoint"), PLUGIN_LOAD_FORM_JSON_FAIL(500, 5004, "Unable to load datasource form configuration. Details: {0}."), + PLUGIN_LOAD_TEMPLATES_FAIL(500, 5005, "Unable to load datasource templates. Details: {0}."), MARKETPLACE_TIMEOUT(504, 5041, "Marketplace is responding too slowly. Please try again later"), DATASOURCE_HAS_ACTIONS(409, 4030, "Cannot delete datasource since it has {0} action(s) using it."), ; diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/PluginService.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/PluginService.java index 630fca324f..486a954f40 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/PluginService.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/PluginService.java @@ -8,6 +8,7 @@ import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import java.io.InputStream; +import java.util.Map; public interface PluginService extends CrudService { @@ -25,7 +26,7 @@ public interface PluginService extends CrudService { Plugin redisInstallPlugin(InstallPluginRedisDTO installPluginRedisDTO); - Mono getFormConfig(String pluginId); + Mono getFormConfig(String pluginId); - Mono getPluginResource(String pluginId, String resourcePath); + Mono loadPluginResource(String pluginId, String resourcePath); } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/PluginServiceImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/PluginServiceImpl.java index 3a806c7805..4867138690 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/PluginServiceImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/PluginServiceImpl.java @@ -17,7 +17,8 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; import org.pf4j.PluginManager; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.ApplicationContext; +import org.springframework.core.io.Resource; +import org.springframework.core.io.support.PathMatchingResourcePatternResolver; import org.springframework.data.mongodb.core.ReactiveMongoTemplate; import org.springframework.data.mongodb.core.convert.MongoConverter; import org.springframework.data.mongodb.core.query.Criteria; @@ -26,17 +27,23 @@ import org.springframework.data.redis.core.ReactiveRedisTemplate; import org.springframework.data.redis.listener.ChannelTopic; import org.springframework.stereotype.Service; import org.springframework.util.MultiValueMap; +import org.springframework.util.StreamUtils; +import reactor.core.Exceptions; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import reactor.core.scheduler.Scheduler; import javax.validation.Validator; import java.io.File; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.net.URL; +import java.nio.charset.Charset; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -45,13 +52,14 @@ import java.util.stream.Collectors; @Service public class PluginServiceImpl extends BaseService implements PluginService { - private final ApplicationContext applicationContext; private final OrganizationService organizationService; private final PluginManager pluginManager; private final ReactiveRedisTemplate reactiveTemplate; private final ChannelTopic topic; private final ObjectMapper objectMapper; - private final SessionUserService sessionUserService; + + private final Map> formCache = new HashMap<>(); + private final Map>> templateCache = new HashMap<>(); private static final int CONNECTION_TIMEOUT = 10000; private static final int READ_TIMEOUT = 10000; @@ -63,21 +71,17 @@ public class PluginServiceImpl extends BaseService reactiveTemplate, ChannelTopic topic, - ObjectMapper objectMapper, - SessionUserService sessionUserService) { + ObjectMapper objectMapper) { super(scheduler, validator, mongoConverter, reactiveMongoTemplate, repository, analyticsService); - this.applicationContext = applicationContext; this.organizationService = organizationService; this.pluginManager = pluginManager; this.reactiveTemplate = reactiveTemplate; this.topic = topic; this.objectMapper = objectMapper; - this.sessionUserService = sessionUserService; } @Override @@ -96,12 +100,12 @@ public class PluginServiceImpl extends BaseService()); + return Flux.empty(); } List pluginIds = org.getPlugins() .stream() - .map(obj -> obj.getPluginId()) + .map(OrganizationPlugin::getPluginId) .collect(Collectors.toList()); Query query = new Query(); query.addCriteria(Criteria.where(FieldName.ID).in(pluginIds)); @@ -112,12 +116,17 @@ public class PluginServiceImpl extends BaseService emptyPlugins = new ArrayList<>(); - return Flux.fromIterable(emptyPlugins); + return Flux.empty(); } } + return mongoTemplate.find(query, Plugin.class); - }); + }) + .flatMap(plugin -> + getTemplates(plugin) + .doOnSuccess(plugin::setTemplates) + .thenReturn(plugin) + ); } @Override @@ -285,19 +294,90 @@ public class PluginServiceImpl extends BaseService getFormConfig(String pluginId) { - return getPluginResource(pluginId, "form.json") - .flatMap(jsonStream -> { - try { - return Mono.just(new ObjectMapper().readValue(jsonStream, Map.class)); - } catch (IOException e) { - return Mono.error(new AppsmithException(AppsmithError.PLUGIN_LOAD_FORM_JSON_FAIL, e.getMessage())); - } - }); + public Mono getFormConfig(String pluginId) { + if (!formCache.containsKey(pluginId)) { + final Mono mono = loadPluginResource(pluginId, "form.json") + .flatMap(jsonStream -> Mono.fromSupplier(() -> { + try { + return new ObjectMapper().readValue(jsonStream, Map.class); + } catch (IOException e) { + log.error("Error loading form JSON for plugin " + pluginId, e); + throw Exceptions.propagate( + new AppsmithException(AppsmithError.PLUGIN_LOAD_FORM_JSON_FAIL, e.getMessage()) + ); + } + })) + .doOnError(throwable -> + // Remove this pluginId from the cache so it is tried again next time. + formCache.remove(pluginId) + ) + .onErrorMap(Exceptions::unwrap) + .cache(); + + formCache.put(pluginId, mono); + } + + return formCache.get(pluginId); + } + + private Mono> getTemplates(Plugin plugin) { + final String pluginId = plugin.getId(); + + if (!templateCache.containsKey(pluginId)) { + final Mono> mono = Mono.fromSupplier(() -> loadTemplatesFromPlugin(plugin)) + .onErrorReturn(FileNotFoundException.class, Collections.emptyMap()) + .doOnError(throwable -> + // Remove this pluginId from the cache so it is tried again next time. + templateCache.remove(pluginId) + ) + // It's okay if the templates folder is not present, we just return empty templates collection. + .onErrorMap(throwable -> new AppsmithException( + AppsmithError.PLUGIN_LOAD_TEMPLATES_FAIL, Exceptions.unwrap(throwable).getMessage()) + ) + .cache(); + + templateCache.put(pluginId, mono); + } + + return templateCache.get(pluginId); + } + + private Map loadTemplatesFromPlugin(Plugin plugin) { + final PathMatchingResourcePatternResolver resolver = new PathMatchingResourcePatternResolver( + pluginManager + .getPlugin(plugin.getPackageName()) + .getPluginClassLoader() + ); + + final Map templates = new HashMap<>(); + + Resource[] resources; + try { + resources = resolver.getResources("templates/*"); + } catch (IOException e) { + log.error("Error resolving templates in plugin for id: " + plugin.getId()); + throw Exceptions.propagate(e); + } + + for (final Resource resource : resources) { + final String filename = resource.getFilename(); + try { + final String templateContent = StreamUtils.copyToString( + resource.getInputStream(), Charset.defaultCharset()); + if (filename != null) { + templates.put(filename.replaceFirst("\\.\\w+$", ""), templateContent); + } + } catch (IOException e) { + log.error("Error loading template " + filename + " for plugin " + plugin.getId()); + throw Exceptions.propagate(e); + } + } + + return templates; } @Override - public Mono getPluginResource(String pluginId, String resourcePath) { + public Mono loadPluginResource(String pluginId, String resourcePath) { return findById(pluginId) .flatMap(plugin -> { InputStream formResourceStream = pluginManager