Add server-side templates support for plugins

app/server/appsmith-plugins/mongoPlugin/src/main/resources/templates/CREATE.json

@@ -0,0 +1,12 @@
+{
+  "insert": "users",
+  "documents": [
+    {
+      "name": "John Smith",
+      "email": [
+        "john@appsmith.com"
+      ],
+      "gender": "M"
+    }
+  ]
+}

app/server/appsmith-plugins/mongoPlugin/src/main/resources/templates/DELETE.json

@@ -0,0 +1,10 @@
+{
+  "delete": "users",
+  "deletes": [
+    {
+      "q": {
+        "id": 10
+      }
+    }
+  ]
+}

app/server/appsmith-plugins/mongoPlugin/src/main/resources/templates/READ.json

@@ -0,0 +1,12 @@
+{
+  "find": "users",
+  "filter": {
+    "id": {
+      "$gte": 10
+    }
+  },
+  "sort": {
+    "id": 1
+  },
+  "limit": 10
+}

app/server/appsmith-plugins/mongoPlugin/src/main/resources/templates/UPDATE.json

@@ -0,0 +1,16 @@
+{
+  "update": "users",
+  "updates": [
+    {
+      "q": {
+        "id": 10
+      },
+      "u": {
+        "name": "Updated Sam",
+        "email": [
+          "updates@appsmith.com"
+        ]
+      }
+    }
+  ]
+}

app/server/appsmith-plugins/postgresPlugin/src/main/resources/form.json

@@ -119,8 +119,7 @@
             },
             {
               "label": "Certificate",
-              "configProperty":
+              "configProperty": "datasourceConfiguration.connection.ssl.certificateFile",
-              "datasourceConfiguration.connection.ssl.certificateFile",
               "controlType": "FILE_PICKER"
             }
           ]
@@ -130,20 +129,17 @@
           "children": [
             {
               "label": "CA Certificate",
-              "configProperty":
+              "configProperty": "datasourceConfiguration.connection.ssl.caCertificateFile",
-              "datasourceConfiguration.connection.ssl.caCertificateFile",
               "controlType": "FILE_PICKER"
             },
             {
               "label": "PEM Certificate",
-              "configProperty":
+              "configProperty": "datasourceConfiguration.connection.ssl.pemCertificate.file",
-              "datasourceConfiguration.connection.ssl.pemCertificate.file",
               "controlType": "FILE_PICKER"
             },
             {
               "label": "PEM Passphrase",
-              "configProperty":
+              "configProperty": "datasourceConfiguration.connection.ssl.pemCertificate.password",
-              "datasourceConfiguration.connection.ssl.pemCertificate.password",
               "dataType": "PASSWORD",
               "controlType": "INPUT_TEXT",
               "placeholderText": "PEM Passphrase"

app/server/appsmith-plugins/postgresPlugin/src/main/resources/templates/CREATE.sql

@@ -0,0 +1,4 @@
+INSERT INTO users
+  (id, name, gender, avatar, email, address, role)
+VALUES
+  (?, ?, ?, ?, ?, ?, ?);

app/server/appsmith-plugins/postgresPlugin/src/main/resources/templates/DELETE.sql

@@ -0,0 +1 @@
+DELETE FROM users WHERE id = ?;

app/server/appsmith-plugins/postgresPlugin/src/main/resources/templates/SELECT.sql

@@ -0,0 +1 @@
+SELECT * FROM users ORDER BY id LIMIT 10;

app/server/appsmith-plugins/postgresPlugin/src/main/resources/templates/UPDATE.sql

@@ -0,0 +1,3 @@
+UPDATE users
+  SET status = 'APPROVED'
+  WHERE id = 1;

app/server/appsmith-server/src/main/java/com/appsmith/server/domains/Plugin.java

@@ -6,9 +6,11 @@ import lombok.NoArgsConstructor;
 import lombok.Setter;
 import lombok.ToString;
 import net.minidev.json.annotate.JsonIgnore;
+import org.springframework.data.annotation.Transient;
 import org.springframework.data.mongodb.core.mapping.Document;
 
 import java.util.List;
+import java.util.Map;
 
 @Getter
 @Setter
@@ -44,4 +46,7 @@ public class Plugin extends BaseDomain {
 
     Boolean allowUserDatasources = true;
 
+    @Transient
+    Map<String, String> templates;
+
 }

app/server/appsmith-server/src/main/java/com/appsmith/server/exceptions/AppsmithError.java

@@ -44,6 +44,7 @@ public enum AppsmithError {
     PLUGIN_RUN_FAILED(500, 5003, "Plugin execution failed with error {0}"),
     PLUGIN_EXECUTION_TIMEOUT(504, 5040, "Plugin Execution exceeded the maximum allowed time. Please increase the timeout in your action settings or check your backend action endpoint"),
     PLUGIN_LOAD_FORM_JSON_FAIL(500, 5004, "Unable to load datasource form configuration. Details: {0}."),
+    PLUGIN_LOAD_TEMPLATES_FAIL(500, 5005, "Unable to load datasource templates. Details: {0}."),
     MARKETPLACE_TIMEOUT(504, 5041, "Marketplace is responding too slowly. Please try again later"),
     DATASOURCE_HAS_ACTIONS(409, 4030, "Cannot delete datasource since it has {0} action(s) using it."),
     ;

app/server/appsmith-server/src/main/java/com/appsmith/server/services/PluginService.java

@@ -8,6 +8,7 @@ import reactor.core.publisher.Flux;
 import reactor.core.publisher.Mono;
 
 import java.io.InputStream;
+import java.util.Map;
 
 public interface PluginService extends CrudService<Plugin, String> {
 
@@ -25,7 +26,7 @@ public interface PluginService extends CrudService<Plugin, String> {
 
     Plugin redisInstallPlugin(InstallPluginRedisDTO installPluginRedisDTO);
 
-    Mono<Object> getFormConfig(String pluginId);
+    Mono<Map> getFormConfig(String pluginId);
 
-    Mono<InputStream> getPluginResource(String pluginId, String resourcePath);
+    Mono<InputStream> loadPluginResource(String pluginId, String resourcePath);
 }

app/server/appsmith-server/src/main/java/com/appsmith/server/services/PluginServiceImpl.java

@@ -17,7 +17,8 @@ import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.io.FileUtils;
 import org.pf4j.PluginManager;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.context.ApplicationContext;
+import org.springframework.core.io.Resource;
+import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
 import org.springframework.data.mongodb.core.ReactiveMongoTemplate;
 import org.springframework.data.mongodb.core.convert.MongoConverter;
 import org.springframework.data.mongodb.core.query.Criteria;
@@ -26,17 +27,23 @@ import org.springframework.data.redis.core.ReactiveRedisTemplate;
 import org.springframework.data.redis.listener.ChannelTopic;
 import org.springframework.stereotype.Service;
 import org.springframework.util.MultiValueMap;
+import org.springframework.util.StreamUtils;
+import reactor.core.Exceptions;
 import reactor.core.publisher.Flux;
 import reactor.core.publisher.Mono;
 import reactor.core.scheduler.Scheduler;
 
 import javax.validation.Validator;
 import java.io.File;
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
 import java.net.URL;
+import java.nio.charset.Charset;
 import java.nio.file.Path;
 import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
@@ -45,13 +52,14 @@ import java.util.stream.Collectors;
 @Service
 public class PluginServiceImpl extends BaseService<PluginRepository, Plugin, String> implements PluginService {
 
-    private final ApplicationContext applicationContext;
     private final OrganizationService organizationService;
     private final PluginManager pluginManager;
     private final ReactiveRedisTemplate<String, String> reactiveTemplate;
     private final ChannelTopic topic;
     private final ObjectMapper objectMapper;
-    private final SessionUserService sessionUserService;
+
+    private final Map<String, Mono<Map>> formCache = new HashMap<>();
+    private final Map<String, Mono<Map<String, String>>> templateCache = new HashMap<>();
 
     private static final int CONNECTION_TIMEOUT = 10000;
     private static final int READ_TIMEOUT = 10000;
@@ -63,21 +71,17 @@ public class PluginServiceImpl extends BaseService<PluginRepository, Plugin, Str
                              ReactiveMongoTemplate reactiveMongoTemplate,
                              PluginRepository repository,
                              AnalyticsService analyticsService,
-                             ApplicationContext applicationContext,
                              OrganizationService organizationService,
                              PluginManager pluginManager,
                              ReactiveRedisTemplate<String, String> reactiveTemplate,
                              ChannelTopic topic,
-                             ObjectMapper objectMapper,
+                             ObjectMapper objectMapper) {
-                             SessionUserService sessionUserService) {
         super(scheduler, validator, mongoConverter, reactiveMongoTemplate, repository, analyticsService);
-        this.applicationContext = applicationContext;
         this.organizationService = organizationService;
         this.pluginManager = pluginManager;
         this.reactiveTemplate = reactiveTemplate;
         this.topic = topic;
         this.objectMapper = objectMapper;
-        this.sessionUserService = sessionUserService;
     }
 
     @Override
@@ -96,12 +100,12 @@ public class PluginServiceImpl extends BaseService<PluginRepository, Plugin, Str
                     log.debug("Fetching plugins by params: {} for org: {}", params, org.getName());
                     if (org.getPlugins() == null) {
                         log.debug("Null installed plugins found for org: {}. Return empty plugins", org.getName());
-                        return Flux.fromIterable(new ArrayList<>());
+                        return Flux.empty();
                     }
 
                     List<String> pluginIds = org.getPlugins()
                             .stream()
-                            .map(obj -> obj.getPluginId())
+                            .map(OrganizationPlugin::getPluginId)
                             .collect(Collectors.toList());
                     Query query = new Query();
                     query.addCriteria(Criteria.where(FieldName.ID).in(pluginIds));
@@ -112,12 +116,17 @@ public class PluginServiceImpl extends BaseService<PluginRepository, Plugin, Str
                             query.addCriteria(Criteria.where(FieldName.TYPE).is(pluginType));
                         } catch (IllegalArgumentException e) {
                             log.error("No plugins for type : {}", params.getFirst(FieldName.TYPE));
-                            List<Plugin> emptyPlugins = new ArrayList<>();
+                            return Flux.empty();
-                            return Flux.fromIterable(emptyPlugins);
                         }
                     }
+
                     return mongoTemplate.find(query, Plugin.class);
-                });
+                })
+                .flatMap(plugin ->
+                        getTemplates(plugin)
+                            .doOnSuccess(plugin::setTemplates)
+                            .thenReturn(plugin)
+                );
     }
 
     @Override
@@ -285,19 +294,90 @@ public class PluginServiceImpl extends BaseService<PluginRepository, Plugin, Str
     }
 
     @Override
-    public Mono<Object> getFormConfig(String pluginId) {
+    public Mono<Map> getFormConfig(String pluginId) {
-        return getPluginResource(pluginId, "form.json")
+        if (!formCache.containsKey(pluginId)) {
-                .flatMap(jsonStream -> {
+            final Mono<Map> mono = loadPluginResource(pluginId, "form.json")
-                    try {
+                    .flatMap(jsonStream -> Mono.fromSupplier(() -> {
-                        return Mono.just(new ObjectMapper().readValue(jsonStream, Map.class));
+                        try {
-                    } catch (IOException e) {
+                            return new ObjectMapper().readValue(jsonStream, Map.class);
-                        return Mono.error(new AppsmithException(AppsmithError.PLUGIN_LOAD_FORM_JSON_FAIL, e.getMessage()));
+                        } catch (IOException e) {
-                    }
+                            log.error("Error loading form JSON for plugin " + pluginId, e);
-                });
+                            throw Exceptions.propagate(
+                                    new AppsmithException(AppsmithError.PLUGIN_LOAD_FORM_JSON_FAIL, e.getMessage())
+                            );
+                        }
+                    }))
+                    .doOnError(throwable ->
+                            // Remove this pluginId from the cache so it is tried again next time.
+                            formCache.remove(pluginId)
+                    )
+                    .onErrorMap(Exceptions::unwrap)
+                    .cache();
+
+            formCache.put(pluginId, mono);
+        }
+
+        return formCache.get(pluginId);
+    }
+
+    private Mono<Map<String, String>> getTemplates(Plugin plugin) {
+        final String pluginId = plugin.getId();
+
+        if (!templateCache.containsKey(pluginId)) {
+            final Mono<Map<String, String>> mono = Mono.fromSupplier(() -> loadTemplatesFromPlugin(plugin))
+                    .onErrorReturn(FileNotFoundException.class, Collections.emptyMap())
+                    .doOnError(throwable ->
+                            // Remove this pluginId from the cache so it is tried again next time.
+                            templateCache.remove(pluginId)
+                    )
+                    // It's okay if the templates folder is not present, we just return empty templates collection.
+                    .onErrorMap(throwable -> new AppsmithException(
+                            AppsmithError.PLUGIN_LOAD_TEMPLATES_FAIL, Exceptions.unwrap(throwable).getMessage())
+                    )
+                    .cache();
+
+            templateCache.put(pluginId, mono);
+        }
+
+        return templateCache.get(pluginId);
+    }
+
+    private Map<String, String> loadTemplatesFromPlugin(Plugin plugin) {
+        final PathMatchingResourcePatternResolver resolver = new PathMatchingResourcePatternResolver(
+                pluginManager
+                        .getPlugin(plugin.getPackageName())
+                        .getPluginClassLoader()
+        );
+
+        final Map<String, String> templates = new HashMap<>();
+
+        Resource[] resources;
+        try {
+            resources = resolver.getResources("templates/*");
+        } catch (IOException e) {
+            log.error("Error resolving templates in plugin for id: " + plugin.getId());
+            throw Exceptions.propagate(e);
+        }
+
+        for (final Resource resource : resources) {
+            final String filename = resource.getFilename();
+            try {
+                final String templateContent = StreamUtils.copyToString(
+                        resource.getInputStream(), Charset.defaultCharset());
+                if (filename != null) {
+                    templates.put(filename.replaceFirst("\\.\\w+$", ""), templateContent);
+                }
+            } catch (IOException e) {
+                log.error("Error loading template " + filename + " for plugin " + plugin.getId());
+                throw Exceptions.propagate(e);
+            }
+        }
+
+        return templates;
     }
 
     @Override
-    public Mono<InputStream> getPluginResource(String pluginId, String resourcePath) {
+    public Mono<InputStream> loadPluginResource(String pluginId, String resourcePath) {
         return findById(pluginId)
                 .flatMap(plugin -> {
                     InputStream formResourceStream = pluginManager