feat: Adding APPSMITH_PLUGIN_MAX_RESPONSE_SIZE which can be configured as an environment variable to increase plugin response size (currently only supported for Postgres Plugin) (#7126)

* Adding `APPSMITH_PLUGIN_MAX_RESPONSE_SIZE` which can be configured as an environment variable to increase plugin response size (currently only supported for Postgres Plugin) * Updated error message + added the environment variable to the template docker env files for new installations
2021-09-05 21:08:36 +05:30 · 2021-09-05 21:08:36 +05:30 · 7526129a3a
commit 7526129a3a
parent a4434872b0
10 changed files with 67 additions and 9 deletions
--- a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/exceptions/pluginExceptions/AppsmithPluginError.java
+++ b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/exceptions/pluginExceptions/AppsmithPluginError.java
@ -16,8 +16,8 @@ public enum AppsmithPluginError {
    PLUGIN_QUERY_TIMEOUT_ERROR(504, 5002, "{0} timed out in {1} milliseconds. " +
            "Please increase timeout. This can be found in Settings tab of {0}.", AppsmithErrorAction.DEFAULT, "Timed" +
            " out on query execution", ErrorType.CONNECTIVITY_ERROR),
-    PLUGIN_MAX_RESULT_SIZE_EXCEEDED(504, 5009, "Result size exceeded the supported"
-            + " size in Appsmith. Please limit the number of data points returned.",
+    PLUGIN_MAX_RESULT_SIZE_EXCEEDED(504, 5009, "Response size exceeded the maximum supported"
+            + " size of {0} MB. Please use LIMIT to reduce the amount of data fetched.",
            AppsmithErrorAction.DEFAULT, "Large Result Set Not Supported", ErrorType.INTERNAL_ERROR),
    PLUGIN_GET_STRUCTURE_TIMEOUT_ERROR(504, 5003, "{0}", AppsmithErrorAction.LOG_EXTERNALLY, "Timed out when fetching" +
            " datasource structure", ErrorType.CONNECTIVITY_ERROR),
--- a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/services/SharedConfig.java
+++ b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/services/SharedConfig.java
@ -3,4 +3,6 @@ package com.appsmith.external.services;
 public interface SharedConfig {

    int getCodecSize();
+
+    int getMaxResponseSize();
 }
--- a/app/server/appsmith-plugins/postgresPlugin/src/main/java/com/external/plugins/PostgresPlugin.java
+++ b/app/server/appsmith-plugins/postgresPlugin/src/main/java/com/external/plugins/PostgresPlugin.java
@ -22,6 +22,7 @@ import com.appsmith.external.models.SSLDetails;
 import com.appsmith.external.plugins.BasePlugin;
 import com.appsmith.external.plugins.PluginExecutor;
 import com.appsmith.external.plugins.SmartSubstitutionInterface;
+import com.appsmith.external.services.SharedConfig;
 import com.zaxxer.hikari.HikariConfig;
 import com.zaxxer.hikari.HikariDataSource;
 import com.zaxxer.hikari.HikariPoolMXBean;
@ -68,11 +69,11 @@ import java.util.stream.IntStream;
 import java.util.stream.Stream;

 import static com.appsmith.external.constants.ActionConstants.ACTION_CONFIGURATION_BODY;
-import static com.appsmith.external.helpers.SmartSubstitutionHelper.replaceQuestionMarkWithDollarIndex;
 import static com.appsmith.external.helpers.PluginUtils.getColumnsListForJdbcPlugin;
 import static com.appsmith.external.helpers.PluginUtils.getIdenticalColumns;
 import static com.appsmith.external.helpers.PluginUtils.getPSParamLabel;
 import static com.appsmith.external.helpers.Sizeof.sizeof;
+import static com.appsmith.external.helpers.SmartSubstitutionHelper.replaceQuestionMarkWithDollarIndex;
 import static com.external.plugins.utils.PostgresDataTypeUtils.DataType.BOOL;
 import static com.external.plugins.utils.PostgresDataTypeUtils.DataType.DATE;
 import static com.external.plugins.utils.PostgresDataTypeUtils.DataType.DECIMAL;
@ -113,7 +114,7 @@ public class PostgresPlugin extends BasePlugin {

    private static final int HEAVY_OP_FREQUENCY = 100;

-    private static final int MAX_SIZE_SUPPORTED = 1000000;
+    private static int MAX_SIZE_SUPPORTED;

    public PostgresPlugin(PluginWrapper wrapper) {
        super(wrapper);
@ -168,6 +169,14 @@ public class PostgresPlugin extends BasePlugin {

        private static final int PREPARED_STATEMENT_INDEX = 0;

+        private final SharedConfig sharedConfig;
+
+        public PostgresPluginExecutor(SharedConfig sharedConfig) {
+            this.sharedConfig = sharedConfig;
+            MAX_SIZE_SUPPORTED = sharedConfig.getMaxResponseSize();
+        }
+
+
        /**
         * Instead of using the default executeParametrized provided by pluginExecutor, this implementation affords an opportunity
         * to use PreparedStatement (if configured) which requires the variable substitution, etc. to happen in a particular format
@ -326,7 +335,6 @@ public class PostgresPlugin extends BasePlugin {

                        int iterator = 0;
                        while (resultSet.next()) {
-                            iterator++;

                            // Only check the data size at low frequency to ensure the performance is not impacted heavily
                            if (iterator% HEAVY_OP_FREQUENCY == 0) {
@ -334,8 +342,9 @@ public class PostgresPlugin extends BasePlugin {

                                if (objectSize > MAX_SIZE_SUPPORTED) {
                                    System.out.println(Thread.currentThread().getName() +
-                                            "[PostgresPlugin] Result size exceeded. Current size : " + objectSize);
-                                    return Mono.error(new AppsmithPluginException(AppsmithPluginError.PLUGIN_MAX_RESULT_SIZE_EXCEEDED));
+                                            "[PostgresPlugin] Result size greater than maximum supported size of "
+                                            + MAX_SIZE_SUPPORTED + "bytes. Current size : " + objectSize);
+                                    return Mono.error(new AppsmithPluginException(AppsmithPluginError.PLUGIN_MAX_RESULT_SIZE_EXCEEDED, (float) (MAX_SIZE_SUPPORTED / (1024 * 1024))));
                                }
                            }

@ -397,6 +406,8 @@ public class PostgresPlugin extends BasePlugin {
                            }

                            rowsList.add(row);
+
+                            iterator++;
                        }
                    }

--- a/app/server/appsmith-plugins/postgresPlugin/src/test/java/com/external/plugins/PostgresPluginTest.java
+++ b/app/server/appsmith-plugins/postgresPlugin/src/test/java/com/external/plugins/PostgresPluginTest.java
@ -14,6 +14,7 @@ import com.appsmith.external.models.Property;
 import com.appsmith.external.models.PsParameterDTO;
 import com.appsmith.external.models.RequestParamDTO;
 import com.appsmith.external.models.SSLDetails;
+import com.appsmith.external.services.SharedConfig;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.ArrayNode;
@ -53,7 +54,21 @@ import static org.junit.Assert.assertTrue;
 */
 public class PostgresPluginTest {

-    PostgresPlugin.PostgresPluginExecutor pluginExecutor = new PostgresPlugin.PostgresPluginExecutor();
+    public class MockSharedConfig implements SharedConfig {
+
+        @Override
+        public int getCodecSize() {
+            return 10 * 1024 * 1024;
+        }
+
+        @Override
+        public int getMaxResponseSize() {
+            return 10000;
+        }
+    }
+
+
+    PostgresPlugin.PostgresPluginExecutor pluginExecutor = new PostgresPlugin.PostgresPluginExecutor(new MockSharedConfig());

    @SuppressWarnings("rawtypes") // The type parameter for the container type is just itself and is pseudo-optional.
    @ClassRule
--- a/app/server/appsmith-plugins/restApiPlugin/src/test/java/com/external/plugins/RestApiPluginTest.java
+++ b/app/server/appsmith-plugins/restApiPlugin/src/test/java/com/external/plugins/RestApiPluginTest.java
@ -10,6 +10,7 @@ import com.appsmith.external.models.DatasourceConfiguration;
 import com.appsmith.external.models.OAuth2;
 import com.appsmith.external.models.Param;
 import com.appsmith.external.models.Property;
+import com.appsmith.external.services.SharedConfig;
 import com.external.connections.APIConnection;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.JsonNode;
@ -44,7 +45,21 @@ import static org.junit.Assert.assertThrows;
 import static org.junit.Assert.assertTrue;

 public class RestApiPluginTest {
-    RestApiPlugin.RestApiPluginExecutor pluginExecutor = new RestApiPlugin.RestApiPluginExecutor(() -> 10 * 1024 * 1024);
+
+    public class MockSharedConfig implements SharedConfig {
+
+        @Override
+        public int getCodecSize() {
+            return 10 * 1024 * 1024;
+        }
+
+        @Override
+        public int getMaxResponseSize() {
+            return 10000;
+        }
+    }
+
+    RestApiPlugin.RestApiPluginExecutor pluginExecutor = new RestApiPlugin.RestApiPluginExecutor(new MockSharedConfig());

    @Before
    public void setUp() {
--- a/app/server/appsmith-server/src/main/java/com/appsmith/server/configurations/SharedConfigImpl.java
+++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/configurations/SharedConfigImpl.java
@ -12,8 +12,16 @@ public class SharedConfigImpl implements SharedConfig {
    @Value("${appsmith.codec.max-in-memory-size:10}")
    private int CODEC_SIZE;

+    @Value("${appsmith.plugin.response.size.max:5}")
+    private float maxPluginResponseSize = 5;
+
    @Override
    public int getCodecSize() {
        return this.CODEC_SIZE * 1024 * 1024;
    }
+
+    @Override
+    public int getMaxResponseSize() {
+        return (int) (this.maxPluginResponseSize * 1024 * 1024);
+    }
 }
--- a/app/server/appsmith-server/src/main/resources/application.properties
+++ b/app/server/appsmith-server/src/main/resources/application.properties
@ -97,3 +97,6 @@ signup.allowed-domains=${APPSMITH_SIGNUP_ALLOWED_DOMAINS:}
 # Google recaptcha config
 google.recaptcha.key.site = ${APPSMITH_RECAPTCHA_SITE_KEY:}
 google.recaptcha.key.secret= ${APPSMITH_RECAPTCHA_SECRET_KEY:}
+
+#Plugin Interface level settings
+appsmith.plugin.response.size.max=${APPSMITH_PLUGIN_MAX_RESPONSE_SIZE_MB:5}
--- a/deploy/fat_container/templates/docker.env.sh
+++ b/deploy/fat_container/templates/docker.env.sh
@ -82,4 +82,5 @@ APPSMITH_ENCRYPTION_PASSWORD=$ENCRYPTION_PASSWORD
 APPSMITH_ENCRYPTION_SALT=$ENCRYPTION_SALT

 APPSMITH_CUSTOM_DOMAIN=
+# APPSMITH_PLUGIN_MAX_RESPONSE_SIZE_MB=5
 EOF
--- a/deploy/k8s/scripts/appsmith-configmap.yaml.sh
+++ b/deploy/k8s/scripts/appsmith-configmap.yaml.sh
@ -34,4 +34,5 @@ data:
  APPSMITH_RECAPTCHA_SECRET_KEY: ""
  APPSMITH_RECAPTCHA_ENABLED: "false"
  APPSMITH_DISABLE_INTERCOM: "false"
+  # APPSMITH_PLUGIN_MAX_RESPONSE_SIZE_MB=5
 EOF
--- a/deploy/template/docker.env.sh
+++ b/deploy/template/docker.env.sh
@ -73,4 +73,6 @@ APPSMITH_DISABLE_TELEMETRY=$disable_telemetry
 # APPSMITH_DISABLE_INTERCOM=
 # *******************************

+# APPSMITH_PLUGIN_MAX_RESPONSE_SIZE_MB=5
+
 EOF