diff --git a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/exceptions/pluginExceptions/AppsmithPluginError.java b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/exceptions/pluginExceptions/AppsmithPluginError.java index 74f827b977..b6debc1994 100644 --- a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/exceptions/pluginExceptions/AppsmithPluginError.java +++ b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/exceptions/pluginExceptions/AppsmithPluginError.java @@ -16,8 +16,8 @@ public enum AppsmithPluginError { PLUGIN_QUERY_TIMEOUT_ERROR(504, 5002, "{0} timed out in {1} milliseconds. " + "Please increase timeout. This can be found in Settings tab of {0}.", AppsmithErrorAction.DEFAULT, "Timed" + " out on query execution", ErrorType.CONNECTIVITY_ERROR), - PLUGIN_MAX_RESULT_SIZE_EXCEEDED(504, 5009, "Result size exceeded the supported" - + " size in Appsmith. Please limit the number of data points returned.", + PLUGIN_MAX_RESULT_SIZE_EXCEEDED(504, 5009, "Response size exceeded the maximum supported" + + " size of {0} MB. Please use LIMIT to reduce the amount of data fetched.", AppsmithErrorAction.DEFAULT, "Large Result Set Not Supported", ErrorType.INTERNAL_ERROR), PLUGIN_GET_STRUCTURE_TIMEOUT_ERROR(504, 5003, "{0}", AppsmithErrorAction.LOG_EXTERNALLY, "Timed out when fetching" + " datasource structure", ErrorType.CONNECTIVITY_ERROR), diff --git a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/services/SharedConfig.java b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/services/SharedConfig.java index e00d977c14..8f1eaa630a 100644 --- a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/services/SharedConfig.java +++ b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/services/SharedConfig.java @@ -3,4 +3,6 @@ package com.appsmith.external.services; public interface SharedConfig { int getCodecSize(); + + int getMaxResponseSize(); } diff --git a/app/server/appsmith-plugins/postgresPlugin/src/main/java/com/external/plugins/PostgresPlugin.java b/app/server/appsmith-plugins/postgresPlugin/src/main/java/com/external/plugins/PostgresPlugin.java index ff122b42d1..413fff8a9e 100644 --- a/app/server/appsmith-plugins/postgresPlugin/src/main/java/com/external/plugins/PostgresPlugin.java +++ b/app/server/appsmith-plugins/postgresPlugin/src/main/java/com/external/plugins/PostgresPlugin.java @@ -22,6 +22,7 @@ import com.appsmith.external.models.SSLDetails; import com.appsmith.external.plugins.BasePlugin; import com.appsmith.external.plugins.PluginExecutor; import com.appsmith.external.plugins.SmartSubstitutionInterface; +import com.appsmith.external.services.SharedConfig; import com.zaxxer.hikari.HikariConfig; import com.zaxxer.hikari.HikariDataSource; import com.zaxxer.hikari.HikariPoolMXBean; @@ -68,11 +69,11 @@ import java.util.stream.IntStream; import java.util.stream.Stream; import static com.appsmith.external.constants.ActionConstants.ACTION_CONFIGURATION_BODY; -import static com.appsmith.external.helpers.SmartSubstitutionHelper.replaceQuestionMarkWithDollarIndex; import static com.appsmith.external.helpers.PluginUtils.getColumnsListForJdbcPlugin; import static com.appsmith.external.helpers.PluginUtils.getIdenticalColumns; import static com.appsmith.external.helpers.PluginUtils.getPSParamLabel; import static com.appsmith.external.helpers.Sizeof.sizeof; +import static com.appsmith.external.helpers.SmartSubstitutionHelper.replaceQuestionMarkWithDollarIndex; import static com.external.plugins.utils.PostgresDataTypeUtils.DataType.BOOL; import static com.external.plugins.utils.PostgresDataTypeUtils.DataType.DATE; import static com.external.plugins.utils.PostgresDataTypeUtils.DataType.DECIMAL; @@ -113,7 +114,7 @@ public class PostgresPlugin extends BasePlugin { private static final int HEAVY_OP_FREQUENCY = 100; - private static final int MAX_SIZE_SUPPORTED = 1000000; + private static int MAX_SIZE_SUPPORTED; public PostgresPlugin(PluginWrapper wrapper) { super(wrapper); @@ -168,6 +169,14 @@ public class PostgresPlugin extends BasePlugin { private static final int PREPARED_STATEMENT_INDEX = 0; + private final SharedConfig sharedConfig; + + public PostgresPluginExecutor(SharedConfig sharedConfig) { + this.sharedConfig = sharedConfig; + MAX_SIZE_SUPPORTED = sharedConfig.getMaxResponseSize(); + } + + /** * Instead of using the default executeParametrized provided by pluginExecutor, this implementation affords an opportunity * to use PreparedStatement (if configured) which requires the variable substitution, etc. to happen in a particular format @@ -326,7 +335,6 @@ public class PostgresPlugin extends BasePlugin { int iterator = 0; while (resultSet.next()) { - iterator++; // Only check the data size at low frequency to ensure the performance is not impacted heavily if (iterator% HEAVY_OP_FREQUENCY == 0) { @@ -334,8 +342,9 @@ public class PostgresPlugin extends BasePlugin { if (objectSize > MAX_SIZE_SUPPORTED) { System.out.println(Thread.currentThread().getName() + - "[PostgresPlugin] Result size exceeded. Current size : " + objectSize); - return Mono.error(new AppsmithPluginException(AppsmithPluginError.PLUGIN_MAX_RESULT_SIZE_EXCEEDED)); + "[PostgresPlugin] Result size greater than maximum supported size of " + + MAX_SIZE_SUPPORTED + "bytes. Current size : " + objectSize); + return Mono.error(new AppsmithPluginException(AppsmithPluginError.PLUGIN_MAX_RESULT_SIZE_EXCEEDED, (float) (MAX_SIZE_SUPPORTED / (1024 * 1024)))); } } @@ -397,6 +406,8 @@ public class PostgresPlugin extends BasePlugin { } rowsList.add(row); + + iterator++; } } diff --git a/app/server/appsmith-plugins/postgresPlugin/src/test/java/com/external/plugins/PostgresPluginTest.java b/app/server/appsmith-plugins/postgresPlugin/src/test/java/com/external/plugins/PostgresPluginTest.java index d0eefc9fd9..5c3d8418dd 100644 --- a/app/server/appsmith-plugins/postgresPlugin/src/test/java/com/external/plugins/PostgresPluginTest.java +++ b/app/server/appsmith-plugins/postgresPlugin/src/test/java/com/external/plugins/PostgresPluginTest.java @@ -14,6 +14,7 @@ import com.appsmith.external.models.Property; import com.appsmith.external.models.PsParameterDTO; import com.appsmith.external.models.RequestParamDTO; import com.appsmith.external.models.SSLDetails; +import com.appsmith.external.services.SharedConfig; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; @@ -53,7 +54,21 @@ import static org.junit.Assert.assertTrue; */ public class PostgresPluginTest { - PostgresPlugin.PostgresPluginExecutor pluginExecutor = new PostgresPlugin.PostgresPluginExecutor(); + public class MockSharedConfig implements SharedConfig { + + @Override + public int getCodecSize() { + return 10 * 1024 * 1024; + } + + @Override + public int getMaxResponseSize() { + return 10000; + } + } + + + PostgresPlugin.PostgresPluginExecutor pluginExecutor = new PostgresPlugin.PostgresPluginExecutor(new MockSharedConfig()); @SuppressWarnings("rawtypes") // The type parameter for the container type is just itself and is pseudo-optional. @ClassRule diff --git a/app/server/appsmith-plugins/restApiPlugin/src/test/java/com/external/plugins/RestApiPluginTest.java b/app/server/appsmith-plugins/restApiPlugin/src/test/java/com/external/plugins/RestApiPluginTest.java index 0cb2169f8c..afcdda0b69 100644 --- a/app/server/appsmith-plugins/restApiPlugin/src/test/java/com/external/plugins/RestApiPluginTest.java +++ b/app/server/appsmith-plugins/restApiPlugin/src/test/java/com/external/plugins/RestApiPluginTest.java @@ -10,6 +10,7 @@ import com.appsmith.external.models.DatasourceConfiguration; import com.appsmith.external.models.OAuth2; import com.appsmith.external.models.Param; import com.appsmith.external.models.Property; +import com.appsmith.external.services.SharedConfig; import com.external.connections.APIConnection; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; @@ -44,7 +45,21 @@ import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; public class RestApiPluginTest { - RestApiPlugin.RestApiPluginExecutor pluginExecutor = new RestApiPlugin.RestApiPluginExecutor(() -> 10 * 1024 * 1024); + + public class MockSharedConfig implements SharedConfig { + + @Override + public int getCodecSize() { + return 10 * 1024 * 1024; + } + + @Override + public int getMaxResponseSize() { + return 10000; + } + } + + RestApiPlugin.RestApiPluginExecutor pluginExecutor = new RestApiPlugin.RestApiPluginExecutor(new MockSharedConfig()); @Before public void setUp() { diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/configurations/SharedConfigImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/configurations/SharedConfigImpl.java index 5193badf04..346e9cb33c 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/configurations/SharedConfigImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/configurations/SharedConfigImpl.java @@ -12,8 +12,16 @@ public class SharedConfigImpl implements SharedConfig { @Value("${appsmith.codec.max-in-memory-size:10}") private int CODEC_SIZE; + @Value("${appsmith.plugin.response.size.max:5}") + private float maxPluginResponseSize = 5; + @Override public int getCodecSize() { return this.CODEC_SIZE * 1024 * 1024; } + + @Override + public int getMaxResponseSize() { + return (int) (this.maxPluginResponseSize * 1024 * 1024); + } } diff --git a/app/server/appsmith-server/src/main/resources/application.properties b/app/server/appsmith-server/src/main/resources/application.properties index b1eaa0ede0..773ed7e622 100644 --- a/app/server/appsmith-server/src/main/resources/application.properties +++ b/app/server/appsmith-server/src/main/resources/application.properties @@ -97,3 +97,6 @@ signup.allowed-domains=${APPSMITH_SIGNUP_ALLOWED_DOMAINS:} # Google recaptcha config google.recaptcha.key.site = ${APPSMITH_RECAPTCHA_SITE_KEY:} google.recaptcha.key.secret= ${APPSMITH_RECAPTCHA_SECRET_KEY:} + +#Plugin Interface level settings +appsmith.plugin.response.size.max=${APPSMITH_PLUGIN_MAX_RESPONSE_SIZE_MB:5} diff --git a/deploy/fat_container/templates/docker.env.sh b/deploy/fat_container/templates/docker.env.sh index 8d1c9ab343..9cce21ec44 100644 --- a/deploy/fat_container/templates/docker.env.sh +++ b/deploy/fat_container/templates/docker.env.sh @@ -82,4 +82,5 @@ APPSMITH_ENCRYPTION_PASSWORD=$ENCRYPTION_PASSWORD APPSMITH_ENCRYPTION_SALT=$ENCRYPTION_SALT APPSMITH_CUSTOM_DOMAIN= +# APPSMITH_PLUGIN_MAX_RESPONSE_SIZE_MB=5 EOF diff --git a/deploy/k8s/scripts/appsmith-configmap.yaml.sh b/deploy/k8s/scripts/appsmith-configmap.yaml.sh index 7c11a6cb43..6a588ec3ef 100644 --- a/deploy/k8s/scripts/appsmith-configmap.yaml.sh +++ b/deploy/k8s/scripts/appsmith-configmap.yaml.sh @@ -34,4 +34,5 @@ data: APPSMITH_RECAPTCHA_SECRET_KEY: "" APPSMITH_RECAPTCHA_ENABLED: "false" APPSMITH_DISABLE_INTERCOM: "false" + # APPSMITH_PLUGIN_MAX_RESPONSE_SIZE_MB=5 EOF diff --git a/deploy/template/docker.env.sh b/deploy/template/docker.env.sh index 88533c8909..8f2cccff00 100644 --- a/deploy/template/docker.env.sh +++ b/deploy/template/docker.env.sh @@ -73,4 +73,6 @@ APPSMITH_DISABLE_TELEMETRY=$disable_telemetry # APPSMITH_DISABLE_INTERCOM= # ******************************* +# APPSMITH_PLUGIN_MAX_RESPONSE_SIZE_MB=5 + EOF