From 3c8583210f7d1b5852bbd7820e3e2e5563830226 Mon Sep 17 00:00:00 2001 From: Sumit Kumar Date: Wed, 3 Nov 2021 16:23:53 +0530 Subject: [PATCH] feat: remove region requirement from s3 plugin (#8829) * Remove Region field from S3 datasource editor page for AWS S3, Upcloud, Digital Ocean Spaces, Dream Objects, Wasabi. * Use SDK provided property for AWS S3 to delegate region selection to the SDK. * Extract region info from endpoint URL for Upcloud, Digital Ocean Spaces, Dream Objects and Wasabi, since the SDK property does not work for these service providers. * Removed some redundant checks from datasourceCreate that were already part of validateDatasource * Fix show clause in list.json --- .../com/external/plugins/AmazonS3Plugin.java | 205 ++-------------- .../{ => constants}/AmazonS3Action.java | 4 +- .../com/external/utils/DatasourceUtils.java | 228 ++++++++++++++++++ .../src/main/resources/editor/list.json | 6 +- .../src/main/resources/form.json | 90 +------ .../external/plugins/AmazonS3PluginTest.java | 94 ++++++-- .../server/migrations/DatabaseChangelog.java | 2 +- 7 files changed, 323 insertions(+), 306 deletions(-) rename app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/plugins/{ => constants}/AmazonS3Action.java (56%) create mode 100644 app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/utils/DatasourceUtils.java diff --git a/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/plugins/AmazonS3Plugin.java b/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/plugins/AmazonS3Plugin.java index 4549eda84f..033138ab04 100644 --- a/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/plugins/AmazonS3Plugin.java +++ b/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/plugins/AmazonS3Plugin.java @@ -1,12 +1,7 @@ package com.external.plugins; import com.amazonaws.HttpMethod; -import com.amazonaws.auth.AWSStaticCredentialsProvider; -import com.amazonaws.auth.BasicAWSCredentials; -import com.amazonaws.client.builder.AwsClientBuilder; -import com.amazonaws.regions.Regions; import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.amazonaws.services.s3.model.AmazonS3Exception; import com.amazonaws.services.s3.model.Bucket; import com.amazonaws.services.s3.model.GeneratePresignedUrlRequest; @@ -32,6 +27,7 @@ import com.appsmith.external.models.Property; import com.appsmith.external.models.RequestParamDTO; import com.appsmith.external.plugins.BasePlugin; import com.appsmith.external.plugins.PluginExecutor; +import com.external.plugins.constants.AmazonS3Action; import lombok.extern.slf4j.Slf4j; import org.pf4j.Extension; import org.pf4j.PluginWrapper; @@ -61,6 +57,7 @@ import java.util.stream.Collectors; import static com.appsmith.external.constants.ActionConstants.ACTION_CONFIGURATION_BODY; import static com.appsmith.external.constants.ActionConstants.ACTION_CONFIGURATION_PATH; +import static com.external.utils.DatasourceUtils.getS3ClientBuilder; import static com.appsmith.external.helpers.PluginUtils.getValueSafelyFromFormData; import static com.appsmith.external.helpers.PluginUtils.getValueSafelyFromFormDataOrDefault; import static com.appsmith.external.helpers.PluginUtils.setValueSafelyInFormData; @@ -78,15 +75,15 @@ import static com.external.plugins.constants.FieldName.READ_USING_BASE64_ENCODIN public class AmazonS3Plugin extends BasePlugin { private static final String S3_DRIVER = "com.amazonaws.services.s3.AmazonS3"; - private static final int AWS_S3_REGION_PROPERTY_INDEX = 0; - private static final int S3_SERVICE_PROVIDER_PROPERTY_INDEX = 1; - private static final int CUSTOM_ENDPOINT_REGION_PROPERTY_INDEX = 2; - private static final int CUSTOM_ENDPOINT_INDEX = 0; + public static final int S3_SERVICE_PROVIDER_PROPERTY_INDEX = 1; + public static final int CUSTOM_ENDPOINT_REGION_PROPERTY_INDEX = 2; + public static final int CUSTOM_ENDPOINT_INDEX = 0; private static final String DEFAULT_URL_EXPIRY_IN_MINUTES = "5"; // max 7 days is possible private static final String YES = "YES"; private static final String NO = "NO"; private static final String BASE64_DELIMITER = ";base64,"; - private static final String AMAZON_S3_SERVICE_PROVIDER = "amazon-s3"; + private static final String OTHER_S3_SERVICE_PROVIDER = "other"; + private static final String AWS_S3_SERVICE_PROVIDER = "amazon-s3"; public AmazonS3Plugin(PluginWrapper wrapper) { super(wrapper); @@ -270,7 +267,6 @@ public class AmazonS3Plugin extends BasePlugin { Map requestProperties = new HashMap<>(); List requestParams = new ArrayList<>(); - return Mono.fromCallable(() -> { /* @@ -282,16 +278,6 @@ public class AmazonS3Plugin extends BasePlugin { return Mono.error(new StaleConnectionException()); } - if (datasourceConfiguration == null) { - return Mono.error( - new AppsmithPluginException( - AppsmithPluginError.PLUGIN_EXECUTE_ARGUMENT_ERROR, - "At least one of the mandatory fields in S3 datasource creation form is empty - " + - "'Access Key'/'Secret Key'/'Region'. Please fill all the mandatory fields and try again." - ) - ); - } - if (actionConfiguration == null) { return Mono.error( new AppsmithPluginException( @@ -580,7 +566,7 @@ public class AmazonS3Plugin extends BasePlugin { return Mono.just(result); }) - // Now set the request in the result to be returned back to the server + // Now set the request in the result to be returned to the server .map(actionExecutionResult -> { ActionExecutionRequest actionExecutionRequest = new ActionExecutionRequest(); actionExecutionRequest.setQuery(query[0]); @@ -595,16 +581,6 @@ public class AmazonS3Plugin extends BasePlugin { @Override public Mono datasourceCreate(DatasourceConfiguration datasourceConfiguration) { - if (datasourceConfiguration == null) { - return Mono.error( - new AppsmithPluginException( - AppsmithPluginError.PLUGIN_DATASOURCE_ARGUMENT_ERROR, - "Mandatory fields 'Access Key', 'Secret Key', 'Region' missing. Did you forget to edit " + - "the 'Access Key'/'Secret Key'/'Region' fields in the datasource creation form?" - ) - ); - } - try { Class.forName(S3_DRIVER); } catch (ClassNotFoundException e) { @@ -617,137 +593,8 @@ public class AmazonS3Plugin extends BasePlugin { ); } - return (Mono) Mono.fromCallable(() -> { - List properties = datasourceConfiguration.getProperties(); - - /* - * - Ideally, properties must never be null because the fields contained in the properties list have a - * default value defined. - * - Ideally, properties.get(S3_SERVICE_PROVIDER_PROPERTY_INDEX) must never be null/empty, because the - * `S3 Service Provider` dropdown has a default value. - */ - if (properties == null - || properties.get(S3_SERVICE_PROVIDER_PROPERTY_INDEX) == null - || StringUtils.isNullOrEmpty((String) properties.get(S3_SERVICE_PROVIDER_PROPERTY_INDEX).getValue())) { - return Mono.error( - new AppsmithPluginException( - AppsmithPluginError.PLUGIN_DATASOURCE_ARGUMENT_ERROR, - "Appsmith has failed to fetch the 'S3 Service Provider' field properties. Please " + - "reach out to Appsmith customer support to resolve this." - ) - ); - } - - final boolean usingCustomEndpoint = - !AMAZON_S3_SERVICE_PROVIDER.equals(properties.get(S3_SERVICE_PROVIDER_PROPERTY_INDEX).getValue()); - - if (!usingCustomEndpoint - && (properties.size() < (AWS_S3_REGION_PROPERTY_INDEX + 1) - || properties.get(AWS_S3_REGION_PROPERTY_INDEX) == null - || StringUtils.isNullOrEmpty((String) properties.get(AWS_S3_REGION_PROPERTY_INDEX).getValue()))) { - return Mono.error( - new AppsmithPluginException( - AppsmithPluginError.PLUGIN_DATASOURCE_ARGUMENT_ERROR, - "Required parameter 'Region' is empty. Did you forget to edit the 'Region' field" + - " in the datasource creation form ? You need to fill it with the region where " + - "your AWS S3 instance is hosted." - ) - ); - } - - if (usingCustomEndpoint - && (datasourceConfiguration.getEndpoints() == null - || CollectionUtils.isEmpty(datasourceConfiguration.getEndpoints()) - || datasourceConfiguration.getEndpoints().get(CUSTOM_ENDPOINT_INDEX) == null - || StringUtils.isNullOrEmpty(datasourceConfiguration.getEndpoints().get(CUSTOM_ENDPOINT_INDEX).getHost()))) { - return Mono.error( - new AppsmithPluginException( - AppsmithPluginError.PLUGIN_DATASOURCE_ARGUMENT_ERROR, - "Required parameter 'Endpoint URL' is empty. Did you forget to edit the 'Endpoint" + - " URL' field in the datasource creation form ? You need to fill it with " + - "the endpoint URL of your S3 instance." - ) - ); - } - - if (usingCustomEndpoint - && (properties.size() < (CUSTOM_ENDPOINT_REGION_PROPERTY_INDEX + 1) - || properties.get(CUSTOM_ENDPOINT_REGION_PROPERTY_INDEX) == null - || StringUtils.isNullOrEmpty((String) properties.get(CUSTOM_ENDPOINT_REGION_PROPERTY_INDEX).getValue()))) { - return Mono.error( - new AppsmithPluginException( - AppsmithPluginError.PLUGIN_DATASOURCE_ARGUMENT_ERROR, - "Required parameter 'Region' is empty. Did you forget to edit the 'Region' field" + - " in the datasource creation form ? You need to fill it with the region where " + - "your S3 instance is hosted." - ) - ); - } - - final String region = (String) (usingCustomEndpoint ? - properties.get(CUSTOM_ENDPOINT_REGION_PROPERTY_INDEX).getValue() : - properties.get(AWS_S3_REGION_PROPERTY_INDEX).getValue()); - - DBAuth authentication = (DBAuth) datasourceConfiguration.getAuthentication(); - if (authentication == null - || StringUtils.isNullOrEmpty(authentication.getUsername()) - || StringUtils.isNullOrEmpty(authentication.getPassword())) { - return Mono.error( - new AppsmithPluginException( - AppsmithPluginError.PLUGIN_DATASOURCE_ARGUMENT_ERROR, - "Mandatory parameters 'Access Key' and/or 'Secret Key' are missing. Did you " + - "forget to edit the 'Access Key'/'Secret Key' fields in the datasource creation form ?" - ) - ); - } - - String accessKey = authentication.getUsername(); - String secretKey = authentication.getPassword(); - - BasicAWSCredentials awsCreds; - try { - awsCreds = new BasicAWSCredentials(accessKey, secretKey); - } catch (IllegalArgumentException e) { - return Mono.error( - new AppsmithPluginException( - AppsmithPluginError.PLUGIN_DATASOURCE_ARGUMENT_ERROR, - "Appsmith server has encountered an error when " + - "parsing AWS credentials from datasource: " + e.getMessage() - ) - ); - } - - AmazonS3ClientBuilder s3ClientBuilder = AmazonS3ClientBuilder - .standard() - .withCredentials(new AWSStaticCredentialsProvider(awsCreds)); - - if (!usingCustomEndpoint) { - Regions clientRegion = null; - - try { - clientRegion = Regions.fromName(region); - } catch (IllegalArgumentException e) { - return Mono.error( - new AppsmithPluginException( - AppsmithPluginError.PLUGIN_DATASOURCE_ARGUMENT_ERROR, - "Appsmith server has encountered an error when " + - "parsing AWS S3 instance region from the AWS S3 datasource configuration " + - "provided: " + e.getMessage() - ) - ); - } - - s3ClientBuilder = s3ClientBuilder.withRegion(clientRegion); - } else { - String endpoint = datasourceConfiguration.getEndpoints().get(CUSTOM_ENDPOINT_INDEX).getHost(); - s3ClientBuilder = s3ClientBuilder - .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, region)); - } - - return Mono.just(s3ClientBuilder.build()); - - }) - .flatMap(obj -> obj) + return Mono.fromCallable(() -> getS3ClientBuilder(datasourceConfiguration).build()) + .flatMap(client -> Mono.just(client)) .onErrorResume(e -> { if (e instanceof AppsmithPluginException) { return Mono.error(e); @@ -757,7 +604,7 @@ public class AmazonS3Plugin extends BasePlugin { new AppsmithPluginException( AppsmithPluginError.PLUGIN_ERROR, "Appsmith server has encountered an error when " + - "connecting to AWS S3 server: " + e.getMessage() + "connecting to your S3 server: " + e.getMessage() ) ); } @@ -817,21 +664,11 @@ public class AmazonS3Plugin extends BasePlugin { invalids.add("Appsmith has failed to fetch the 'S3 Service Provider' field properties. Please " + "reach out to Appsmith customer support to resolve this."); } - final boolean usingCustomEndpoint = - !AMAZON_S3_SERVICE_PROVIDER.equals(properties.get(S3_SERVICE_PROVIDER_PROPERTY_INDEX).getValue()); - if (!usingCustomEndpoint - && (properties.size() < (AWS_S3_REGION_PROPERTY_INDEX + 1) - || properties.get(AWS_S3_REGION_PROPERTY_INDEX) == null - || StringUtils.isNullOrEmpty((String) properties.get(AWS_S3_REGION_PROPERTY_INDEX).getValue()))) { - invalids.add("Required parameter 'Region' is empty. Did you forget to edit the 'Region' field" + - " in the datasource creation form ? You need to fill it with the region where " + - "your AWS S3 instance is hosted."); - } - - if (usingCustomEndpoint - && (datasourceConfiguration.getEndpoints() == null - || CollectionUtils.isEmpty(datasourceConfiguration.getEndpoints()) + final boolean usingAWSS3ServiceProvider = + AWS_S3_SERVICE_PROVIDER.equals(properties.get(S3_SERVICE_PROVIDER_PROPERTY_INDEX).getValue()); + if (!usingAWSS3ServiceProvider + && (CollectionUtils.isEmpty(datasourceConfiguration.getEndpoints()) || datasourceConfiguration.getEndpoints().get(CUSTOM_ENDPOINT_INDEX) == null || StringUtils.isNullOrEmpty(datasourceConfiguration.getEndpoints().get(CUSTOM_ENDPOINT_INDEX).getHost()))) { invalids.add("Required parameter 'Endpoint URL' is empty. Did you forget to edit the 'Endpoint" + @@ -839,7 +676,9 @@ public class AmazonS3Plugin extends BasePlugin { "the endpoint URL of your S3 instance."); } - if (usingCustomEndpoint + final boolean usingCustomServiceProvider = + OTHER_S3_SERVICE_PROVIDER.equals(properties.get(S3_SERVICE_PROVIDER_PROPERTY_INDEX).getValue()); + if (usingCustomServiceProvider && (properties.size() < (CUSTOM_ENDPOINT_REGION_PROPERTY_INDEX + 1) || properties.get(CUSTOM_ENDPOINT_REGION_PROPERTY_INDEX) == null || StringUtils.isNullOrEmpty((String) properties.get(CUSTOM_ENDPOINT_REGION_PROPERTY_INDEX).getValue()))) { @@ -853,14 +692,6 @@ public class AmazonS3Plugin extends BasePlugin { @Override public Mono testDatasource(DatasourceConfiguration datasourceConfiguration) { - if (datasourceConfiguration == null) { - return Mono.just( - new DatasourceTestResult( - "At least one of the mandatory fields in S3 datasource creation form is empty - " + - "'Access Key'/'Secret Key'/'Region'. Please fill all the mandatory fields and try again." - ) - ); - } return datasourceCreate(datasourceConfiguration) .map(connection -> { diff --git a/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/plugins/AmazonS3Action.java b/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/plugins/constants/AmazonS3Action.java similarity index 56% rename from app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/plugins/AmazonS3Action.java rename to app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/plugins/constants/AmazonS3Action.java index ec53187c98..1691661312 100644 --- a/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/plugins/AmazonS3Action.java +++ b/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/plugins/constants/AmazonS3Action.java @@ -1,6 +1,6 @@ -package com.external.plugins; +package com.external.plugins.constants; -enum AmazonS3Action { +public enum AmazonS3Action { LIST, UPLOAD_FILE_FROM_BODY, READ_FILE, diff --git a/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/utils/DatasourceUtils.java b/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/utils/DatasourceUtils.java new file mode 100644 index 0000000000..b72a313cf6 --- /dev/null +++ b/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/utils/DatasourceUtils.java @@ -0,0 +1,228 @@ +package com.external.utils; + +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.client.builder.AwsClientBuilder; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginError; +import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginException; +import com.appsmith.external.models.DBAuth; +import com.appsmith.external.models.DatasourceConfiguration; +import com.appsmith.external.models.Property; +import org.apache.commons.lang.StringUtils; + +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static com.amazonaws.regions.Regions.DEFAULT_REGION; +import static com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginError.PLUGIN_ERROR; +import static com.external.plugins.AmazonS3Plugin.CUSTOM_ENDPOINT_INDEX; +import static com.external.plugins.AmazonS3Plugin.CUSTOM_ENDPOINT_REGION_PROPERTY_INDEX; +import static com.external.plugins.AmazonS3Plugin.S3_SERVICE_PROVIDER_PROPERTY_INDEX; +import static com.external.utils.DatasourceUtils.S3ServiceProvider.AMAZON; + +public class DatasourceUtils { + + /** + * Example endpoint : appsmith-test-storage-2.de-fra1.upcloudobjects.com + * Group 2 match: de-fra1 + */ + public static String UPCLOUD_URL_ENDPOINT_PATTERN = "^([^\\.]+)\\.([^\\.]+)\\.upcloudobjects\\.com$"; + public static int UPCLOUD_REGION_GROUP_INDEX = 2; + + /** + * Example endpoint : s3.ap-northeast-2.wasabisys.com + * Group 2 match: ap-northeast-2 + */ + public static String WASABI_URL_ENDPOINT_PATTERN = "^([^\\.]+)\\.([^\\.]+)\\.wasabisys\\.com$"; + public static int WASABI_REGION_GROUP_INDEX = 2; + + /** + * Example endpoint : fra1.digitaloceanspaces.com + * Group 1 match: fra1 + */ + public static String DIGITAL_OCEAN_URL_ENDPOINT_PATTERN = "^([^\\.]+)\\.digitaloceanspaces\\.com$"; + public static int DIGITAL_OCEAN_REGION_GROUP_INDEX = 1; + + /** + * Example endpoint : objects-us-east-1.dream.io + * Group 1 match: us-east-1 + */ + public static String DREAM_OBJECTS_URL_ENDPOINT_PATTERN = "^objects-([^\\.]+)\\.dream\\.io$"; + public static int DREAM_OBJECTS_REGION_GROUP_INDEX = 1; + + /* This enum lists various types of S3 service providers that we support. */ + public enum S3ServiceProvider { + AMAZON ("amazon-s3"), + UPCLOUD ("upcloud"), + WASABI ("wasabi"), + DIGITAL_OCEAN_SPACES ("digital-ocean-spaces"), + DREAM_OBJECTS ("dream-objects"), + OTHER ("other"); + + private String name; + + S3ServiceProvider(String name) { + this.name = name; + } + + public static S3ServiceProvider fromString(String name) throws AppsmithPluginException { + for (S3ServiceProvider s3ServiceProvider : S3ServiceProvider.values()) { + if (s3ServiceProvider.name.equals(name.toLowerCase())) { + return s3ServiceProvider; + } + } + + throw new AppsmithPluginException(PLUGIN_ERROR, "Appsmith S3 plugin service has " + + "failed to identify the S3 service provider type. Please reach out to Appsmith customer support" + + " to resolve this"); + } + } + + /** + * This method builds an `AmazonS3ClientBuilder` object from the datasourceConfiguration provided by user. The + * `AmazonS3ClientBuilder` object can then be used to get a connection object to connect to the S3 service. + * + * @param datasourceConfiguration + * @return AmazonS3ClientBuilder object + * @throws AppsmithPluginException when (1) there is an error with parsing credentials (2) required + * datasourceConfiguration properties are missing (3) endpoint URL is found incorrect. + */ + public static AmazonS3ClientBuilder getS3ClientBuilder (DatasourceConfiguration datasourceConfiguration) + throws AppsmithPluginException { + + DBAuth authentication = (DBAuth) datasourceConfiguration.getAuthentication(); + String accessKey = authentication.getUsername(); + String secretKey = authentication.getPassword(); + BasicAWSCredentials awsCreds; + try { + awsCreds = new BasicAWSCredentials(accessKey, secretKey); + } catch (IllegalArgumentException e) { + throw new AppsmithPluginException( + AppsmithPluginError.PLUGIN_DATASOURCE_ARGUMENT_ERROR, + "Appsmith server has encountered an error when parsing AWS credentials from datasource: " + + e.getMessage() + ); + } + + /* Set credentials in client builder. */ + AmazonS3ClientBuilder s3ClientBuilder = AmazonS3ClientBuilder + .standard() + .withCredentials(new AWSStaticCredentialsProvider(awsCreds)); + + List properties = datasourceConfiguration.getProperties(); + + /** + * Return error if no service provider is chosen. + * + * Ideally, properties.get(S3_SERVICE_PROVIDER_PROPERTY_INDEX) must always exist, because the `S3 + * Service Provider` dropdown has a default value. + */ + if (properties == null + || properties.get(S3_SERVICE_PROVIDER_PROPERTY_INDEX) == null + || StringUtils.isEmpty((String) properties.get(S3_SERVICE_PROVIDER_PROPERTY_INDEX).getValue())) { + throw new AppsmithPluginException( + AppsmithPluginError.PLUGIN_DATASOURCE_ARGUMENT_ERROR, + "Appsmith has failed to fetch the 'S3 Service Provider' field properties. Please reach out to" + + " Appsmith customer support to resolve this." + ); + } + + S3ServiceProvider s3ServiceProvider = + S3ServiceProvider.fromString((String) properties.get(S3_SERVICE_PROVIDER_PROPERTY_INDEX).getValue()); + + /** + * AmazonS3 provides an attribute `forceGlobalBucketAccessEnabled` that automatically routes the request to a + * region such that request should succeed. + * Ref: https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/S3ClientOptions + * .Builder.html#enableForceGlobalBucketAccess-- + * + * However, no mention of the attribute `forceGlobalBucketAccessEnabled` could be found within the + * documentation of other listed S3 service providers like Upcloud, Wasabi, Dream Objects, or Digital Ocean + * Spaces. Also, some these services failed on usage of this attribute - hence this attribute could not be + * reliably used for these S3 service providers. For these service providers, the region information is + * chained in the endpoint URL. Hence, the endpoint URL is used to extract the exact object storage region. + * + * Apart from the listed S3 services - AWS, Upcloud, Wasabi, Dream Objects and Digital Ocean Spaces, any other + * service provider falls in the category `other` and there is no special handling defined for it since we + * cannot assume any information about them beforehand. For this S3 service provider type region must be + * explicitly provided. + */ + if (s3ServiceProvider.equals(AMAZON)) { + s3ClientBuilder = s3ClientBuilder + .withRegion(DEFAULT_REGION) + .enableForceGlobalBucketAccess(); + } + else { + String endpoint = datasourceConfiguration.getEndpoints().get(CUSTOM_ENDPOINT_INDEX).getHost(); + String region = ""; + + switch(s3ServiceProvider) { + case AMAZON: + /* This case can never be reached because of the if condition above. Just adding for sake of + completeness. */ + + break; + case UPCLOUD: + region = getRegionFromEndpointPattern(endpoint, UPCLOUD_URL_ENDPOINT_PATTERN, + UPCLOUD_REGION_GROUP_INDEX); + + break; + case WASABI: + region = getRegionFromEndpointPattern(endpoint, WASABI_URL_ENDPOINT_PATTERN, + WASABI_REGION_GROUP_INDEX); + + break; + case DIGITAL_OCEAN_SPACES: + region = getRegionFromEndpointPattern(endpoint, DIGITAL_OCEAN_URL_ENDPOINT_PATTERN, + DIGITAL_OCEAN_REGION_GROUP_INDEX); + + break; + case DREAM_OBJECTS: + region = getRegionFromEndpointPattern(endpoint, DREAM_OBJECTS_URL_ENDPOINT_PATTERN, + DREAM_OBJECTS_REGION_GROUP_INDEX); + + break; + default: + region = (String) properties.get(CUSTOM_ENDPOINT_REGION_PROPERTY_INDEX).getValue(); + } + + s3ClientBuilder = s3ClientBuilder + .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, region)); + } + + return s3ClientBuilder; + } + + /** + * This method checks if the S3 endpoint URL has correct format and extracts region information from it. + * + * @param endpoint : endpoint URL + * @param regex : expected endpoint URL pattern + * @param regionGroupIndex : pattern group index for region string + * @return S3 object storage region. + * @throws AppsmithPluginException when then endpoint URL does not match the expected regex pattern. + */ + private static String getRegionFromEndpointPattern(String endpoint, String regex, int regionGroupIndex) + throws AppsmithPluginException { + + /* endpoint is expected to be non-null at this point */ + if (!endpoint.matches(regex)) { + throw new AppsmithPluginException(AppsmithPluginError.PLUGIN_DATASOURCE_ARGUMENT_ERROR, "Your S3 endpoint" + + " URL seems to be incorrect for the selected S3 service provider. Please check your endpoint URL " + + "and the selected S3 service provider."); + } + + Pattern pattern = Pattern.compile(regex); + Matcher matcher = pattern.matcher(endpoint); + if (matcher.find()) { + return matcher.group(regionGroupIndex); + } + + /* Code flow is never expected to reach here. */ + throw new AppsmithPluginException(AppsmithPluginError.PLUGIN_ERROR, "Your S3 endpoint URL seems to be " + + "incorrect for the selected S3 service provider. Please contact Appsmith customer " + + "support to resolve this."); + } +} diff --git a/app/server/appsmith-plugins/amazons3Plugin/src/main/resources/editor/list.json b/app/server/appsmith-plugins/amazons3Plugin/src/main/resources/editor/list.json index 921fb79d20..f3a9fb4a33 100644 --- a/app/server/appsmith-plugins/amazons3Plugin/src/main/resources/editor/list.json +++ b/app/server/appsmith-plugins/amazons3Plugin/src/main/resources/editor/list.json @@ -57,10 +57,8 @@ "configProperty": "actionConfiguration.formData.list.expiry", "controlType": "QUERY_DYNAMIC_INPUT_TEXT", "initialValue": "5", - "show": { - "path": "actionConfiguration.formData.list.signedUrl", - "comparison": "EQUALS", - "value": "YES" + "conditionals": { + "show": "{{actionConfiguration.formData.list.signedUrl === 'YES'}}" } }, { diff --git a/app/server/appsmith-plugins/amazons3Plugin/src/main/resources/form.json b/app/server/appsmith-plugins/amazons3Plugin/src/main/resources/form.json index db7bd0da7d..4c15897d27 100644 --- a/app/server/appsmith-plugins/amazons3Plugin/src/main/resources/form.json +++ b/app/server/appsmith-plugins/amazons3Plugin/src/main/resources/form.json @@ -58,92 +58,6 @@ "initialValue": "", "encrypted": true }, - { - "label": "Region", - "configProperty": "datasourceConfiguration.properties[0].value", - "controlType": "DROP_DOWN", - "isRequired": true, - "hidden": { - "path": "datasourceConfiguration.properties[1].value", - "comparison": "NOT_EQUALS", - "value": "amazon-s3" - }, - "initialValue": "ap-south-1", - "options": [ - { - "label": "ap-south-1", - "value": "ap-south-1" - }, - { - "label": "us-gov-west-1", - "value": "us-gov-west-1" - }, - { - "label": "us-east-1", - "value": "us-east-1" - }, - { - "label": "us-east-2", - "value": "us-east-2" - }, - { - "label": "us-west-1", - "value": "us-west-1" - }, - { - "label": "us-west-2", - "value": "us-west-2" - }, - { - "label": "eu-west-1", - "value": "eu-west-1" - }, - { - "label": "eu-west-2", - "value": "eu-west-2" - }, - { - "label": "eu-west-3", - "value": "eu-west-3" - }, - { - "label": "eu-central-1", - "value": "eu-central-1" - }, - { - "label": "ap-southeast-1", - "value": "ap-southeast-1" - }, - { - "label": "ap-southeast-2", - "value": "ap-southeast-2" - }, - { - "label": "ap-northeast-1", - "value": "ap-northeast-1" - }, - { - "label": "ap-northeast-2", - "value": "ap-northeast-2" - }, - { - "label": "sa-east-1", - "value": "sa-east-1" - }, - { - "label": "cn-north-1", - "value": "cn-north-1" - }, - { - "label": "cn-northwest-1", - "value": "cn-northwest-1" - }, - { - "label": "ca-central-1", - "value": "ca-central-1" - } - ] - }, { "label": "Endpoint URL", "configProperty": "datasourceConfiguration.endpoints[0].host", @@ -171,8 +85,8 @@ "placeholderText": "de-fra1", "hidden": { "path": "datasourceConfiguration.properties[1].value", - "comparison": "EQUALS", - "value": "amazon-s3" + "comparison": "NOT_EQUALS", + "value": "other" } } ] diff --git a/app/server/appsmith-plugins/amazons3Plugin/src/test/java/com/external/plugins/AmazonS3PluginTest.java b/app/server/appsmith-plugins/amazons3Plugin/src/test/java/com/external/plugins/AmazonS3PluginTest.java index 1c7f276f2b..342d0cdf52 100644 --- a/app/server/appsmith-plugins/amazons3Plugin/src/test/java/com/external/plugins/AmazonS3PluginTest.java +++ b/app/server/appsmith-plugins/amazons3Plugin/src/test/java/com/external/plugins/AmazonS3PluginTest.java @@ -1,6 +1,7 @@ package com.external.plugins; import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.amazonaws.services.s3.model.Bucket; import com.amazonaws.services.s3.model.ObjectListing; import com.amazonaws.services.s3.model.S3Object; @@ -44,6 +45,7 @@ import static com.external.plugins.constants.FieldName.LIST_PREFIX; import static com.external.plugins.constants.FieldName.LIST_SIGNED_URL; import static com.external.plugins.constants.FieldName.LIST_UNSIGNED_URL; import static com.external.plugins.constants.FieldName.READ_USING_BASE64_ENCODING; +import static com.external.utils.DatasourceUtils.getS3ClientBuilder; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -80,7 +82,7 @@ public class AmazonS3PluginTest { DatasourceConfiguration dsConfig = new DatasourceConfiguration(); dsConfig.setAuthentication(authDTO); ArrayList properties = new ArrayList<>(); - properties.add(new Property("amazon s3 region", region)); + properties.add(null); // since index 0 is not used anymore. properties.add(new Property("s3 service provider", serviceProvider)); properties.add(new Property("custom endpoint region", region)); dsConfig.setProperties(properties); @@ -137,30 +139,9 @@ public class AmazonS3PluginTest { } @Test - public void testValidateDatasourceWithMissingRegionWithAmazonS3() { + public void testValidateDatasourceWithMissingRegionWithOtherS3ServiceProvider() { DatasourceConfiguration datasourceConfiguration = createDatasourceConfiguration(); - datasourceConfiguration.getProperties().get(0).setValue(""); - - AmazonS3Plugin.S3PluginExecutor pluginExecutor = new AmazonS3Plugin.S3PluginExecutor(); - Mono pluginExecutorMono = Mono.just(pluginExecutor); - - StepVerifier.create(pluginExecutorMono) - .assertNext(executor -> { - Set res = executor.validateDatasource(datasourceConfiguration); - assertNotEquals(0, res.size()); - - List errorList = new ArrayList<>(res); - assertTrue(errorList.get(0).contains("Required parameter 'Region' is empty. Did you forget to " + - "edit the 'Region' field in the datasource creation form ? You need to fill it with the " + - "region where your AWS S3 instance is hosted.")); - }) - .verifyComplete(); - } - - @Test - public void testValidateDatasourceWithMissingRegionWithNonAmazonProvider() { - DatasourceConfiguration datasourceConfiguration = createDatasourceConfiguration(); - datasourceConfiguration.getProperties().get(1).setValue("upcloud"); + datasourceConfiguration.getProperties().get(1).setValue("other"); datasourceConfiguration.getProperties().get(2).setValue(""); AmazonS3Plugin.S3PluginExecutor pluginExecutor = new AmazonS3Plugin.S3PluginExecutor(); @@ -179,6 +160,22 @@ public class AmazonS3PluginTest { .verifyComplete(); } + @Test + public void testValidateDatasourceWithMissingRegionWithListedProvider() { + DatasourceConfiguration datasourceConfiguration = createDatasourceConfiguration(); + datasourceConfiguration.getProperties().get(2).setValue(""); + + AmazonS3Plugin.S3PluginExecutor pluginExecutor = new AmazonS3Plugin.S3PluginExecutor(); + Mono pluginExecutorMono = Mono.just(pluginExecutor); + + StepVerifier.create(pluginExecutorMono) + .assertNext(executor -> { + Set res = executor.validateDatasource(datasourceConfiguration); + assertEquals(0, res.size()); + }) + .verifyComplete(); + } + @Test public void testValidateDatasourceWithMissingUrlWithNonAmazonProvider() { DatasourceConfiguration datasourceConfiguration = createDatasourceConfiguration(); @@ -935,4 +932,53 @@ public class AmazonS3PluginTest { .verifyComplete(); } + @Test + public void testExtractRegionFromEndpoint() { + DatasourceConfiguration datasourceConfiguration = createDatasourceConfiguration(); + + // Test for Upcloud + datasourceConfiguration.getProperties().get(1).setValue("upcloud"); + datasourceConfiguration.getEndpoints().get(0).setHost("appsmith-test-storage-2.de-fra1.upcloudobjects.com"); + + AmazonS3ClientBuilder s3ClientBuilder = getS3ClientBuilder(datasourceConfiguration); + assertEquals("de-fra1", s3ClientBuilder.getEndpoint().getSigningRegion()); + + // Test for Wasabi + datasourceConfiguration.getProperties().get(1).setValue("wasabi"); + datasourceConfiguration.getEndpoints().get(0).setHost("s3.ap-northeast-1.wasabisys.com"); + + s3ClientBuilder = getS3ClientBuilder(datasourceConfiguration); + assertEquals("ap-northeast-1", s3ClientBuilder.getEndpoint().getSigningRegion()); + + // Test for Digital Ocean Spaces + datasourceConfiguration.getProperties().get(1).setValue("digital-ocean-spaces"); + datasourceConfiguration.getEndpoints().get(0).setHost("fra1.digitaloceanspaces.com"); + + s3ClientBuilder = getS3ClientBuilder(datasourceConfiguration); + assertEquals("fra1", s3ClientBuilder.getEndpoint().getSigningRegion()); + + // Test for Dream Objects + datasourceConfiguration.getProperties().get(1).setValue("dream-objects"); + datasourceConfiguration.getEndpoints().get(0).setHost("objects-us-east-1.dream.io"); + + s3ClientBuilder = getS3ClientBuilder(datasourceConfiguration); + assertEquals("us-east-1", s3ClientBuilder.getEndpoint().getSigningRegion()); + } + + @Test + public void testExtractRegionFromEndpointWithBadEndpointFormat() { + DatasourceConfiguration datasourceConfiguration = createDatasourceConfiguration(); + + // Testing for Upcloud here. Flow for other listed service providers is same, hence not testing separately. + datasourceConfiguration.getProperties().get(1).setValue("upcloud"); + datasourceConfiguration.getEndpoints().get(0).setHost("appsmith-test-storage-2..de-fra1.upcloudobjects.com"); + + StepVerifier.create(Mono.fromCallable(() -> getS3ClientBuilder(datasourceConfiguration))) + .expectErrorSatisfies(error -> { + String expectedErrorMessage = "Your S3 endpoint URL seems to be incorrect for the selected S3 " + + "service provider. Please check your endpoint URL and the selected S3 service provider."; + assertEquals(expectedErrorMessage, error.getMessage()); + }) + .verify(); + } } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/DatabaseChangelog.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/DatabaseChangelog.java index 45bda26c5f..870ed9790b 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/DatabaseChangelog.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/DatabaseChangelog.java @@ -3467,7 +3467,7 @@ public class DatabaseChangelog { return new HashMap<>(); } - @ChangeSet(order = "093", id = "migrate-s3-to-uqi", author = "") + @ChangeSet(order = "094", id = "migrate-s3-to-uqi", author = "") public void migrateS3PluginToUqi(MongockTemplate mongockTemplate) { ObjectMapper objectMapper = new ObjectMapper();