Protei/PromucFlow_constructor
2
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

feat: remove region requirement from s3 plugin (#8829)

Browse Source 
* Remove Region field from S3 datasource editor page for AWS S3, Upcloud, Digital Ocean Spaces, Dream Objects, Wasabi.
  * Use SDK provided property for AWS S3 to delegate region selection to the SDK.
  * Extract region info from endpoint URL for Upcloud, Digital Ocean Spaces, Dream Objects and Wasabi, since the SDK property does not work for these service providers.
* Removed some redundant checks from datasourceCreate that were already part of validateDatasource
* Fix show clause in list.json
This commit is contained in:
Sumit Kumar 2021-11-03 16:23:53 +05:30 committed by GitHub
parent fe73e97206
commit 3c8583210f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 323 additions and 306 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

205
app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/plugins/AmazonS3Plugin.java vendored
View File

@ -1,12 +1,7 @@
package com.external.plugins;
import com.amazonaws.HttpMethod;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.client.builder.AwsClientBuilder;
import com.amazonaws.regions.Regions;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.AmazonS3Exception;
import com.amazonaws.services.s3.model.Bucket;
import com.amazonaws.services.s3.model.GeneratePresignedUrlRequest;
@ -32,6 +27,7 @@ import com.appsmith.external.models.Property;
import com.appsmith.external.models.RequestParamDTO;
import com.appsmith.external.plugins.BasePlugin;
import com.appsmith.external.plugins.PluginExecutor;
import com.external.plugins.constants.AmazonS3Action;
import lombok.extern.slf4j.Slf4j;
import org.pf4j.Extension;
import org.pf4j.PluginWrapper;
@ -61,6 +57,7 @@ import java.util.stream.Collectors;
import static com.appsmith.external.constants.ActionConstants.ACTION_CONFIGURATION_BODY;
import static com.appsmith.external.constants.ActionConstants.ACTION_CONFIGURATION_PATH;
import static com.external.utils.DatasourceUtils.getS3ClientBuilder;
import static com.appsmith.external.helpers.PluginUtils.getValueSafelyFromFormData;
import static com.appsmith.external.helpers.PluginUtils.getValueSafelyFromFormDataOrDefault;
import static com.appsmith.external.helpers.PluginUtils.setValueSafelyInFormData;
@ -78,15 +75,15 @@ import static com.external.plugins.constants.FieldName.READ_USING_BASE64_ENCODIN
public class AmazonS3Plugin extends BasePlugin {
private static final String S3_DRIVER = "com.amazonaws.services.s3.AmazonS3";
private static final int AWS_S3_REGION_PROPERTY_INDEX = 0;
private static final int S3_SERVICE_PROVIDER_PROPERTY_INDEX = 1;
private static final int CUSTOM_ENDPOINT_REGION_PROPERTY_INDEX = 2;
private static final int CUSTOM_ENDPOINT_INDEX = 0;
public static final int S3_SERVICE_PROVIDER_PROPERTY_INDEX = 1;
public static final int CUSTOM_ENDPOINT_REGION_PROPERTY_INDEX = 2;
public static final int CUSTOM_ENDPOINT_INDEX = 0;
private static final String DEFAULT_URL_EXPIRY_IN_MINUTES = "5"; // max 7 days is possible
private static final String YES = "YES";
private static final String NO = "NO";
private static final String BASE64_DELIMITER = ";base64,";
private static final String AMAZON_S3_SERVICE_PROVIDER = "amazon-s3";
private static final String OTHER_S3_SERVICE_PROVIDER = "other";
private static final String AWS_S3_SERVICE_PROVIDER = "amazon-s3";
public AmazonS3Plugin(PluginWrapper wrapper) {
super(wrapper);
@ -270,7 +267,6 @@ public class AmazonS3Plugin extends BasePlugin {
Map<String, Object> requestProperties = new HashMap<>();
List<RequestParamDTO> requestParams = new ArrayList<>();
return Mono.fromCallable(() -> {
/*
@ -282,16 +278,6 @@ public class AmazonS3Plugin extends BasePlugin {
return Mono.error(new StaleConnectionException());
}
if (datasourceConfiguration == null) {
return Mono.error(
new AppsmithPluginException(
AppsmithPluginError.PLUGIN_EXECUTE_ARGUMENT_ERROR,
"At least one of the mandatory fields in S3 datasource creation form is empty - " +
"'Access Key'/'Secret Key'/'Region'. Please fill all the mandatory fields and try again."
)
);
}
if (actionConfiguration == null) {
return Mono.error(
new AppsmithPluginException(
@ -580,7 +566,7 @@ public class AmazonS3Plugin extends BasePlugin {
return Mono.just(result);
})
// Now set the request in the result to be returned back to the server
// Now set the request in the result to be returned to the server
.map(actionExecutionResult -> {
ActionExecutionRequest actionExecutionRequest = new ActionExecutionRequest();
actionExecutionRequest.setQuery(query[0]);
@ -595,16 +581,6 @@ public class AmazonS3Plugin extends BasePlugin {
@Override
public Mono<AmazonS3> datasourceCreate(DatasourceConfiguration datasourceConfiguration) {
if (datasourceConfiguration == null) {
return Mono.error(
new AppsmithPluginException(
AppsmithPluginError.PLUGIN_DATASOURCE_ARGUMENT_ERROR,
"Mandatory fields 'Access Key', 'Secret Key', 'Region' missing. Did you forget to edit " +
"the 'Access Key'/'Secret Key'/'Region' fields in the datasource creation form?"
)
);
}
try {
Class.forName(S3_DRIVER);
} catch (ClassNotFoundException e) {
@ -617,137 +593,8 @@ public class AmazonS3Plugin extends BasePlugin {
);
}
return (Mono<AmazonS3>) Mono.fromCallable(() -> {
List<Property> properties = datasourceConfiguration.getProperties();
/*
* - Ideally, properties must never be null because the fields contained in the properties list have a
* default value defined.
* - Ideally, properties.get(S3_SERVICE_PROVIDER_PROPERTY_INDEX) must never be null/empty, because the
* `S3 Service Provider` dropdown has a default value.
*/
if (properties == null
|| properties.get(S3_SERVICE_PROVIDER_PROPERTY_INDEX) == null
|| StringUtils.isNullOrEmpty((String) properties.get(S3_SERVICE_PROVIDER_PROPERTY_INDEX).getValue())) {
return Mono.error(
new AppsmithPluginException(
AppsmithPluginError.PLUGIN_DATASOURCE_ARGUMENT_ERROR,
"Appsmith has failed to fetch the 'S3 Service Provider' field properties. Please " +
"reach out to Appsmith customer support to resolve this."
)
);
}
final boolean usingCustomEndpoint =
!AMAZON_S3_SERVICE_PROVIDER.equals(properties.get(S3_SERVICE_PROVIDER_PROPERTY_INDEX).getValue());
if (!usingCustomEndpoint
&& (properties.size() < (AWS_S3_REGION_PROPERTY_INDEX + 1)
|| properties.get(AWS_S3_REGION_PROPERTY_INDEX) == null
|| StringUtils.isNullOrEmpty((String) properties.get(AWS_S3_REGION_PROPERTY_INDEX).getValue()))) {
return Mono.error(
new AppsmithPluginException(
AppsmithPluginError.PLUGIN_DATASOURCE_ARGUMENT_ERROR,
"Required parameter 'Region' is empty. Did you forget to edit the 'Region' field" +
" in the datasource creation form ? You need to fill it with the region where " +
"your AWS S3 instance is hosted."
)
);
}
if (usingCustomEndpoint
&& (datasourceConfiguration.getEndpoints() == null
|| CollectionUtils.isEmpty(datasourceConfiguration.getEndpoints())
|| datasourceConfiguration.getEndpoints().get(CUSTOM_ENDPOINT_INDEX) == null
|| StringUtils.isNullOrEmpty(datasourceConfiguration.getEndpoints().get(CUSTOM_ENDPOINT_INDEX).getHost()))) {
return Mono.error(
new AppsmithPluginException(
AppsmithPluginError.PLUGIN_DATASOURCE_ARGUMENT_ERROR,
"Required parameter 'Endpoint URL' is empty. Did you forget to edit the 'Endpoint" +
" URL' field in the datasource creation form ? You need to fill it with " +
"the endpoint URL of your S3 instance."
)
);
}
if (usingCustomEndpoint
&& (properties.size() < (CUSTOM_ENDPOINT_REGION_PROPERTY_INDEX + 1)
|| properties.get(CUSTOM_ENDPOINT_REGION_PROPERTY_INDEX) == null
|| StringUtils.isNullOrEmpty((String) properties.get(CUSTOM_ENDPOINT_REGION_PROPERTY_INDEX).getValue()))) {
return Mono.error(
new AppsmithPluginException(
AppsmithPluginError.PLUGIN_DATASOURCE_ARGUMENT_ERROR,
"Required parameter 'Region' is empty. Did you forget to edit the 'Region' field" +
" in the datasource creation form ? You need to fill it with the region where " +
"your S3 instance is hosted."
)
);
}
final String region = (String) (usingCustomEndpoint ?
properties.get(CUSTOM_ENDPOINT_REGION_PROPERTY_INDEX).getValue() :
properties.get(AWS_S3_REGION_PROPERTY_INDEX).getValue());
DBAuth authentication = (DBAuth) datasourceConfiguration.getAuthentication();
if (authentication == null
|| StringUtils.isNullOrEmpty(authentication.getUsername())
|| StringUtils.isNullOrEmpty(authentication.getPassword())) {
return Mono.error(
new AppsmithPluginException(
AppsmithPluginError.PLUGIN_DATASOURCE_ARGUMENT_ERROR,
"Mandatory parameters 'Access Key' and/or 'Secret Key' are missing. Did you " +
"forget to edit the 'Access Key'/'Secret Key' fields in the datasource creation form ?"
)
);
}
String accessKey = authentication.getUsername();
String secretKey = authentication.getPassword();
BasicAWSCredentials awsCreds;
try {
awsCreds = new BasicAWSCredentials(accessKey, secretKey);
} catch (IllegalArgumentException e) {
return Mono.error(
new AppsmithPluginException(
AppsmithPluginError.PLUGIN_DATASOURCE_ARGUMENT_ERROR,
"Appsmith server has encountered an error when " +
"parsing AWS credentials from datasource: " + e.getMessage()
)
);
}
AmazonS3ClientBuilder s3ClientBuilder = AmazonS3ClientBuilder
.standard()
.withCredentials(new AWSStaticCredentialsProvider(awsCreds));
if (!usingCustomEndpoint) {
Regions clientRegion = null;
try {
clientRegion = Regions.fromName(region);
} catch (IllegalArgumentException e) {
return Mono.error(
new AppsmithPluginException(
AppsmithPluginError.PLUGIN_DATASOURCE_ARGUMENT_ERROR,
"Appsmith server has encountered an error when " +
"parsing AWS S3 instance region from the AWS S3 datasource configuration " +
"provided: " + e.getMessage()
)
);
}
s3ClientBuilder = s3ClientBuilder.withRegion(clientRegion);
} else {
String endpoint = datasourceConfiguration.getEndpoints().get(CUSTOM_ENDPOINT_INDEX).getHost();
s3ClientBuilder = s3ClientBuilder
.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, region));
}
return Mono.just(s3ClientBuilder.build());
})
.flatMap(obj -> obj)
return Mono.fromCallable(() -> getS3ClientBuilder(datasourceConfiguration).build())
.flatMap(client -> Mono.just(client))
.onErrorResume(e -> {
if (e instanceof AppsmithPluginException) {
return Mono.error(e);
@ -757,7 +604,7 @@ public class AmazonS3Plugin extends BasePlugin {
new AppsmithPluginException(
AppsmithPluginError.PLUGIN_ERROR,
"Appsmith server has encountered an error when " +
"connecting to AWS S3 server: " + e.getMessage()
"connecting to your S3 server: " + e.getMessage()
)
);
}
@ -817,21 +664,11 @@ public class AmazonS3Plugin extends BasePlugin {
invalids.add("Appsmith has failed to fetch the 'S3 Service Provider' field properties. Please " +
"reach out to Appsmith customer support to resolve this.");
}
final boolean usingCustomEndpoint =
!AMAZON_S3_SERVICE_PROVIDER.equals(properties.get(S3_SERVICE_PROVIDER_PROPERTY_INDEX).getValue());
if (!usingCustomEndpoint
&& (properties.size() < (AWS_S3_REGION_PROPERTY_INDEX + 1)
|| properties.get(AWS_S3_REGION_PROPERTY_INDEX) == null
|| StringUtils.isNullOrEmpty((String) properties.get(AWS_S3_REGION_PROPERTY_INDEX).getValue()))) {
invalids.add("Required parameter 'Region' is empty. Did you forget to edit the 'Region' field" +
" in the datasource creation form ? You need to fill it with the region where " +
"your AWS S3 instance is hosted.");
}
if (usingCustomEndpoint
&& (datasourceConfiguration.getEndpoints() == null
|| CollectionUtils.isEmpty(datasourceConfiguration.getEndpoints())
final boolean usingAWSS3ServiceProvider =
AWS_S3_SERVICE_PROVIDER.equals(properties.get(S3_SERVICE_PROVIDER_PROPERTY_INDEX).getValue());
if (!usingAWSS3ServiceProvider
&& (CollectionUtils.isEmpty(datasourceConfiguration.getEndpoints())
|| datasourceConfiguration.getEndpoints().get(CUSTOM_ENDPOINT_INDEX) == null
|| StringUtils.isNullOrEmpty(datasourceConfiguration.getEndpoints().get(CUSTOM_ENDPOINT_INDEX).getHost()))) {
invalids.add("Required parameter 'Endpoint URL' is empty. Did you forget to edit the 'Endpoint" +
@ -839,7 +676,9 @@ public class AmazonS3Plugin extends BasePlugin {
"the endpoint URL of your S3 instance.");
}
if (usingCustomEndpoint
final boolean usingCustomServiceProvider =
OTHER_S3_SERVICE_PROVIDER.equals(properties.get(S3_SERVICE_PROVIDER_PROPERTY_INDEX).getValue());
if (usingCustomServiceProvider
&& (properties.size() < (CUSTOM_ENDPOINT_REGION_PROPERTY_INDEX + 1)
|| properties.get(CUSTOM_ENDPOINT_REGION_PROPERTY_INDEX) == null
|| StringUtils.isNullOrEmpty((String) properties.get(CUSTOM_ENDPOINT_REGION_PROPERTY_INDEX).getValue()))) {
@ -853,14 +692,6 @@ public class AmazonS3Plugin extends BasePlugin {
@Override
public Mono<DatasourceTestResult> testDatasource(DatasourceConfiguration datasourceConfiguration) {
if (datasourceConfiguration == null) {
return Mono.just(
new DatasourceTestResult(
"At least one of the mandatory fields in S3 datasource creation form is empty - " +
"'Access Key'/'Secret Key'/'Region'. Please fill all the mandatory fields and try again."
)
);
}
return datasourceCreate(datasourceConfiguration)
.map(connection -> {

4
app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/plugins/AmazonS3Action.java → app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/plugins/constants/AmazonS3Action.java vendored
View File

@ -1,6 +1,6 @@
package com.external.plugins;
package com.external.plugins.constants;
enum AmazonS3Action {
public enum AmazonS3Action {
LIST,
UPLOAD_FILE_FROM_BODY,
READ_FILE,

228
app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/utils/DatasourceUtils.java vendored Normal file
View File

@ -0,0 +1,228 @@
package com.external.utils;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.client.builder.AwsClientBuilder;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginError;
import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginException;
import com.appsmith.external.models.DBAuth;
import com.appsmith.external.models.DatasourceConfiguration;
import com.appsmith.external.models.Property;
import org.apache.commons.lang.StringUtils;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import static com.amazonaws.regions.Regions.DEFAULT_REGION;
import static com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginError.PLUGIN_ERROR;
import static com.external.plugins.AmazonS3Plugin.CUSTOM_ENDPOINT_INDEX;
import static com.external.plugins.AmazonS3Plugin.CUSTOM_ENDPOINT_REGION_PROPERTY_INDEX;
import static com.external.plugins.AmazonS3Plugin.S3_SERVICE_PROVIDER_PROPERTY_INDEX;
import static com.external.utils.DatasourceUtils.S3ServiceProvider.AMAZON;
public class DatasourceUtils {
/**
* Example endpoint : appsmith-test-storage-2.de-fra1.upcloudobjects.com
* Group 2 match: de-fra1
*/
public static String UPCLOUD_URL_ENDPOINT_PATTERN = "^([^\\.]+)\\.([^\\.]+)\\.upcloudobjects\\.com$";
public static int UPCLOUD_REGION_GROUP_INDEX = 2;
/**
* Example endpoint : s3.ap-northeast-2.wasabisys.com
* Group 2 match: ap-northeast-2
*/
public static String WASABI_URL_ENDPOINT_PATTERN = "^([^\\.]+)\\.([^\\.]+)\\.wasabisys\\.com$";
public static int WASABI_REGION_GROUP_INDEX = 2;
/**
* Example endpoint : fra1.digitaloceanspaces.com
* Group 1 match: fra1
*/
public static String DIGITAL_OCEAN_URL_ENDPOINT_PATTERN = "^([^\\.]+)\\.digitaloceanspaces\\.com$";
public static int DIGITAL_OCEAN_REGION_GROUP_INDEX = 1;
/**
* Example endpoint : objects-us-east-1.dream.io
* Group 1 match: us-east-1
*/
public static String DREAM_OBJECTS_URL_ENDPOINT_PATTERN = "^objects-([^\\.]+)\\.dream\\.io$";
public static int DREAM_OBJECTS_REGION_GROUP_INDEX = 1;
/* This enum lists various types of S3 service providers that we support. */
public enum S3ServiceProvider {
AMAZON ("amazon-s3"),
UPCLOUD ("upcloud"),
WASABI ("wasabi"),
DIGITAL_OCEAN_SPACES ("digital-ocean-spaces"),
DREAM_OBJECTS ("dream-objects"),
OTHER ("other");
private String name;
S3ServiceProvider(String name) {
this.name = name;
}
public static S3ServiceProvider fromString(String name) throws AppsmithPluginException {
for (S3ServiceProvider s3ServiceProvider : S3ServiceProvider.values()) {
if (s3ServiceProvider.name.equals(name.toLowerCase())) {
return s3ServiceProvider;
}
}
throw new AppsmithPluginException(PLUGIN_ERROR, "Appsmith S3 plugin service has " +
"failed to identify the S3 service provider type. Please reach out to Appsmith customer support" +
" to resolve this");
}
}
/**
* This method builds an `AmazonS3ClientBuilder` object from the datasourceConfiguration provided by user. The
* `AmazonS3ClientBuilder` object can then be used to get a connection object to connect to the S3 service.
*
* @param datasourceConfiguration
* @return AmazonS3ClientBuilder object
* @throws AppsmithPluginException when (1) there is an error with parsing credentials (2) required
* datasourceConfiguration properties are missing (3) endpoint URL is found incorrect.
*/
public static AmazonS3ClientBuilder getS3ClientBuilder (DatasourceConfiguration datasourceConfiguration)
throws AppsmithPluginException {
DBAuth authentication = (DBAuth) datasourceConfiguration.getAuthentication();
String accessKey = authentication.getUsername();
String secretKey = authentication.getPassword();
BasicAWSCredentials awsCreds;
try {
awsCreds = new BasicAWSCredentials(accessKey, secretKey);
} catch (IllegalArgumentException e) {
throw new AppsmithPluginException(
AppsmithPluginError.PLUGIN_DATASOURCE_ARGUMENT_ERROR,
"Appsmith server has encountered an error when parsing AWS credentials from datasource: "
+ e.getMessage()
);
}
/* Set credentials in client builder. */
AmazonS3ClientBuilder s3ClientBuilder = AmazonS3ClientBuilder
.standard()
.withCredentials(new AWSStaticCredentialsProvider(awsCreds));
List<Property> properties = datasourceConfiguration.getProperties();
/**
* Return error if no service provider is chosen.
*
* Ideally, properties.get(S3_SERVICE_PROVIDER_PROPERTY_INDEX) must always exist, because the `S3
* Service Provider` dropdown has a default value.
*/
if (properties == null
|| properties.get(S3_SERVICE_PROVIDER_PROPERTY_INDEX) == null
|| StringUtils.isEmpty((String) properties.get(S3_SERVICE_PROVIDER_PROPERTY_INDEX).getValue())) {
throw new AppsmithPluginException(
AppsmithPluginError.PLUGIN_DATASOURCE_ARGUMENT_ERROR,
"Appsmith has failed to fetch the 'S3 Service Provider' field properties. Please reach out to" +
" Appsmith customer support to resolve this."
);
}
S3ServiceProvider s3ServiceProvider =
S3ServiceProvider.fromString((String) properties.get(S3_SERVICE_PROVIDER_PROPERTY_INDEX).getValue());
/**
* AmazonS3 provides an attribute `forceGlobalBucketAccessEnabled` that automatically routes the request to a
* region such that request should succeed.
* Ref: https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/S3ClientOptions
* .Builder.html#enableForceGlobalBucketAccess--
*
* However, no mention of the attribute `forceGlobalBucketAccessEnabled` could be found within the
* documentation of other listed S3 service providers like Upcloud, Wasabi, Dream Objects, or Digital Ocean
* Spaces. Also, some these services failed on usage of this attribute - hence this attribute could not be
* reliably used for these S3 service providers. For these service providers, the region information is
* chained in the endpoint URL. Hence, the endpoint URL is used to extract the exact object storage region.
*
* Apart from the listed S3 services - AWS, Upcloud, Wasabi, Dream Objects and Digital Ocean Spaces, any other
* service provider falls in the category `other` and there is no special handling defined for it since we
* cannot assume any information about them beforehand. For this S3 service provider type region must be
* explicitly provided.
*/
if (s3ServiceProvider.equals(AMAZON)) {
s3ClientBuilder = s3ClientBuilder
.withRegion(DEFAULT_REGION)
.enableForceGlobalBucketAccess();
}
else {
String endpoint = datasourceConfiguration.getEndpoints().get(CUSTOM_ENDPOINT_INDEX).getHost();
String region = "";
switch(s3ServiceProvider) {
case AMAZON:
/* This case can never be reached because of the if condition above. Just adding for sake of
completeness. */
break;
case UPCLOUD:
region = getRegionFromEndpointPattern(endpoint, UPCLOUD_URL_ENDPOINT_PATTERN,
UPCLOUD_REGION_GROUP_INDEX);
break;
case WASABI:
region = getRegionFromEndpointPattern(endpoint, WASABI_URL_ENDPOINT_PATTERN,
WASABI_REGION_GROUP_INDEX);
break;
case DIGITAL_OCEAN_SPACES:
region = getRegionFromEndpointPattern(endpoint, DIGITAL_OCEAN_URL_ENDPOINT_PATTERN,
DIGITAL_OCEAN_REGION_GROUP_INDEX);
break;
case DREAM_OBJECTS:
region = getRegionFromEndpointPattern(endpoint, DREAM_OBJECTS_URL_ENDPOINT_PATTERN,
DREAM_OBJECTS_REGION_GROUP_INDEX);
break;
default:
region = (String) properties.get(CUSTOM_ENDPOINT_REGION_PROPERTY_INDEX).getValue();
}
s3ClientBuilder = s3ClientBuilder
.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, region));
}
return s3ClientBuilder;
}
/**
* This method checks if the S3 endpoint URL has correct format and extracts region information from it.
*
* @param endpoint : endpoint URL
* @param regex : expected endpoint URL pattern
* @param regionGroupIndex : pattern group index for region string
* @return S3 object storage region.
* @throws AppsmithPluginException when then endpoint URL does not match the expected regex pattern.
*/
private static String getRegionFromEndpointPattern(String endpoint, String regex, int regionGroupIndex)
throws AppsmithPluginException {
/* endpoint is expected to be non-null at this point */
if (!endpoint.matches(regex)) {
throw new AppsmithPluginException(AppsmithPluginError.PLUGIN_DATASOURCE_ARGUMENT_ERROR, "Your S3 endpoint" +
" URL seems to be incorrect for the selected S3 service provider. Please check your endpoint URL " +
"and the selected S3 service provider.");
}
Pattern pattern = Pattern.compile(regex);
Matcher matcher = pattern.matcher(endpoint);
if (matcher.find()) {
return matcher.group(regionGroupIndex);
}
/* Code flow is never expected to reach here. */
throw new AppsmithPluginException(AppsmithPluginError.PLUGIN_ERROR, "Your S3 endpoint URL seems to be " +
"incorrect for the selected S3 service provider. Please contact Appsmith customer " +
"support to resolve this.");
}
}

6
app/server/appsmith-plugins/amazons3Plugin/src/main/resources/editor/list.json
View File

@ -57,10 +57,8 @@
"configProperty": "actionConfiguration.formData.list.expiry",
"controlType": "QUERY_DYNAMIC_INPUT_TEXT",
"initialValue": "5",
"show": {
"path": "actionConfiguration.formData.list.signedUrl",
"comparison": "EQUALS",
"value": "YES"
"conditionals": {
"show": "{{actionConfiguration.formData.list.signedUrl === 'YES'}}"
}
},
{

90
app/server/appsmith-plugins/amazons3Plugin/src/main/resources/form.json
View File

@ -58,92 +58,6 @@
"initialValue": "",
"encrypted": true
},
{
"label": "Region",
"configProperty": "datasourceConfiguration.properties[0].value",
"controlType": "DROP_DOWN",
"isRequired": true,
"hidden": {
"path": "datasourceConfiguration.properties[1].value",
"comparison": "NOT_EQUALS",
"value": "amazon-s3"
},
"initialValue": "ap-south-1",
"options": [
{
"label": "ap-south-1",
"value": "ap-south-1"
},
{
"label": "us-gov-west-1",
"value": "us-gov-west-1"
},
{
"label": "us-east-1",
"value": "us-east-1"
},
{
"label": "us-east-2",
"value": "us-east-2"
},
{
"label": "us-west-1",
"value": "us-west-1"
},
{
"label": "us-west-2",
"value": "us-west-2"
},
{
"label": "eu-west-1",
"value": "eu-west-1"
},
{
"label": "eu-west-2",
"value": "eu-west-2"
},
{
"label": "eu-west-3",
"value": "eu-west-3"
},
{
"label": "eu-central-1",
"value": "eu-central-1"
},
{
"label": "ap-southeast-1",
"value": "ap-southeast-1"
},
{
"label": "ap-southeast-2",
"value": "ap-southeast-2"
},
{
"label": "ap-northeast-1",
"value": "ap-northeast-1"
},
{
"label": "ap-northeast-2",
"value": "ap-northeast-2"
},
{
"label": "sa-east-1",
"value": "sa-east-1"
},
{
"label": "cn-north-1",
"value": "cn-north-1"
},
{
"label": "cn-northwest-1",
"value": "cn-northwest-1"
},
{
"label": "ca-central-1",
"value": "ca-central-1"
}
]
},
{
"label": "Endpoint URL",
"configProperty": "datasourceConfiguration.endpoints[0].host",
@ -171,8 +85,8 @@
"placeholderText": "de-fra1",
"hidden": {
"path": "datasourceConfiguration.properties[1].value",
"comparison": "EQUALS",
"value": "amazon-s3"
"comparison": "NOT_EQUALS",
"value": "other"
}
}
]

94
app/server/appsmith-plugins/amazons3Plugin/src/test/java/com/external/plugins/AmazonS3PluginTest.java vendored
View File

@ -1,6 +1,7 @@
package com.external.plugins;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.Bucket;
import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.S3Object;
@ -44,6 +45,7 @@ import static com.external.plugins.constants.FieldName.LIST_PREFIX;
import static com.external.plugins.constants.FieldName.LIST_SIGNED_URL;
import static com.external.plugins.constants.FieldName.LIST_UNSIGNED_URL;
import static com.external.plugins.constants.FieldName.READ_USING_BASE64_ENCODING;
import static com.external.utils.DatasourceUtils.getS3ClientBuilder;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@ -80,7 +82,7 @@ public class AmazonS3PluginTest {
DatasourceConfiguration dsConfig = new DatasourceConfiguration();
dsConfig.setAuthentication(authDTO);
ArrayList<Property> properties = new ArrayList<>();
properties.add(new Property("amazon s3 region", region));
properties.add(null); // since index 0 is not used anymore.
properties.add(new Property("s3 service provider", serviceProvider));
properties.add(new Property("custom endpoint region", region));
dsConfig.setProperties(properties);
@ -137,30 +139,9 @@ public class AmazonS3PluginTest {
}
@Test
public void testValidateDatasourceWithMissingRegionWithAmazonS3() {
public void testValidateDatasourceWithMissingRegionWithOtherS3ServiceProvider() {
DatasourceConfiguration datasourceConfiguration = createDatasourceConfiguration();
datasourceConfiguration.getProperties().get(0).setValue("");
AmazonS3Plugin.S3PluginExecutor pluginExecutor = new AmazonS3Plugin.S3PluginExecutor();
Mono<AmazonS3Plugin.S3PluginExecutor> pluginExecutorMono = Mono.just(pluginExecutor);
StepVerifier.create(pluginExecutorMono)
.assertNext(executor -> {
Set<String> res = executor.validateDatasource(datasourceConfiguration);
assertNotEquals(0, res.size());
List<String> errorList = new ArrayList<>(res);
assertTrue(errorList.get(0).contains("Required parameter 'Region' is empty. Did you forget to " +
"edit the 'Region' field in the datasource creation form ? You need to fill it with the " +
"region where your AWS S3 instance is hosted."));
})
.verifyComplete();
}
@Test
public void testValidateDatasourceWithMissingRegionWithNonAmazonProvider() {
DatasourceConfiguration datasourceConfiguration = createDatasourceConfiguration();
datasourceConfiguration.getProperties().get(1).setValue("upcloud");
datasourceConfiguration.getProperties().get(1).setValue("other");
datasourceConfiguration.getProperties().get(2).setValue("");
AmazonS3Plugin.S3PluginExecutor pluginExecutor = new AmazonS3Plugin.S3PluginExecutor();
@ -179,6 +160,22 @@ public class AmazonS3PluginTest {
.verifyComplete();
}
@Test
public void testValidateDatasourceWithMissingRegionWithListedProvider() {
DatasourceConfiguration datasourceConfiguration = createDatasourceConfiguration();
datasourceConfiguration.getProperties().get(2).setValue("");
AmazonS3Plugin.S3PluginExecutor pluginExecutor = new AmazonS3Plugin.S3PluginExecutor();
Mono<AmazonS3Plugin.S3PluginExecutor> pluginExecutorMono = Mono.just(pluginExecutor);
StepVerifier.create(pluginExecutorMono)
.assertNext(executor -> {
Set<String> res = executor.validateDatasource(datasourceConfiguration);
assertEquals(0, res.size());
})
.verifyComplete();
}
@Test
public void testValidateDatasourceWithMissingUrlWithNonAmazonProvider() {
DatasourceConfiguration datasourceConfiguration = createDatasourceConfiguration();
@ -935,4 +932,53 @@ public class AmazonS3PluginTest {
.verifyComplete();
}
@Test
public void testExtractRegionFromEndpoint() {
DatasourceConfiguration datasourceConfiguration = createDatasourceConfiguration();
// Test for Upcloud
datasourceConfiguration.getProperties().get(1).setValue("upcloud");
datasourceConfiguration.getEndpoints().get(0).setHost("appsmith-test-storage-2.de-fra1.upcloudobjects.com");
AmazonS3ClientBuilder s3ClientBuilder = getS3ClientBuilder(datasourceConfiguration);
assertEquals("de-fra1", s3ClientBuilder.getEndpoint().getSigningRegion());
// Test for Wasabi
datasourceConfiguration.getProperties().get(1).setValue("wasabi");
datasourceConfiguration.getEndpoints().get(0).setHost("s3.ap-northeast-1.wasabisys.com");
s3ClientBuilder = getS3ClientBuilder(datasourceConfiguration);
assertEquals("ap-northeast-1", s3ClientBuilder.getEndpoint().getSigningRegion());
// Test for Digital Ocean Spaces
datasourceConfiguration.getProperties().get(1).setValue("digital-ocean-spaces");
datasourceConfiguration.getEndpoints().get(0).setHost("fra1.digitaloceanspaces.com");
s3ClientBuilder = getS3ClientBuilder(datasourceConfiguration);
assertEquals("fra1", s3ClientBuilder.getEndpoint().getSigningRegion());
// Test for Dream Objects
datasourceConfiguration.getProperties().get(1).setValue("dream-objects");
datasourceConfiguration.getEndpoints().get(0).setHost("objects-us-east-1.dream.io");
s3ClientBuilder = getS3ClientBuilder(datasourceConfiguration);
assertEquals("us-east-1", s3ClientBuilder.getEndpoint().getSigningRegion());
}
@Test
public void testExtractRegionFromEndpointWithBadEndpointFormat() {
DatasourceConfiguration datasourceConfiguration = createDatasourceConfiguration();
// Testing for Upcloud here. Flow for other listed service providers is same, hence not testing separately.
datasourceConfiguration.getProperties().get(1).setValue("upcloud");
datasourceConfiguration.getEndpoints().get(0).setHost("appsmith-test-storage-2..de-fra1.upcloudobjects.com");
StepVerifier.create(Mono.fromCallable(() -> getS3ClientBuilder(datasourceConfiguration)))
.expectErrorSatisfies(error -> {
String expectedErrorMessage = "Your S3 endpoint URL seems to be incorrect for the selected S3 " +
"service provider. Please check your endpoint URL and the selected S3 service provider.";
assertEquals(expectedErrorMessage, error.getMessage());
})
.verify();
}
}

2
app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/DatabaseChangelog.java
View File

@ -3467,7 +3467,7 @@ public class DatabaseChangelog {
return new HashMap<>();
}
@ChangeSet(order = "093", id = "migrate-s3-to-uqi", author = "")
@ChangeSet(order = "094", id = "migrate-s3-to-uqi", author = "")
public void migrateS3PluginToUqi(MongockTemplate mongockTemplate) {
ObjectMapper objectMapper = new ObjectMapper();