Protei/PromucFlow_constructor
2
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

fix: Class cast exception during file upload in S3 bucket (#15832) (#16746)

Browse Source
This commit is contained in:
subratadeypappu 2022-09-16 21:14:27 +06:00 committed by GitHub
parent cb84bc68c5
commit e791d7f0d9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 199 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/plugins/AmazonS3Plugin.java vendored
View File

@ -68,6 +68,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.LinkedHashMap;
import static com.appsmith.external.constants.ActionConstants.ACTION_CONFIGURATION_BODY;
import static com.appsmith.external.constants.ActionConstants.ACTION_CONFIGURATION_PATH;
@ -253,7 +254,8 @@ public class AmazonS3Plugin extends BasePlugin {
}
if (Boolean.TRUE.equals(usingFilePicker)) {
String encodedPayload = (String) multipartFormDataDTO.getData();
String encodedPayload = getEncodedPayloadFromMultipartDTO(multipartFormDataDTO);
/*
* - For files uploaded using Filepicker.xyz.base64, body format is "<content-type>;base64,<actual-
* base64-encoded-payload>".
@ -274,25 +276,17 @@ public class AmazonS3Plugin extends BasePlugin {
);
}
} else {
payload = ((String) multipartFormDataDTO.getData()).getBytes();
payload = getEncodedPayloadFromMultipartDTO(multipartFormDataDTO).getBytes();
}
InputStream inputStream = new ByteArrayInputStream(payload);
TransferManager transferManager = TransferManagerBuilder.standard().withS3Client(connection).build();
final ObjectMetadata objectMetadata = new ObjectMetadata();
// Only add content type if the user has mentioned it in the body
if (multipartFormDataDTO.getType() != null) {
objectMetadata.setContentType(multipartFormDataDTO.getType());
}
transferManager.upload(bucketName, path, inputStream, objectMetadata).waitForUploadResult();
uploadFileInS3(payload, connection, multipartFormDataDTO, bucketName, path);
ArrayList<String> listOfFiles = new ArrayList<>();
listOfFiles.add(path);
ArrayList<String> listOfUrls = getSignedUrls(connection, bucketName, listOfFiles, expiryDateTime);
if (listOfUrls.size() != 1) {
throw new AppsmithPluginException(
AppsmithPluginError.PLUGIN_ERROR,
"Appsmith has encountered an unexpected error when fetching url from AmazonS3 after file " +
"Appsmith has encountered an unexpected error while fetching url from AmazonS3 after file " +
"creation. Please reach out to Appsmith customer support to resolve this."
);
}
@ -311,7 +305,7 @@ public class AmazonS3Plugin extends BasePlugin {
String body,
Boolean usingFilePicker,
Date expiryDateTime)
throws InterruptedException, AppsmithPluginException {
throws AppsmithPluginException {
List<MultipartFormDataDTO> multipartFormDataDTOs;
@ -332,7 +326,7 @@ public class AmazonS3Plugin extends BasePlugin {
byte[] payload;
if (Boolean.TRUE.equals(usingFilePicker)) {
String encodedPayload = (String) multipartFormDataDTO.getData();
String encodedPayload = getEncodedPayloadFromMultipartDTO(multipartFormDataDTO);
/*
* - For files uploaded using Filepicker.xyz.base64, body format is "<content-type>;base64,<actual-
* base64-encoded-payload>".
@ -353,18 +347,11 @@ public class AmazonS3Plugin extends BasePlugin {
);
}
} else {
payload = ((String) multipartFormDataDTO.getData()).getBytes();
payload = getEncodedPayloadFromMultipartDTO(multipartFormDataDTO).getBytes();
}
InputStream inputStream = new ByteArrayInputStream(payload);
TransferManager transferManager = TransferManagerBuilder.standard().withS3Client(connection).build();
final ObjectMetadata objectMetadata = new ObjectMetadata();
// Only add content type if the user has mentioned it in the body
if (multipartFormDataDTO.getType() != null) {
objectMetadata.setContentType(multipartFormDataDTO.getType());
}
try {
transferManager.upload(bucketName, filePath, inputStream, objectMetadata).waitForUploadResult();
uploadFileInS3(payload, connection, multipartFormDataDTO, bucketName, filePath);
} catch (InterruptedException e) {
throw new AppsmithPluginException(
AppsmithPluginError.PLUGIN_ERROR,
@ -1067,5 +1054,27 @@ public class AmazonS3Plugin extends BasePlugin {
return DataTypeStringUtils.jsonSmartReplacementPlaceholderWithValue(jsonBody, value, null, insertedParams, null);
}
private String getEncodedPayloadFromMultipartDTO(MultipartFormDataDTO multipartFormDataDTO) {
String encodedPayload;
if (multipartFormDataDTO.getData() instanceof LinkedHashMap) {
encodedPayload = ((LinkedHashMap<?, ?>) multipartFormDataDTO.getData()).get("data").toString();
} else {
encodedPayload = (String) multipartFormDataDTO.getData();
}
return encodedPayload;
}
void uploadFileInS3(byte[] payload, AmazonS3 connection, MultipartFormDataDTO multipartFormDataDTO,
String bucketName, String path) throws InterruptedException {
InputStream inputStream = new ByteArrayInputStream(payload);
TransferManager transferManager = TransferManagerBuilder.standard().withS3Client(connection).build();
final ObjectMetadata objectMetadata = new ObjectMetadata();
// Only add content type if the user has mentioned it in the body
if (multipartFormDataDTO.getType() != null) {
objectMetadata.setContentType(multipartFormDataDTO.getType());
}
transferManager.upload(bucketName, path, inputStream, objectMetadata).waitForUploadResult();
}
}
}

170
app/server/appsmith-plugins/amazons3Plugin/src/test/java/com/external/plugins/AmazonS3PluginTest.java vendored
View File

@ -82,9 +82,7 @@ import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import static org.mockito.Mockito.*;
@Slf4j
public class AmazonS3PluginTest {
@ -1283,4 +1281,170 @@ public class AmazonS3PluginTest {
assertEquals(actionExecutionResult.getReadableError(),errorCode+": "+errorMessage);
}
@Test
public void uploadsSingleFileWithFilePicker() throws InterruptedException {
DatasourceConfiguration datasourceConfiguration = createDatasourceConfiguration();
ExecuteActionDTO executeActionDTO = new ExecuteActionDTO();
AmazonS3Plugin.S3PluginExecutor spyS3PluginExecutor = spy(AmazonS3Plugin.S3PluginExecutor.class);
ActionConfiguration actionConfiguration = new ActionConfiguration();
Map<String, Object> configMap = new HashMap<>();
setDataValueSafelyInFormData(configMap, BODY, "{\n" +
"\t\"type\":\"text/plain\",\n" +
"\t\"data\": \"data:text/plain;base64,SGVsbG8gV29ybGQhCg==\"\n" +
"}");
setDataValueSafelyInFormData(configMap, PATH, "path");
setDataValueSafelyInFormData(configMap, COMMAND, "UPLOAD_FILE_FROM_BODY");
setDataValueSafelyInFormData(configMap, BUCKET, "bucket_name");
setDataValueSafelyInFormData(configMap, CREATE_DATATYPE, "YES");
setDataValueSafelyInFormData(configMap, CREATE_EXPIRY, "100000");
actionConfiguration.setFormData(configMap);
AmazonS3 connection = spyS3PluginExecutor.datasourceCreate(datasourceConfiguration).block();
ArrayList<String> signedURLS = new ArrayList<>();
signedURLS.add("https://example.signed.url");
doNothing().when(spyS3PluginExecutor).uploadFileInS3(any(),any(),any(),anyString(),anyString());
doReturn(signedURLS).when(spyS3PluginExecutor).getSignedUrls(any(), anyString(),any(), any());
Mono<ActionExecutionResult> resultMono = spyS3PluginExecutor.executeParameterized(
connection,
executeActionDTO,
datasourceConfiguration,
actionConfiguration);
StepVerifier.create(resultMono)
.assertNext(result -> {
assertTrue(result.getIsExecutionSuccess());
assertEquals(((HashMap) result.getBody()).get("signedUrl"), signedURLS.get(0));
})
.verifyComplete();
}
@Test
public void uploadsMultipleFilesWithFilePicker() throws InterruptedException {
DatasourceConfiguration datasourceConfiguration = createDatasourceConfiguration();
ExecuteActionDTO executeActionDTO = new ExecuteActionDTO();
AmazonS3Plugin.S3PluginExecutor spyS3PluginExecutor = spy(AmazonS3Plugin.S3PluginExecutor.class);
ActionConfiguration actionConfiguration = new ActionConfiguration();
Map<String, Object> configMap = new HashMap<>();
setDataValueSafelyInFormData(configMap, BODY, "[{\"data\":\"data:application/json;base64,ewogICAgIm1vc3QiOiAic2ltcGxlIgp9Cg==\",\"size\":25,\"dataFormat\":\"Base64\",\"name\":\"testfile.json\",\"id\":\"uppy-testfile/json-1e-application/json-25-1661283894345\",\"type\":\"application/json\"},{\"data\":\"data:text/plain;base64,SGVsbG8gV29ybGQhCg==\",\"size\":13,\"dataFormat\":\"Base64\",\"name\":\"testFile.txt\",\"id\":\"uppy-testfile/txt-1e-text/plain-13-1659676685242\",\"type\":\"text/plain\"}]");
setDataValueSafelyInFormData(configMap, PATH, "path");
setDataValueSafelyInFormData(configMap, COMMAND, "UPLOAD_MULTIPLE_FILES_FROM_BODY");
setDataValueSafelyInFormData(configMap, BUCKET, "bucket_name");
setDataValueSafelyInFormData(configMap, CREATE_DATATYPE, "YES");
setDataValueSafelyInFormData(configMap, CREATE_EXPIRY, "100000");
actionConfiguration.setFormData(configMap);
AmazonS3 connection = spyS3PluginExecutor.datasourceCreate(datasourceConfiguration).block();
ArrayList<String> signedURLS = new ArrayList<>();
signedURLS.add("https://example.signed.url1");
signedURLS.add("https://example.signed.url2");
doNothing().when(spyS3PluginExecutor).uploadFileInS3(any(),any(),any(),anyString(),anyString());
doReturn(signedURLS).when(spyS3PluginExecutor).getSignedUrls(any(), anyString(),any(), any());
Mono<ActionExecutionResult> resultMono = spyS3PluginExecutor.executeParameterized(
connection,
executeActionDTO,
datasourceConfiguration,
actionConfiguration);
StepVerifier.create(resultMono)
.assertNext(result -> {
assertTrue(result.getIsExecutionSuccess());
ArrayList<String> x = (ArrayList<String>) ((HashMap) result.getBody()).get("signedUrls");
assertEquals(x.size() , signedURLS.size());
assertEquals(x.get(0), signedURLS.get(0));
assertEquals(x.get(1), signedURLS.get(1));
})
.verifyComplete();
}
@Test
public void uploadsSingleFileWithoutFilePicker() throws InterruptedException {
DatasourceConfiguration datasourceConfiguration = createDatasourceConfiguration();
ExecuteActionDTO executeActionDTO = new ExecuteActionDTO();
AmazonS3Plugin.S3PluginExecutor spyS3PluginExecutor = spy(AmazonS3Plugin.S3PluginExecutor.class);
ActionConfiguration actionConfiguration = new ActionConfiguration();
Map<String, Object> configMap = new HashMap<>();
setDataValueSafelyInFormData(configMap, BODY, "{\n" +
" \"type\": \"text/json\",\n" +
" \"data\": {\"data\":\"{\\n \\\"most\\\": \\\"simple\\\"\\n}\\n\",\"size\":25,\"dataFormat\":\"Text\",\"name\":\"testfile.json\",\"id\":\"uppy-testfile/json-1e-application/json-25-1661283894345\",\"type\":\"application/json\"}\n" +
"}");
setDataValueSafelyInFormData(configMap, PATH, "path");
setDataValueSafelyInFormData(configMap, COMMAND, "UPLOAD_FILE_FROM_BODY");
setDataValueSafelyInFormData(configMap, BUCKET, "bucket_name");
setDataValueSafelyInFormData(configMap, CREATE_DATATYPE, "NO");
setDataValueSafelyInFormData(configMap, CREATE_EXPIRY, "100000");
actionConfiguration.setFormData(configMap);
AmazonS3 connection = spyS3PluginExecutor.datasourceCreate(datasourceConfiguration).block();
ArrayList<String> signedURLS = new ArrayList<>();
signedURLS.add("https://example.signed.url");
doNothing().when(spyS3PluginExecutor).uploadFileInS3(any(),any(),any(),anyString(),anyString());
doReturn(signedURLS).when(spyS3PluginExecutor).getSignedUrls(any(), anyString(),any(), any());
Mono<ActionExecutionResult> resultMono = spyS3PluginExecutor.executeParameterized(
connection,
executeActionDTO,
datasourceConfiguration,
actionConfiguration);
StepVerifier.create(resultMono)
.assertNext(result -> {
assertTrue(result.getIsExecutionSuccess());
assertEquals(((HashMap) result.getBody()).get("signedUrl"), signedURLS.get(0));
})
.verifyComplete();
}
@Test
public void uploadsMultipleFilesWithoutFilePicker() throws InterruptedException {
DatasourceConfiguration datasourceConfiguration = createDatasourceConfiguration();
ExecuteActionDTO executeActionDTO = new ExecuteActionDTO();
AmazonS3Plugin.S3PluginExecutor spyS3PluginExecutor = spy(AmazonS3Plugin.S3PluginExecutor.class);
ActionConfiguration actionConfiguration = new ActionConfiguration();
Map<String, Object> configMap = new HashMap<>();
setDataValueSafelyInFormData(configMap, BODY, "[{\"data\":\"data:application/json;base64,ewogICAgIm1vc3QiOiAic2ltcGxlIgp9Cg==\",\"size\":25,\"dataFormat\":\"Base64\",\"name\":\"testfile.json\",\"id\":\"uppy-testfile/json-1e-application/json-25-1661283894345\",\"type\":\"application/json\"},{\"data\":\"data:text/plain;base64,SGVsbG8gV29ybGQhCg==\",\"size\":13,\"dataFormat\":\"Base64\",\"name\":\"testFile.txt\",\"id\":\"uppy-testfile/txt-1e-text/plain-13-1659676685242\",\"type\":\"text/plain\"}]");
setDataValueSafelyInFormData(configMap, PATH, "path");
setDataValueSafelyInFormData(configMap, COMMAND, "UPLOAD_MULTIPLE_FILES_FROM_BODY");
setDataValueSafelyInFormData(configMap, BUCKET, "bucket_name");
setDataValueSafelyInFormData(configMap, CREATE_DATATYPE, "NO");
setDataValueSafelyInFormData(configMap, CREATE_EXPIRY, "100000");
actionConfiguration.setFormData(configMap);
AmazonS3 connection = spyS3PluginExecutor.datasourceCreate(datasourceConfiguration).block();
ArrayList<String> signedURLS = new ArrayList<>();
signedURLS.add("https://example.signed.url1");
signedURLS.add("https://example.signed.url2");
doNothing().when(spyS3PluginExecutor).uploadFileInS3(any(),any(),any(),anyString(),anyString());
doReturn(signedURLS).when(spyS3PluginExecutor).getSignedUrls(any(), anyString(),any(), any());
Mono<ActionExecutionResult> resultMono = spyS3PluginExecutor.executeParameterized(
connection,
executeActionDTO,
datasourceConfiguration,
actionConfiguration);
StepVerifier.create(resultMono)
.assertNext(result -> {
assertTrue(result.getIsExecutionSuccess());
ArrayList<String> x = (ArrayList<String>) ((HashMap) result.getBody()).get("signedUrls");
assertEquals(x.size() , signedURLS.size());
assertEquals(x.get(0), signedURLS.get(0));
assertEquals(x.get(1), signedURLS.get(1));
})
.verifyComplete();
}
}