diff --git a/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/plugins/AmazonS3Plugin.java b/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/plugins/AmazonS3Plugin.java index 8c8f39e05c..c4e4632707 100644 --- a/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/plugins/AmazonS3Plugin.java +++ b/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/plugins/AmazonS3Plugin.java @@ -68,6 +68,7 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; +import java.util.LinkedHashMap; import static com.appsmith.external.constants.ActionConstants.ACTION_CONFIGURATION_BODY; import static com.appsmith.external.constants.ActionConstants.ACTION_CONFIGURATION_PATH; @@ -253,7 +254,8 @@ public class AmazonS3Plugin extends BasePlugin { } if (Boolean.TRUE.equals(usingFilePicker)) { - String encodedPayload = (String) multipartFormDataDTO.getData(); + String encodedPayload = getEncodedPayloadFromMultipartDTO(multipartFormDataDTO); + /* * - For files uploaded using Filepicker.xyz.base64, body format is ";base64,". @@ -274,25 +276,17 @@ public class AmazonS3Plugin extends BasePlugin { ); } } else { - payload = ((String) multipartFormDataDTO.getData()).getBytes(); + payload = getEncodedPayloadFromMultipartDTO(multipartFormDataDTO).getBytes(); } - InputStream inputStream = new ByteArrayInputStream(payload); - TransferManager transferManager = TransferManagerBuilder.standard().withS3Client(connection).build(); - final ObjectMetadata objectMetadata = new ObjectMetadata(); - // Only add content type if the user has mentioned it in the body - if (multipartFormDataDTO.getType() != null) { - objectMetadata.setContentType(multipartFormDataDTO.getType()); - } - transferManager.upload(bucketName, path, inputStream, objectMetadata).waitForUploadResult(); - + uploadFileInS3(payload, connection, multipartFormDataDTO, bucketName, path); ArrayList listOfFiles = new ArrayList<>(); listOfFiles.add(path); ArrayList listOfUrls = getSignedUrls(connection, bucketName, listOfFiles, expiryDateTime); if (listOfUrls.size() != 1) { throw new AppsmithPluginException( AppsmithPluginError.PLUGIN_ERROR, - "Appsmith has encountered an unexpected error when fetching url from AmazonS3 after file " + + "Appsmith has encountered an unexpected error while fetching url from AmazonS3 after file " + "creation. Please reach out to Appsmith customer support to resolve this." ); } @@ -311,7 +305,7 @@ public class AmazonS3Plugin extends BasePlugin { String body, Boolean usingFilePicker, Date expiryDateTime) - throws InterruptedException, AppsmithPluginException { + throws AppsmithPluginException { List multipartFormDataDTOs; @@ -332,7 +326,7 @@ public class AmazonS3Plugin extends BasePlugin { byte[] payload; if (Boolean.TRUE.equals(usingFilePicker)) { - String encodedPayload = (String) multipartFormDataDTO.getData(); + String encodedPayload = getEncodedPayloadFromMultipartDTO(multipartFormDataDTO); /* * - For files uploaded using Filepicker.xyz.base64, body format is ";base64,". @@ -353,18 +347,11 @@ public class AmazonS3Plugin extends BasePlugin { ); } } else { - payload = ((String) multipartFormDataDTO.getData()).getBytes(); + payload = getEncodedPayloadFromMultipartDTO(multipartFormDataDTO).getBytes(); } - InputStream inputStream = new ByteArrayInputStream(payload); - TransferManager transferManager = TransferManagerBuilder.standard().withS3Client(connection).build(); - final ObjectMetadata objectMetadata = new ObjectMetadata(); - // Only add content type if the user has mentioned it in the body - if (multipartFormDataDTO.getType() != null) { - objectMetadata.setContentType(multipartFormDataDTO.getType()); - } try { - transferManager.upload(bucketName, filePath, inputStream, objectMetadata).waitForUploadResult(); + uploadFileInS3(payload, connection, multipartFormDataDTO, bucketName, filePath); } catch (InterruptedException e) { throw new AppsmithPluginException( AppsmithPluginError.PLUGIN_ERROR, @@ -1067,5 +1054,27 @@ public class AmazonS3Plugin extends BasePlugin { return DataTypeStringUtils.jsonSmartReplacementPlaceholderWithValue(jsonBody, value, null, insertedParams, null); } + private String getEncodedPayloadFromMultipartDTO(MultipartFormDataDTO multipartFormDataDTO) { + String encodedPayload; + if (multipartFormDataDTO.getData() instanceof LinkedHashMap) { + encodedPayload = ((LinkedHashMap) multipartFormDataDTO.getData()).get("data").toString(); + } else { + encodedPayload = (String) multipartFormDataDTO.getData(); + } + return encodedPayload; + } + + void uploadFileInS3(byte[] payload, AmazonS3 connection, MultipartFormDataDTO multipartFormDataDTO, + String bucketName, String path) throws InterruptedException { + InputStream inputStream = new ByteArrayInputStream(payload); + TransferManager transferManager = TransferManagerBuilder.standard().withS3Client(connection).build(); + final ObjectMetadata objectMetadata = new ObjectMetadata(); + // Only add content type if the user has mentioned it in the body + if (multipartFormDataDTO.getType() != null) { + objectMetadata.setContentType(multipartFormDataDTO.getType()); + } + transferManager.upload(bucketName, path, inputStream, objectMetadata).waitForUploadResult(); + } + } } diff --git a/app/server/appsmith-plugins/amazons3Plugin/src/test/java/com/external/plugins/AmazonS3PluginTest.java b/app/server/appsmith-plugins/amazons3Plugin/src/test/java/com/external/plugins/AmazonS3PluginTest.java index b9a72e06c2..c7026bd153 100644 --- a/app/server/appsmith-plugins/amazons3Plugin/src/test/java/com/external/plugins/AmazonS3PluginTest.java +++ b/app/server/appsmith-plugins/amazons3Plugin/src/test/java/com/external/plugins/AmazonS3PluginTest.java @@ -82,9 +82,7 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.doNothing; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; +import static org.mockito.Mockito.*; @Slf4j public class AmazonS3PluginTest { @@ -1283,4 +1281,170 @@ public class AmazonS3PluginTest { assertEquals(actionExecutionResult.getReadableError(),errorCode+": "+errorMessage); } + + @Test + public void uploadsSingleFileWithFilePicker() throws InterruptedException { + DatasourceConfiguration datasourceConfiguration = createDatasourceConfiguration(); + ExecuteActionDTO executeActionDTO = new ExecuteActionDTO(); + AmazonS3Plugin.S3PluginExecutor spyS3PluginExecutor = spy(AmazonS3Plugin.S3PluginExecutor.class); + + ActionConfiguration actionConfiguration = new ActionConfiguration(); + + Map configMap = new HashMap<>(); + setDataValueSafelyInFormData(configMap, BODY, "{\n" + + "\t\"type\":\"text/plain\",\n" + + "\t\"data\": \"data:text/plain;base64,SGVsbG8gV29ybGQhCg==\"\n" + + "}"); + setDataValueSafelyInFormData(configMap, PATH, "path"); + setDataValueSafelyInFormData(configMap, COMMAND, "UPLOAD_FILE_FROM_BODY"); + setDataValueSafelyInFormData(configMap, BUCKET, "bucket_name"); + setDataValueSafelyInFormData(configMap, CREATE_DATATYPE, "YES"); + setDataValueSafelyInFormData(configMap, CREATE_EXPIRY, "100000"); + + actionConfiguration.setFormData(configMap); + + AmazonS3 connection = spyS3PluginExecutor.datasourceCreate(datasourceConfiguration).block(); + ArrayList signedURLS = new ArrayList<>(); + signedURLS.add("https://example.signed.url"); + doNothing().when(spyS3PluginExecutor).uploadFileInS3(any(),any(),any(),anyString(),anyString()); + doReturn(signedURLS).when(spyS3PluginExecutor).getSignedUrls(any(), anyString(),any(), any()); + Mono resultMono = spyS3PluginExecutor.executeParameterized( + connection, + executeActionDTO, + datasourceConfiguration, + actionConfiguration); + + StepVerifier.create(resultMono) + .assertNext(result -> { + assertTrue(result.getIsExecutionSuccess()); + assertEquals(((HashMap) result.getBody()).get("signedUrl"), signedURLS.get(0)); + + }) + .verifyComplete(); + } + + @Test + public void uploadsMultipleFilesWithFilePicker() throws InterruptedException { + DatasourceConfiguration datasourceConfiguration = createDatasourceConfiguration(); + ExecuteActionDTO executeActionDTO = new ExecuteActionDTO(); + AmazonS3Plugin.S3PluginExecutor spyS3PluginExecutor = spy(AmazonS3Plugin.S3PluginExecutor.class); + + ActionConfiguration actionConfiguration = new ActionConfiguration(); + + Map configMap = new HashMap<>(); + setDataValueSafelyInFormData(configMap, BODY, "[{\"data\":\"data:application/json;base64,ewogICAgIm1vc3QiOiAic2ltcGxlIgp9Cg==\",\"size\":25,\"dataFormat\":\"Base64\",\"name\":\"testfile.json\",\"id\":\"uppy-testfile/json-1e-application/json-25-1661283894345\",\"type\":\"application/json\"},{\"data\":\"data:text/plain;base64,SGVsbG8gV29ybGQhCg==\",\"size\":13,\"dataFormat\":\"Base64\",\"name\":\"testFile.txt\",\"id\":\"uppy-testfile/txt-1e-text/plain-13-1659676685242\",\"type\":\"text/plain\"}]"); + setDataValueSafelyInFormData(configMap, PATH, "path"); + setDataValueSafelyInFormData(configMap, COMMAND, "UPLOAD_MULTIPLE_FILES_FROM_BODY"); + setDataValueSafelyInFormData(configMap, BUCKET, "bucket_name"); + setDataValueSafelyInFormData(configMap, CREATE_DATATYPE, "YES"); + setDataValueSafelyInFormData(configMap, CREATE_EXPIRY, "100000"); + + actionConfiguration.setFormData(configMap); + + AmazonS3 connection = spyS3PluginExecutor.datasourceCreate(datasourceConfiguration).block(); + ArrayList signedURLS = new ArrayList<>(); + signedURLS.add("https://example.signed.url1"); + signedURLS.add("https://example.signed.url2"); + doNothing().when(spyS3PluginExecutor).uploadFileInS3(any(),any(),any(),anyString(),anyString()); + doReturn(signedURLS).when(spyS3PluginExecutor).getSignedUrls(any(), anyString(),any(), any()); + Mono resultMono = spyS3PluginExecutor.executeParameterized( + connection, + executeActionDTO, + datasourceConfiguration, + actionConfiguration); + + StepVerifier.create(resultMono) + .assertNext(result -> { + assertTrue(result.getIsExecutionSuccess()); + ArrayList x = (ArrayList) ((HashMap) result.getBody()).get("signedUrls"); + assertEquals(x.size() , signedURLS.size()); + assertEquals(x.get(0), signedURLS.get(0)); + assertEquals(x.get(1), signedURLS.get(1)); + + }) + .verifyComplete(); + } + + @Test + public void uploadsSingleFileWithoutFilePicker() throws InterruptedException { + DatasourceConfiguration datasourceConfiguration = createDatasourceConfiguration(); + ExecuteActionDTO executeActionDTO = new ExecuteActionDTO(); + AmazonS3Plugin.S3PluginExecutor spyS3PluginExecutor = spy(AmazonS3Plugin.S3PluginExecutor.class); + + ActionConfiguration actionConfiguration = new ActionConfiguration(); + + Map configMap = new HashMap<>(); + setDataValueSafelyInFormData(configMap, BODY, "{\n" + + " \"type\": \"text/json\",\n" + + " \"data\": {\"data\":\"{\\n \\\"most\\\": \\\"simple\\\"\\n}\\n\",\"size\":25,\"dataFormat\":\"Text\",\"name\":\"testfile.json\",\"id\":\"uppy-testfile/json-1e-application/json-25-1661283894345\",\"type\":\"application/json\"}\n" + + "}"); + setDataValueSafelyInFormData(configMap, PATH, "path"); + setDataValueSafelyInFormData(configMap, COMMAND, "UPLOAD_FILE_FROM_BODY"); + setDataValueSafelyInFormData(configMap, BUCKET, "bucket_name"); + setDataValueSafelyInFormData(configMap, CREATE_DATATYPE, "NO"); + setDataValueSafelyInFormData(configMap, CREATE_EXPIRY, "100000"); + + actionConfiguration.setFormData(configMap); + + AmazonS3 connection = spyS3PluginExecutor.datasourceCreate(datasourceConfiguration).block(); + ArrayList signedURLS = new ArrayList<>(); + signedURLS.add("https://example.signed.url"); + doNothing().when(spyS3PluginExecutor).uploadFileInS3(any(),any(),any(),anyString(),anyString()); + doReturn(signedURLS).when(spyS3PluginExecutor).getSignedUrls(any(), anyString(),any(), any()); + Mono resultMono = spyS3PluginExecutor.executeParameterized( + connection, + executeActionDTO, + datasourceConfiguration, + actionConfiguration); + + StepVerifier.create(resultMono) + .assertNext(result -> { + assertTrue(result.getIsExecutionSuccess()); + assertEquals(((HashMap) result.getBody()).get("signedUrl"), signedURLS.get(0)); + + }) + .verifyComplete(); + } + + @Test + public void uploadsMultipleFilesWithoutFilePicker() throws InterruptedException { + DatasourceConfiguration datasourceConfiguration = createDatasourceConfiguration(); + ExecuteActionDTO executeActionDTO = new ExecuteActionDTO(); + AmazonS3Plugin.S3PluginExecutor spyS3PluginExecutor = spy(AmazonS3Plugin.S3PluginExecutor.class); + + ActionConfiguration actionConfiguration = new ActionConfiguration(); + + Map configMap = new HashMap<>(); + setDataValueSafelyInFormData(configMap, BODY, "[{\"data\":\"data:application/json;base64,ewogICAgIm1vc3QiOiAic2ltcGxlIgp9Cg==\",\"size\":25,\"dataFormat\":\"Base64\",\"name\":\"testfile.json\",\"id\":\"uppy-testfile/json-1e-application/json-25-1661283894345\",\"type\":\"application/json\"},{\"data\":\"data:text/plain;base64,SGVsbG8gV29ybGQhCg==\",\"size\":13,\"dataFormat\":\"Base64\",\"name\":\"testFile.txt\",\"id\":\"uppy-testfile/txt-1e-text/plain-13-1659676685242\",\"type\":\"text/plain\"}]"); + setDataValueSafelyInFormData(configMap, PATH, "path"); + setDataValueSafelyInFormData(configMap, COMMAND, "UPLOAD_MULTIPLE_FILES_FROM_BODY"); + setDataValueSafelyInFormData(configMap, BUCKET, "bucket_name"); + setDataValueSafelyInFormData(configMap, CREATE_DATATYPE, "NO"); + setDataValueSafelyInFormData(configMap, CREATE_EXPIRY, "100000"); + + actionConfiguration.setFormData(configMap); + + AmazonS3 connection = spyS3PluginExecutor.datasourceCreate(datasourceConfiguration).block(); + ArrayList signedURLS = new ArrayList<>(); + signedURLS.add("https://example.signed.url1"); + signedURLS.add("https://example.signed.url2"); + doNothing().when(spyS3PluginExecutor).uploadFileInS3(any(),any(),any(),anyString(),anyString()); + doReturn(signedURLS).when(spyS3PluginExecutor).getSignedUrls(any(), anyString(),any(), any()); + Mono resultMono = spyS3PluginExecutor.executeParameterized( + connection, + executeActionDTO, + datasourceConfiguration, + actionConfiguration); + + StepVerifier.create(resultMono) + .assertNext(result -> { + assertTrue(result.getIsExecutionSuccess()); + ArrayList x = (ArrayList) ((HashMap) result.getBody()).get("signedUrls"); + assertEquals(x.size() , signedURLS.size()); + assertEquals(x.get(0), signedURLS.get(0)); + assertEquals(x.get(1), signedURLS.get(1)); + + }) + .verifyComplete(); + } }