From 1f524827b9f0d0c63cff3127506cb5a7a7feff75 Mon Sep 17 00:00:00 2001 From: Shrikant Kandula Date: Wed, 1 Apr 2020 08:50:36 +0000 Subject: [PATCH] Datasource CRUD APIs --- .../external/models/AuthenticationDTO.java | 16 +++- .../appsmith/external/models/Connection.java | 29 ++++++ .../models/DatasourceConfiguration.java | 14 +-- .../appsmith/external/models/Endpoint.java | 20 ++++ .../external/models/PEMCertificate.java | 20 ++++ .../external/models/SSHConnection.java | 26 ++++++ .../external/models/SSHPrivateKey.java | 18 ++++ .../appsmith/external/models/SSLDetails.java | 32 +++++++ .../com/external/plugins/MongoPlugin.java | 3 +- .../server/helpers/BeanCopyUtils.java | 46 +++++++++ .../services/DatasourceServiceImpl.java | 23 +++-- .../services/DatasourceServiceTest.java | 52 +++++++++++ .../server/utils/BeanCopyUtilsTest.java | 93 +++++++++++++++++++ 13 files changed, 373 insertions(+), 19 deletions(-) create mode 100644 app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/Connection.java create mode 100644 app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/Endpoint.java create mode 100644 app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/PEMCertificate.java create mode 100644 app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/SSHConnection.java create mode 100644 app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/SSHPrivateKey.java create mode 100644 app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/SSLDetails.java create mode 100644 app/server/appsmith-server/src/test/java/com/appsmith/server/utils/BeanCopyUtilsTest.java diff --git a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/AuthenticationDTO.java b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/AuthenticationDTO.java index a1a92dc83d..e679ae347b 100644 --- a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/AuthenticationDTO.java +++ b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/AuthenticationDTO.java @@ -1,5 +1,6 @@ package com.appsmith.external.models; +import com.fasterxml.jackson.annotation.JsonProperty; import lombok.AllArgsConstructor; import lombok.Getter; import lombok.NoArgsConstructor; @@ -12,7 +13,18 @@ import lombok.ToString; @NoArgsConstructor @AllArgsConstructor public class AuthenticationDTO { - String authType; + + public enum Type { + SCRAM_SHA_1, SCRAM_SHA_256, MONGODB_CR, USERNAME_PASSWORD + } + + Type authType; + String username; + + @JsonProperty(access = JsonProperty.Access.WRITE_ONLY) String password; -} \ No newline at end of file + + String databaseName; + +} diff --git a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/Connection.java b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/Connection.java new file mode 100644 index 0000000000..3dbe060210 --- /dev/null +++ b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/Connection.java @@ -0,0 +1,29 @@ +package com.appsmith.external.models; + +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.ToString; +import org.springframework.data.mongodb.core.mapping.Document; + +@Getter +@Setter +@ToString +@NoArgsConstructor +@Document +public class Connection { + + public enum Mode { + ReadOnly, ReadWrite + } + + public enum Type { + DirectConnection, ReplicaSet + } + + Mode mode; + + Type type; + + SSLDetails ssl; +} diff --git a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/DatasourceConfiguration.java b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/DatasourceConfiguration.java index 2220813ed6..055f8c6b92 100644 --- a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/DatasourceConfiguration.java +++ b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/DatasourceConfiguration.java @@ -15,17 +15,19 @@ import java.util.List; @Document public class DatasourceConfiguration { - String url; + Connection connection; + + List endpoints; AuthenticationDTO authentication; + SSHConnection sshProxy; + List properties; - //For REST API + // For REST API. + String url; + List headers; - //This field is for plugins which allow the database name to be specified outside of the connection URL. The - //expectation from the plugins is that if the database name has been provided via this field, the database name - //should be according to this, else if database name is null, pick the database name from the URL. - String databaseName; } diff --git a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/Endpoint.java b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/Endpoint.java new file mode 100644 index 0000000000..53d76d7f1d --- /dev/null +++ b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/Endpoint.java @@ -0,0 +1,20 @@ +package com.appsmith.external.models; + +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.ToString; +import org.springframework.data.mongodb.core.mapping.Document; + +@Getter +@Setter +@ToString +@NoArgsConstructor +@Document +public class Endpoint { + + String host; + + Long port; + +} diff --git a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/PEMCertificate.java b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/PEMCertificate.java new file mode 100644 index 0000000000..5ffdef5264 --- /dev/null +++ b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/PEMCertificate.java @@ -0,0 +1,20 @@ +package com.appsmith.external.models; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import org.springframework.data.mongodb.core.mapping.Document; + +@Getter +@Setter +@ToString +@Document +public class PEMCertificate { + + String file; + + @JsonProperty(access = JsonProperty.Access.WRITE_ONLY) + String password; + +} diff --git a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/SSHConnection.java b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/SSHConnection.java new file mode 100644 index 0000000000..cee3b7e916 --- /dev/null +++ b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/SSHConnection.java @@ -0,0 +1,26 @@ +package com.appsmith.external.models; + +import lombok.*; + +@Getter +@Setter +@ToString +@NoArgsConstructor +@AllArgsConstructor +public class SSHConnection { + + public enum AuthType { + PrivateKey, Password + } + + String host; + + Long port; + + String username; + + AuthType authType; + + SSHPrivateKey privateKey; + +} diff --git a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/SSHPrivateKey.java b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/SSHPrivateKey.java new file mode 100644 index 0000000000..c219c3dde8 --- /dev/null +++ b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/SSHPrivateKey.java @@ -0,0 +1,18 @@ +package com.appsmith.external.models; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.*; + +@Getter +@Setter +@ToString +@NoArgsConstructor +@AllArgsConstructor +public class SSHPrivateKey { + + String keyFile; + + @JsonProperty(access = JsonProperty.Access.WRITE_ONLY) + String password; + +} diff --git a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/SSLDetails.java b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/SSLDetails.java new file mode 100644 index 0000000000..419853dc11 --- /dev/null +++ b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/SSLDetails.java @@ -0,0 +1,32 @@ +package com.appsmith.external.models; + +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.ToString; +import org.springframework.data.mongodb.core.mapping.Document; + +@Getter +@Setter +@ToString +@NoArgsConstructor +@Document +public class SSLDetails { + + public enum AuthType { + CACertificate, SelfSignedCertificate + } + + AuthType authType; + + String keyFile; + + String certificateFile; + + String caCertificateFile; + + Boolean usePemCertificate; + + PEMCertificate pemCertificate; + +} diff --git a/app/server/appsmith-plugins/mongoPlugin/src/main/java/com/external/plugins/MongoPlugin.java b/app/server/appsmith-plugins/mongoPlugin/src/main/java/com/external/plugins/MongoPlugin.java index f55214cf02..9f56a58c4a 100644 --- a/app/server/appsmith-plugins/mongoPlugin/src/main/java/com/external/plugins/MongoPlugin.java +++ b/app/server/appsmith-plugins/mongoPlugin/src/main/java/com/external/plugins/MongoPlugin.java @@ -57,7 +57,8 @@ public class MongoPlugin extends BasePlugin { MongoClientURI mongoClientURI = new MongoClientURI(datasourceConfiguration.getUrl()); - String databaseName = datasourceConfiguration.getDatabaseName(); + // CONFIRM: Verify that the database name correctly shows up here. + String databaseName = datasourceConfiguration.getAuthentication().getDatabaseName(); if (databaseName == null) { databaseName = mongoClientURI.getDatabase(); } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/BeanCopyUtils.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/BeanCopyUtils.java index d0b0cd12ac..1be3c2c4ba 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/BeanCopyUtils.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/BeanCopyUtils.java @@ -3,13 +3,19 @@ package com.appsmith.server.helpers; import org.springframework.beans.BeanUtils; import org.springframework.beans.BeanWrapper; import org.springframework.beans.BeanWrapperImpl; +import org.springframework.beans.PropertyAccessorFactory; +import java.beans.PropertyDescriptor; +import java.util.ArrayList; import java.util.HashSet; +import java.util.List; import java.util.Set; public final class BeanCopyUtils { private static String[] getNullPropertyNames(Object source) { + // TODO: The `BeanWrapperImpl` class has been declared to be an internal class. Migrate to using + // `PropertyAccessorFactory.forBeanPropertyAccess` instead. final BeanWrapper src = new BeanWrapperImpl(source); java.beans.PropertyDescriptor[] pds = src.getPropertyDescriptors(); @@ -43,4 +49,44 @@ public final class BeanCopyUtils { return count++; } + + public static void copyNestedNonNullProperties(Object source, Object target) { + if (source == null || target == null) { + return; + } + + final BeanWrapper sourceBeanWrapper = PropertyAccessorFactory.forBeanPropertyAccess(source); + BeanWrapper targetBeanWrapper = PropertyAccessorFactory.forBeanPropertyAccess(target); + PropertyDescriptor[] propertyDescriptors = sourceBeanWrapper.getPropertyDescriptors(); + + for (PropertyDescriptor propertyDescriptor : propertyDescriptors) { + String name = propertyDescriptor.getName(); + + // For properties like `class` that don't have a set method, we can't copy so we just ignore them. + if (targetBeanWrapper.getPropertyDescriptor(name).getWriteMethod() == null) { + continue; + } + + Object sourceValue = sourceBeanWrapper.getPropertyValue(name); + + // If sourceValue is null, don't copy it over to target and just move on to the next property. + if (sourceValue == null) { + continue; + } + + Object targetValue = targetBeanWrapper.getPropertyValue(name); + + if (targetValue != null && isDomainModel(propertyDescriptor.getPropertyType())) { + // Go deeper *only* if the property belongs to Appsmith's models, and both the source and target values + // are not null. + copyNestedNonNullProperties(sourceValue, targetValue); + } else { + targetBeanWrapper.setPropertyValue(name, sourceValue); + } + } + } + + public static boolean isDomainModel(Class type) { + return type.getPackageName().startsWith("com.appsmith."); + } } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/DatasourceServiceImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/DatasourceServiceImpl.java index fc0e0f682f..83dbab57c6 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/DatasourceServiceImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/DatasourceServiceImpl.java @@ -1,5 +1,6 @@ package com.appsmith.server.services; +import com.appsmith.external.models.DatasourceConfiguration; import com.appsmith.external.plugins.PluginExecutor; import com.appsmith.server.constants.FieldName; import com.appsmith.server.domains.Datasource; @@ -17,6 +18,7 @@ import org.springframework.data.mongodb.core.ReactiveMongoTemplate; import org.springframework.data.mongodb.core.convert.MongoConverter; import org.springframework.stereotype.Service; import org.springframework.util.MultiValueMap; +import org.springframework.util.StringUtils; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import reactor.core.scheduler.Scheduler; @@ -26,7 +28,7 @@ import javax.validation.constraints.NotNull; import java.util.HashSet; import java.util.Set; -import static com.appsmith.server.helpers.BeanCopyUtils.copyNewFieldValuesIntoOldObject; +import static com.appsmith.server.helpers.BeanCopyUtils.copyNestedNonNullProperties; import static com.appsmith.server.helpers.MustacheHelper.extractMustacheKeys; @Slf4j @@ -81,7 +83,7 @@ public class DatasourceServiceImpl extends BaseService { - copyNewFieldValuesIntoOldObject(datasource, dbDatasource); + copyNestedNonNullProperties(datasource, dbDatasource); return dbDatasource; }) .flatMap(this::validateAndSaveDatasourceToRepository); @@ -90,9 +92,8 @@ public class DatasourceServiceImpl extends BaseService validateDatasource(Datasource datasource) { Set invalids = new HashSet<>(); - Mono userMono = sessionUserService.getCurrentUser(); - if (datasource.getName() == null || datasource.getName().trim().isEmpty()) { + if (!StringUtils.hasText(datasource.getName())) { return Mono.error(new AppsmithException(AppsmithError.INVALID_PARAMETER, FieldName.NAME)); } @@ -103,8 +104,11 @@ public class DatasourceServiceImpl extends BaseService userMono = sessionUserService.getCurrentUser(); + Mono organizationMono = userMono - .flatMap(user -> organizationService.findByIdAndPluginsPluginId(user.getCurrentOrganizationId(), datasource.getPluginId())) + .flatMap(user -> organizationService.findByIdAndPluginsPluginId( + user.getCurrentOrganizationId(), datasource.getPluginId())) .switchIfEmpty(Mono.defer(() -> { datasource.setIsValid(false); invalids.add(AppsmithError.PLUGIN_NOT_INSTALLED.getMessage(datasource.getPluginId())); @@ -132,14 +136,13 @@ public class DatasourceServiceImpl extends BaseService { Datasource datasource1 = tuple.getT1(); PluginExecutor pluginExecutor = tuple.getT2(); - Boolean isDatasourceConfigValid = false; - if (pluginExecutor != null && datasource1.getDatasourceConfiguration() != null) { - isDatasourceConfigValid = pluginExecutor.isDatasourceValid(datasource1.getDatasourceConfiguration()); - } - if (!isDatasourceConfigValid) { + + DatasourceConfiguration datasourceConfiguration = datasource1.getDatasourceConfiguration(); + if (datasourceConfiguration != null && !pluginExecutor.isDatasourceValid(datasourceConfiguration)) { datasource1.setIsValid(false); invalids.add(AppsmithError.INVALID_DATASOURCE_CONFIGURATION.getMessage()); } + datasource1.setInvalids(invalids); return Mono.just(datasource1); }); diff --git a/app/server/appsmith-server/src/test/java/com/appsmith/server/services/DatasourceServiceTest.java b/app/server/appsmith-server/src/test/java/com/appsmith/server/services/DatasourceServiceTest.java index eda60992a8..0976ef7f0d 100644 --- a/app/server/appsmith-server/src/test/java/com/appsmith/server/services/DatasourceServiceTest.java +++ b/app/server/appsmith-server/src/test/java/com/appsmith/server/services/DatasourceServiceTest.java @@ -1,7 +1,9 @@ package com.appsmith.server.services; import com.appsmith.external.models.ActionConfiguration; +import com.appsmith.external.models.Connection; import com.appsmith.external.models.DatasourceConfiguration; +import com.appsmith.external.models.SSLDetails; import com.appsmith.external.plugins.PluginExecutor; import com.appsmith.server.constants.FieldName; import com.appsmith.server.domains.Datasource; @@ -159,4 +161,54 @@ public class DatasourceServiceTest { }) .verifyComplete(); } + + @Test + @WithUserDetails(value = "api_user") + public void createAndUpdateDatasourceValidDB() { + Mockito.when(pluginExecutorHelper.getPluginExecutor(Mockito.any())).thenReturn(Mono.just(new TestPluginExecutor())); + + Datasource datasource = new Datasource(); + datasource.setName("test db datasource"); + DatasourceConfiguration datasourceConfiguration = new DatasourceConfiguration(); + Connection connection = new Connection(); + connection.setMode(Connection.Mode.ReadOnly); + connection.setType(Connection.Type.ReplicaSet); + SSLDetails sslDetails = new SSLDetails(); + sslDetails.setAuthType(SSLDetails.AuthType.CACertificate); + sslDetails.setKeyFile("ssl_key_file_id"); + sslDetails.setCertificateFile("ssl_cert_file_id"); + connection.setSsl(sslDetails); + datasourceConfiguration.setConnection(connection); + datasource.setDatasourceConfiguration(datasourceConfiguration); + + datasource.setOrganizationId("fixme-put-valid-org-id-here"); + + Mono pluginMono = pluginService.findByName("Installed Plugin Name"); + + Mono datasourceMono = pluginMono + .map(plugin -> { + datasource.setPluginId(plugin.getId()); + return datasource; + }).flatMap(datasourceService::create) + .flatMap(datasource1 -> { + Datasource updates = new Datasource(); + DatasourceConfiguration datasourceConfiguration1 = new DatasourceConfiguration(); + Connection connection1 = new Connection(); + SSLDetails ssl = new SSLDetails(); + ssl.setKeyFile("ssl_key_file_id"); + connection1.setSsl(ssl); + datasourceConfiguration1.setConnection(connection1); + return datasourceService.update(datasource1.getId(), updates); + }); + + StepVerifier + .create(datasourceMono) + .assertNext(createdDatasource -> { + assertThat(createdDatasource.getId()).isNotEmpty(); + assertThat(createdDatasource.getPluginId()).isEqualTo(datasource.getPluginId()); + assertThat(createdDatasource.getName()).isEqualTo(datasource.getName()); + assertThat(createdDatasource.getDatasourceConfiguration().getConnection().getSsl().getKeyFile()).isEqualTo("ssl_key_file_id"); + }) + .verifyComplete(); + } } diff --git a/app/server/appsmith-server/src/test/java/com/appsmith/server/utils/BeanCopyUtilsTest.java b/app/server/appsmith-server/src/test/java/com/appsmith/server/utils/BeanCopyUtilsTest.java new file mode 100644 index 0000000000..689d82c308 --- /dev/null +++ b/app/server/appsmith-server/src/test/java/com/appsmith/server/utils/BeanCopyUtilsTest.java @@ -0,0 +1,93 @@ +package com.appsmith.server.utils; + +import com.appsmith.server.helpers.BeanCopyUtils; +import lombok.*; +import lombok.extern.slf4j.Slf4j; +import org.junit.Test; +import org.mockito.Mockito; + +import java.time.LocalDate; + +import static org.assertj.core.api.Assertions.assertThat; + +@Slf4j +public class BeanCopyUtilsTest { + + @Getter + @Setter + @AllArgsConstructor + @NoArgsConstructor + static class Person { + private String name; + private Integer age; + private Boolean isApproved; + private LocalDate joinDate; + private Person mentor = null; + } + + @Test + public void copyProperties() { + Person source = new Person(); + source.setAge(30); + + Person target = new Person( + "Luke", + 25, + true, + LocalDate.of(2000, 1, 1), + null + ); + + BeanCopyUtils.copyNestedNonNullProperties(source, target); + assertThat(target.getName()).isEqualTo("Luke"); + assertThat(target.getAge()).isEqualTo(30); + assertThat(target.getIsApproved()).isEqualTo(true); + assertThat(target.getJoinDate()).isEqualTo(LocalDate.of(2000, 1, 1)); + } + + @Test + public void copyNestedProperty() { + Person source = new Person(), mentor = new Person(); + mentor.setName("The new mentor name"); + source.setMentor(mentor); + + Person target = new Person( + "Luke", + 25, + true, + LocalDate.of(2000, 1, 1), + new Person( + "Leia", + 25, + true, + LocalDate.of(2000, 1, 1), + null + ) + ); + + BeanCopyUtils.copyNestedNonNullProperties(source, target); + assertThat(target.getName()).isEqualTo("Luke"); + assertThat(target.getMentor().getName()).isEqualTo("The new mentor name"); + assertThat(target.getMentor().getAge()).isEqualTo(25); + } + + @Test + public void copyNestedPropertyWithTargetNull() { + Person source = new Person(), mentor = new Person(); + mentor.setName("The new mentor name"); + source.setMentor(mentor); + + Person target = new Person( + "Luke", + 25, + true, + LocalDate.of(2000, 1, 1), + null + ); + + BeanCopyUtils.copyNestedNonNullProperties(source, target); + assertThat(target.getName()).isEqualTo("Luke"); + assertThat(target.getMentor().getName()).isEqualTo("The new mentor name"); + } + +}