Datasource CRUD APIs

app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/AuthenticationDTO.java

@@ -1,5 +1,6 @@
 package com.appsmith.external.models;
 
+import com.fasterxml.jackson.annotation.JsonProperty;
 import lombok.AllArgsConstructor;
 import lombok.Getter;
 import lombok.NoArgsConstructor;
@@ -12,7 +13,18 @@ import lombok.ToString;
 @NoArgsConstructor
 @AllArgsConstructor
 public class AuthenticationDTO {
-    String authType;
+
+    public enum Type {
+        SCRAM_SHA_1, SCRAM_SHA_256, MONGODB_CR, USERNAME_PASSWORD
+    }
+
+    Type authType;
+
     String username;
+
+    @JsonProperty(access = JsonProperty.Access.WRITE_ONLY)
     String password;
-}
+
+    String databaseName;
+
+}

app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/Connection.java

@@ -0,0 +1,29 @@
+package com.appsmith.external.models;
+
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+import lombok.ToString;
+import org.springframework.data.mongodb.core.mapping.Document;
+
+@Getter
+@Setter
+@ToString
+@NoArgsConstructor
+@Document
+public class Connection {
+
+    public enum Mode {
+        ReadOnly, ReadWrite
+    }
+
+    public enum Type {
+        DirectConnection, ReplicaSet
+    }
+
+    Mode mode;
+
+    Type type;
+
+    SSLDetails ssl;
+}

app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/DatasourceConfiguration.java

@@ -15,17 +15,19 @@ import java.util.List;
 @Document
 public class DatasourceConfiguration {
 
-    String url;
+    Connection connection;
+
+    List<Endpoint> endpoints;
 
     AuthenticationDTO authentication;
 
+    SSHConnection sshProxy;
+
     List<Property> properties;
 
-    //For REST API
+    // For REST API.
+    String url;
+
     List<Property> headers;
 
-    //This field is for plugins which allow the database name to be specified outside of the connection URL. The
-    //expectation from the plugins is that if the database name has been provided via this field, the database name
-    //should be according to this, else if database name is null, pick the database name from the URL.
-    String databaseName;
 }

app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/Endpoint.java

@@ -0,0 +1,20 @@
+package com.appsmith.external.models;
+
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+import lombok.ToString;
+import org.springframework.data.mongodb.core.mapping.Document;
+
+@Getter
+@Setter
+@ToString
+@NoArgsConstructor
+@Document
+public class Endpoint {
+
+    String host;
+
+    Long port;
+
+}

app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/PEMCertificate.java

@@ -0,0 +1,20 @@
+package com.appsmith.external.models;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import lombok.Getter;
+import lombok.Setter;
+import lombok.ToString;
+import org.springframework.data.mongodb.core.mapping.Document;
+
+@Getter
+@Setter
+@ToString
+@Document
+public class PEMCertificate {
+
+    String file;
+
+    @JsonProperty(access = JsonProperty.Access.WRITE_ONLY)
+    String password;
+
+}

app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/SSHConnection.java

@@ -0,0 +1,26 @@
+package com.appsmith.external.models;
+
+import lombok.*;
+
+@Getter
+@Setter
+@ToString
+@NoArgsConstructor
+@AllArgsConstructor
+public class SSHConnection {
+
+    public enum AuthType {
+        PrivateKey, Password
+    }
+
+    String host;
+
+    Long port;
+
+    String username;
+
+    AuthType authType;
+
+    SSHPrivateKey privateKey;
+
+}

app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/SSHPrivateKey.java

@@ -0,0 +1,18 @@
+package com.appsmith.external.models;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import lombok.*;
+
+@Getter
+@Setter
+@ToString
+@NoArgsConstructor
+@AllArgsConstructor
+public class SSHPrivateKey {
+
+    String keyFile;
+
+    @JsonProperty(access = JsonProperty.Access.WRITE_ONLY)
+    String password;
+
+}

app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/SSLDetails.java

@@ -0,0 +1,32 @@
+package com.appsmith.external.models;
+
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+import lombok.ToString;
+import org.springframework.data.mongodb.core.mapping.Document;
+
+@Getter
+@Setter
+@ToString
+@NoArgsConstructor
+@Document
+public class SSLDetails {
+
+    public enum AuthType {
+        CACertificate, SelfSignedCertificate
+    }
+
+    AuthType authType;
+
+    String keyFile;
+
+    String certificateFile;
+
+    String caCertificateFile;
+
+    Boolean usePemCertificate;
+
+    PEMCertificate pemCertificate;
+
+}

app/server/appsmith-plugins/mongoPlugin/src/main/java/com/external/plugins/MongoPlugin.java

@@ -57,7 +57,8 @@ public class MongoPlugin extends BasePlugin {
 
             MongoClientURI mongoClientURI = new MongoClientURI(datasourceConfiguration.getUrl());
 
-            String databaseName = datasourceConfiguration.getDatabaseName();
+            // CONFIRM: Verify that the database name correctly shows up here.
+            String databaseName = datasourceConfiguration.getAuthentication().getDatabaseName();
             if (databaseName == null) {
                 databaseName = mongoClientURI.getDatabase();
             }

app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/BeanCopyUtils.java

@@ -3,13 +3,19 @@ package com.appsmith.server.helpers;
 import org.springframework.beans.BeanUtils;
 import org.springframework.beans.BeanWrapper;
 import org.springframework.beans.BeanWrapperImpl;
+import org.springframework.beans.PropertyAccessorFactory;
 
+import java.beans.PropertyDescriptor;
+import java.util.ArrayList;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Set;
 
 public final class BeanCopyUtils {
 
     private static String[] getNullPropertyNames(Object source) {
+        // TODO: The `BeanWrapperImpl` class has been declared to be an internal class. Migrate to using
+        //  `PropertyAccessorFactory.forBeanPropertyAccess` instead.
         final BeanWrapper src = new BeanWrapperImpl(source);
         java.beans.PropertyDescriptor[] pds = src.getPropertyDescriptors();
 
@@ -43,4 +49,44 @@ public final class BeanCopyUtils {
 
         return count++;
     }
+
+    public static void copyNestedNonNullProperties(Object source, Object target) {
+        if (source == null || target ==  null) {
+            return;
+        }
+
+        final BeanWrapper sourceBeanWrapper = PropertyAccessorFactory.forBeanPropertyAccess(source);
+        BeanWrapper targetBeanWrapper = PropertyAccessorFactory.forBeanPropertyAccess(target);
+        PropertyDescriptor[] propertyDescriptors = sourceBeanWrapper.getPropertyDescriptors();
+
+        for (PropertyDescriptor propertyDescriptor : propertyDescriptors) {
+            String name = propertyDescriptor.getName();
+
+            // For properties like `class` that don't have a set method, we can't copy so we just ignore them.
+            if (targetBeanWrapper.getPropertyDescriptor(name).getWriteMethod() == null) {
+                continue;
+            }
+
+            Object sourceValue = sourceBeanWrapper.getPropertyValue(name);
+
+            // If sourceValue is null, don't copy it over to target and just move on to the next property.
+            if (sourceValue == null) {
+                continue;
+            }
+
+            Object targetValue = targetBeanWrapper.getPropertyValue(name);
+
+            if (targetValue != null && isDomainModel(propertyDescriptor.getPropertyType())) {
+                // Go deeper *only* if the property belongs to Appsmith's models, and both the source and target values
+                // are not null.
+                copyNestedNonNullProperties(sourceValue, targetValue);
+            } else {
+                targetBeanWrapper.setPropertyValue(name, sourceValue);
+            }
+        }
+    }
+
+    public static boolean isDomainModel(Class<?> type) {
+        return type.getPackageName().startsWith("com.appsmith.");
+    }
 }

app/server/appsmith-server/src/main/java/com/appsmith/server/services/DatasourceServiceImpl.java

@@ -1,5 +1,6 @@
 package com.appsmith.server.services;
 
+import com.appsmith.external.models.DatasourceConfiguration;
 import com.appsmith.external.plugins.PluginExecutor;
 import com.appsmith.server.constants.FieldName;
 import com.appsmith.server.domains.Datasource;
@@ -17,6 +18,7 @@ import org.springframework.data.mongodb.core.ReactiveMongoTemplate;
 import org.springframework.data.mongodb.core.convert.MongoConverter;
 import org.springframework.stereotype.Service;
 import org.springframework.util.MultiValueMap;
+import org.springframework.util.StringUtils;
 import reactor.core.publisher.Flux;
 import reactor.core.publisher.Mono;
 import reactor.core.scheduler.Scheduler;
@@ -26,7 +28,7 @@ import javax.validation.constraints.NotNull;
 import java.util.HashSet;
 import java.util.Set;
 
-import static com.appsmith.server.helpers.BeanCopyUtils.copyNewFieldValuesIntoOldObject;
+import static com.appsmith.server.helpers.BeanCopyUtils.copyNestedNonNullProperties;
 import static com.appsmith.server.helpers.MustacheHelper.extractMustacheKeys;
 
 @Slf4j
@@ -81,7 +83,7 @@ public class DatasourceServiceImpl extends BaseService<DatasourceRepository, Dat
 
         return datasourceMono
                 .map(dbDatasource -> {
-                    copyNewFieldValuesIntoOldObject(datasource, dbDatasource);
+                    copyNestedNonNullProperties(datasource, dbDatasource);
                     return dbDatasource;
                 })
                 .flatMap(this::validateAndSaveDatasourceToRepository);
@@ -90,9 +92,8 @@ public class DatasourceServiceImpl extends BaseService<DatasourceRepository, Dat
     @Override
     public Mono<Datasource> validateDatasource(Datasource datasource) {
         Set<String> invalids = new HashSet<>();
-        Mono<User> userMono = sessionUserService.getCurrentUser();
 
-        if (datasource.getName() == null || datasource.getName().trim().isEmpty()) {
+        if (!StringUtils.hasText(datasource.getName())) {
             return Mono.error(new AppsmithException(AppsmithError.INVALID_PARAMETER, FieldName.NAME));
         }
 
@@ -103,8 +104,11 @@ public class DatasourceServiceImpl extends BaseService<DatasourceRepository, Dat
             return Mono.just(datasource);
         }
 
+        Mono<User> userMono = sessionUserService.getCurrentUser();
+
         Mono<Organization> organizationMono = userMono
-                .flatMap(user -> organizationService.findByIdAndPluginsPluginId(user.getCurrentOrganizationId(), datasource.getPluginId()))
+                .flatMap(user -> organizationService.findByIdAndPluginsPluginId(
+                        user.getCurrentOrganizationId(), datasource.getPluginId()))
                 .switchIfEmpty(Mono.defer(() -> {
                     datasource.setIsValid(false);
                     invalids.add(AppsmithError.PLUGIN_NOT_INSTALLED.getMessage(datasource.getPluginId()));
@@ -132,14 +136,13 @@ public class DatasourceServiceImpl extends BaseService<DatasourceRepository, Dat
                 .flatMap(tuple -> {
                     Datasource datasource1 = tuple.getT1();
                     PluginExecutor pluginExecutor = tuple.getT2();
-                    Boolean isDatasourceConfigValid = false;
-                    if (pluginExecutor != null && datasource1.getDatasourceConfiguration() != null) {
-                        isDatasourceConfigValid = pluginExecutor.isDatasourceValid(datasource1.getDatasourceConfiguration());
-                    }
-                    if (!isDatasourceConfigValid) {
+
+                    DatasourceConfiguration datasourceConfiguration = datasource1.getDatasourceConfiguration();
+                    if (datasourceConfiguration != null && !pluginExecutor.isDatasourceValid(datasourceConfiguration)) {
                         datasource1.setIsValid(false);
                         invalids.add(AppsmithError.INVALID_DATASOURCE_CONFIGURATION.getMessage());
                     }
+
                     datasource1.setInvalids(invalids);
                     return Mono.just(datasource1);
                 });

app/server/appsmith-server/src/test/java/com/appsmith/server/services/DatasourceServiceTest.java

@@ -1,7 +1,9 @@
 package com.appsmith.server.services;
 
 import com.appsmith.external.models.ActionConfiguration;
+import com.appsmith.external.models.Connection;
 import com.appsmith.external.models.DatasourceConfiguration;
+import com.appsmith.external.models.SSLDetails;
 import com.appsmith.external.plugins.PluginExecutor;
 import com.appsmith.server.constants.FieldName;
 import com.appsmith.server.domains.Datasource;
@@ -159,4 +161,54 @@ public class DatasourceServiceTest {
                 })
                 .verifyComplete();
     }
+
+    @Test
+    @WithUserDetails(value = "api_user")
+    public void createAndUpdateDatasourceValidDB() {
+        Mockito.when(pluginExecutorHelper.getPluginExecutor(Mockito.any())).thenReturn(Mono.just(new TestPluginExecutor()));
+
+        Datasource datasource = new Datasource();
+        datasource.setName("test db datasource");
+        DatasourceConfiguration datasourceConfiguration = new DatasourceConfiguration();
+        Connection connection = new Connection();
+        connection.setMode(Connection.Mode.ReadOnly);
+        connection.setType(Connection.Type.ReplicaSet);
+        SSLDetails sslDetails = new SSLDetails();
+        sslDetails.setAuthType(SSLDetails.AuthType.CACertificate);
+        sslDetails.setKeyFile("ssl_key_file_id");
+        sslDetails.setCertificateFile("ssl_cert_file_id");
+        connection.setSsl(sslDetails);
+        datasourceConfiguration.setConnection(connection);
+        datasource.setDatasourceConfiguration(datasourceConfiguration);
+
+        datasource.setOrganizationId("fixme-put-valid-org-id-here");
+
+        Mono<Plugin> pluginMono = pluginService.findByName("Installed Plugin Name");
+
+        Mono<Datasource> datasourceMono = pluginMono
+                .map(plugin -> {
+                    datasource.setPluginId(plugin.getId());
+                    return datasource;
+                }).flatMap(datasourceService::create)
+                .flatMap(datasource1 -> {
+                    Datasource updates = new Datasource();
+                    DatasourceConfiguration datasourceConfiguration1 = new DatasourceConfiguration();
+                    Connection connection1 = new Connection();
+                    SSLDetails ssl = new SSLDetails();
+                    ssl.setKeyFile("ssl_key_file_id");
+                    connection1.setSsl(ssl);
+                    datasourceConfiguration1.setConnection(connection1);
+                    return datasourceService.update(datasource1.getId(), updates);
+                });
+
+        StepVerifier
+                .create(datasourceMono)
+                .assertNext(createdDatasource -> {
+                    assertThat(createdDatasource.getId()).isNotEmpty();
+                    assertThat(createdDatasource.getPluginId()).isEqualTo(datasource.getPluginId());
+                    assertThat(createdDatasource.getName()).isEqualTo(datasource.getName());
+                    assertThat(createdDatasource.getDatasourceConfiguration().getConnection().getSsl().getKeyFile()).isEqualTo("ssl_key_file_id");
+                })
+                .verifyComplete();
+    }
 }

app/server/appsmith-server/src/test/java/com/appsmith/server/utils/BeanCopyUtilsTest.java

@@ -0,0 +1,93 @@
+package com.appsmith.server.utils;
+
+import com.appsmith.server.helpers.BeanCopyUtils;
+import lombok.*;
+import lombok.extern.slf4j.Slf4j;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import java.time.LocalDate;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+@Slf4j
+public class BeanCopyUtilsTest {
+
+    @Getter
+    @Setter
+    @AllArgsConstructor
+    @NoArgsConstructor
+    static class Person {
+        private String name;
+        private Integer age;
+        private Boolean isApproved;
+        private LocalDate joinDate;
+        private Person mentor = null;
+    }
+
+    @Test
+    public void copyProperties() {
+        Person source = new Person();
+        source.setAge(30);
+
+        Person target = new Person(
+                "Luke",
+                25,
+                true,
+                LocalDate.of(2000, 1, 1),
+                null
+        );
+
+        BeanCopyUtils.copyNestedNonNullProperties(source, target);
+        assertThat(target.getName()).isEqualTo("Luke");
+        assertThat(target.getAge()).isEqualTo(30);
+        assertThat(target.getIsApproved()).isEqualTo(true);
+        assertThat(target.getJoinDate()).isEqualTo(LocalDate.of(2000, 1, 1));
+    }
+
+    @Test
+    public void copyNestedProperty() {
+        Person source = new Person(), mentor = new Person();
+        mentor.setName("The new mentor name");
+        source.setMentor(mentor);
+
+        Person target = new Person(
+                "Luke",
+                25,
+                true,
+                LocalDate.of(2000, 1, 1),
+                new Person(
+                        "Leia",
+                        25,
+                        true,
+                        LocalDate.of(2000, 1, 1),
+                        null
+                )
+        );
+
+        BeanCopyUtils.copyNestedNonNullProperties(source, target);
+        assertThat(target.getName()).isEqualTo("Luke");
+        assertThat(target.getMentor().getName()).isEqualTo("The new mentor name");
+        assertThat(target.getMentor().getAge()).isEqualTo(25);
+    }
+
+    @Test
+    public void copyNestedPropertyWithTargetNull() {
+        Person source = new Person(), mentor = new Person();
+        mentor.setName("The new mentor name");
+        source.setMentor(mentor);
+
+        Person target = new Person(
+                "Luke",
+                25,
+                true,
+                LocalDate.of(2000, 1, 1),
+                null
+        );
+
+        BeanCopyUtils.copyNestedNonNullProperties(source, target);
+        assertThat(target.getName()).isEqualTo("Luke");
+        assertThat(target.getMentor().getName()).isEqualTo("The new mentor name");
+    }
+
+}