From 2dbf9d1c6b01530b51b69272b961000b3ecbdbcd Mon Sep 17 00:00:00 2001 From: Shrikant Kandula Date: Wed, 15 Apr 2020 10:02:09 +0000 Subject: [PATCH] Test API for data sources --- .../external/models/DatasourceTestResult.java | 35 +++++++++ .../external/plugins/PluginExecutor.java | 5 +- .../com/external/plugins/MongoPlugin.java | 11 ++- .../com/external/plugins/PostgresPlugin.java | 77 ++++++++++++++++--- .../com/external/plugins/RapidApiPlugin.java | 14 ++-- .../com/external/plugins/RestApiPlugin.java | 30 +++++++- .../controllers/DatasourceController.java | 32 ++++++++ .../DatasourceContextServiceImpl.java | 36 ++++++--- .../server/services/DatasourceService.java | 3 + .../services/DatasourceServiceImpl.java | 10 +++ .../services/DatasourceServiceTest.java | 9 ++- 11 files changed, 226 insertions(+), 36 deletions(-) create mode 100644 app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/DatasourceTestResult.java diff --git a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/DatasourceTestResult.java b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/DatasourceTestResult.java new file mode 100644 index 0000000000..4ff7f9d15a --- /dev/null +++ b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/DatasourceTestResult.java @@ -0,0 +1,35 @@ +package com.appsmith.external.models; + +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import org.springframework.util.CollectionUtils; + +import java.util.Set; + +@Getter +@Setter +@ToString +public class DatasourceTestResult { + + Set invalids; + + /** + * Convenience constructor to create a result object with one or more error messages. This constructor also ensures + * that the `invalids` field is never null. + * @param invalids String messages that explain why the test failed. + */ + public DatasourceTestResult(String... invalids) { + this.invalids = Set.of(invalids); + } + + public DatasourceTestResult(Set invalids) { + this.invalids = invalids; + } + + public boolean isSuccess() { + // This method exists so that a `"success"` boolean key is present in the JSON response to the frontend. + return CollectionUtils.isEmpty(invalids); + } + +} diff --git a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/plugins/PluginExecutor.java b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/plugins/PluginExecutor.java index c6ab7dbaf0..fb564f5ae5 100644 --- a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/plugins/PluginExecutor.java +++ b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/plugins/PluginExecutor.java @@ -2,6 +2,7 @@ package com.appsmith.external.plugins; import com.appsmith.external.models.ActionConfiguration; import com.appsmith.external.models.DatasourceConfiguration; +import com.appsmith.external.models.DatasourceTestResult; import org.pf4j.ExtensionPoint; import org.springframework.util.CollectionUtils; import reactor.core.publisher.Mono; @@ -28,7 +29,7 @@ public interface PluginExecutor extends ExtensionPoint { * @param datasourceConfiguration * @return Connection object */ - Object datasourceCreate(DatasourceConfiguration datasourceConfiguration); + Mono datasourceCreate(DatasourceConfiguration datasourceConfiguration); /** * This function is used to bring down/destroy the connection to the data source. @@ -42,4 +43,6 @@ public interface PluginExecutor extends ExtensionPoint { } Set validateDatasource(DatasourceConfiguration datasourceConfiguration); + + Mono testDatasource(DatasourceConfiguration datasourceConfiguration); } diff --git a/app/server/appsmith-plugins/mongoPlugin/src/main/java/com/external/plugins/MongoPlugin.java b/app/server/appsmith-plugins/mongoPlugin/src/main/java/com/external/plugins/MongoPlugin.java index 933dcac907..96c1e19d42 100644 --- a/app/server/appsmith-plugins/mongoPlugin/src/main/java/com/external/plugins/MongoPlugin.java +++ b/app/server/appsmith-plugins/mongoPlugin/src/main/java/com/external/plugins/MongoPlugin.java @@ -3,6 +3,7 @@ package com.external.plugins; import com.appsmith.external.models.ActionConfiguration; import com.appsmith.external.models.ActionExecutionResult; import com.appsmith.external.models.DatasourceConfiguration; +import com.appsmith.external.models.DatasourceTestResult; import com.appsmith.external.plugins.BasePlugin; import com.appsmith.external.plugins.PluginExecutor; import com.fasterxml.jackson.databind.ObjectMapper; @@ -114,10 +115,9 @@ public class MongoPlugin extends BasePlugin { } @Override - public Object datasourceCreate(DatasourceConfiguration datasourceConfiguration) { - + public Mono datasourceCreate(DatasourceConfiguration datasourceConfiguration) { MongoClientURI mongoClientURI = new MongoClientURI(datasourceConfiguration.getUrl()); - return new MongoClient(mongoClientURI); + return Mono.just(new MongoClient(mongoClientURI)); } @Override @@ -134,6 +134,11 @@ public class MongoPlugin extends BasePlugin { return new HashSet<>(); } + @Override + public Mono testDatasource(DatasourceConfiguration datasourceConfiguration) { + return Mono.just(new DatasourceTestResult()); + } + } } diff --git a/app/server/appsmith-plugins/postgresPlugin/src/main/java/com/external/plugins/PostgresPlugin.java b/app/server/appsmith-plugins/postgresPlugin/src/main/java/com/external/plugins/PostgresPlugin.java index 9e38b1ac6c..77af2cdb84 100644 --- a/app/server/appsmith-plugins/postgresPlugin/src/main/java/com/external/plugins/PostgresPlugin.java +++ b/app/server/appsmith-plugins/postgresPlugin/src/main/java/com/external/plugins/PostgresPlugin.java @@ -1,6 +1,11 @@ package com.external.plugins; -import com.appsmith.external.models.*; +import com.appsmith.external.models.ActionConfiguration; +import com.appsmith.external.models.ActionExecutionResult; +import com.appsmith.external.models.AuthenticationDTO; +import com.appsmith.external.models.DatasourceConfiguration; +import com.appsmith.external.models.DatasourceTestResult; +import com.appsmith.external.models.Endpoint; import com.appsmith.external.pluginExceptions.AppsmithPluginError; import com.appsmith.external.pluginExceptions.AppsmithPluginException; import com.appsmith.external.plugins.BasePlugin; @@ -16,9 +21,18 @@ import org.springframework.util.StringUtils; import reactor.core.publisher.Mono; import java.sql.Connection; -import java.sql.*; -import java.util.*; -import java.util.stream.Stream; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; import static com.appsmith.external.models.Connection.Mode.READ_ONLY; @@ -64,9 +78,11 @@ public class PostgresPlugin extends BasePlugin { List> rowsList = new ArrayList<>(50); + Statement statement = null; + ResultSet resultSet = null; try { - Statement statement = conn.createStatement(); - ResultSet resultSet = statement.executeQuery(query); + statement = conn.createStatement(); + resultSet = statement.executeQuery(query); ResultSetMetaData metaData = resultSet.getMetaData(); int colCount = metaData.getColumnCount(); while (resultSet.next()) { @@ -80,12 +96,29 @@ public class PostgresPlugin extends BasePlugin { } catch (SQLException e) { return pluginErrorMono(e); + } finally { + if (resultSet != null) { + try { + resultSet.close(); + } catch (SQLException e) { + log.warn("Error closing Postgres ResultSet", e); + } + } + + if (statement != null) { + try { + statement.close(); + } catch (SQLException e) { + log.warn("Error closing Postgres Statement", e); + } + } + } ActionExecutionResult result = new ActionExecutionResult(); result.setBody(objectMapper.valueToTree(rowsList)); result.setShouldCacheResponse(true); - System.out.println("In the PostgresPlugin, got action execution result: " + result.toString()); + log.debug("In the PostgresPlugin, got action execution result: " + result.toString()); return Mono.just(result); } @@ -94,7 +127,7 @@ public class PostgresPlugin extends BasePlugin { } @Override - public Object datasourceCreate(DatasourceConfiguration datasourceConfiguration) { + public Mono datasourceCreate(DatasourceConfiguration datasourceConfiguration) { try { Class.forName(JDBC_DRIVER); } catch (ClassNotFoundException e) { @@ -104,12 +137,14 @@ public class PostgresPlugin extends BasePlugin { String url; AuthenticationDTO authentication = datasourceConfiguration.getAuthentication(); + com.appsmith.external.models.Connection configurationConnection = datasourceConfiguration.getConnection(); + Properties properties = new Properties(); properties.putAll(Map.of( USER, authentication.getUsername(), PASSWORD, authentication.getPassword(), // TODO: Set SSL connection parameters. - SSL, datasourceConfiguration.getConnection().getSsl() != null + SSL, configurationConnection != null && configurationConnection.getSsl() != null )); if (CollectionUtils.isEmpty(datasourceConfiguration.getEndpoints())) { @@ -131,11 +166,12 @@ public class PostgresPlugin extends BasePlugin { try { Connection connection = DriverManager.getConnection(url, properties); - connection.setReadOnly(READ_ONLY.equals(datasourceConfiguration.getConnection().getMode())); - return connection; + connection.setReadOnly( + configurationConnection != null && READ_ONLY.equals(configurationConnection.getMode())); + return Mono.just(connection); } catch (SQLException e) { - return pluginErrorMono("Error connecting to Postgres."); + return pluginErrorMono("Error connecting to Postgres.", e); } } @@ -186,6 +222,23 @@ public class PostgresPlugin extends BasePlugin { return invalids; } + @Override + public Mono testDatasource(DatasourceConfiguration datasourceConfiguration) { + return datasourceCreate(datasourceConfiguration) + .map(connection -> { + try { + if (connection != null) { + ((Connection) connection).close(); + } + } catch (SQLException e) { + log.warn("Error closing Postgres connection that was made for testing.", e); + } + + return new DatasourceTestResult(); + }) + .onErrorResume(error -> Mono.just(new DatasourceTestResult(error.getMessage()))); + } + } } diff --git a/app/server/appsmith-plugins/rapidApiPlugin/src/main/java/com/external/plugins/RapidApiPlugin.java b/app/server/appsmith-plugins/rapidApiPlugin/src/main/java/com/external/plugins/RapidApiPlugin.java index e5933204b8..53ec22ab70 100644 --- a/app/server/appsmith-plugins/rapidApiPlugin/src/main/java/com/external/plugins/RapidApiPlugin.java +++ b/app/server/appsmith-plugins/rapidApiPlugin/src/main/java/com/external/plugins/RapidApiPlugin.java @@ -1,9 +1,6 @@ package com.external.plugins; -import com.appsmith.external.models.ActionConfiguration; -import com.appsmith.external.models.ActionExecutionResult; -import com.appsmith.external.models.DatasourceConfiguration; -import com.appsmith.external.models.Property; +import com.appsmith.external.models.*; import com.appsmith.external.pluginExceptions.AppsmithPluginError; import com.appsmith.external.pluginExceptions.AppsmithPluginException; import com.appsmith.external.plugins.BasePlugin; @@ -238,8 +235,8 @@ public class RapidApiPlugin extends BasePlugin { } @Override - public Object datasourceCreate(DatasourceConfiguration datasourceConfiguration) { - return null; + public Mono datasourceCreate(DatasourceConfiguration datasourceConfiguration) { + return Mono.empty(); } @Override @@ -254,6 +251,11 @@ public class RapidApiPlugin extends BasePlugin { return new HashSet<>(); } + @Override + public Mono testDatasource(DatasourceConfiguration datasourceConfiguration) { + return Mono.just(new DatasourceTestResult()); + } + private void addHeadersToRequest(WebClient.Builder webClientBuilder, List headers) { for (Property header : headers) { if (header.getKey() != null && !header.getKey().isEmpty()) { diff --git a/app/server/appsmith-plugins/restApiPlugin/src/main/java/com/external/plugins/RestApiPlugin.java b/app/server/appsmith-plugins/restApiPlugin/src/main/java/com/external/plugins/RestApiPlugin.java index dd60d40dae..80e3475ff2 100644 --- a/app/server/appsmith-plugins/restApiPlugin/src/main/java/com/external/plugins/RestApiPlugin.java +++ b/app/server/appsmith-plugins/restApiPlugin/src/main/java/com/external/plugins/RestApiPlugin.java @@ -3,6 +3,7 @@ package com.external.plugins; import com.appsmith.external.models.ActionConfiguration; import com.appsmith.external.models.ActionExecutionResult; import com.appsmith.external.models.DatasourceConfiguration; +import com.appsmith.external.models.DatasourceTestResult; import com.appsmith.external.models.Property; import com.appsmith.external.pluginExceptions.AppsmithPluginError; import com.appsmith.external.pluginExceptions.AppsmithPluginException; @@ -27,6 +28,9 @@ import org.springframework.web.util.UriComponentsBuilder; import reactor.core.publisher.Mono; import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.MalformedURLException; +import java.net.Socket; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; @@ -208,8 +212,8 @@ public class RestApiPlugin extends BasePlugin { } @Override - public Object datasourceCreate(DatasourceConfiguration datasourceConfiguration) { - return null; + public Mono datasourceCreate(DatasourceConfiguration datasourceConfiguration) { + return Mono.empty(); } @Override @@ -235,6 +239,28 @@ public class RestApiPlugin extends BasePlugin { return invalids; } + @Override + public Mono testDatasource(DatasourceConfiguration datasourceConfiguration) { + URL url; + try { + url = new URL(datasourceConfiguration.getUrl()); + } catch (MalformedURLException e) { + return Mono.just(new DatasourceTestResult("Invalid URL: '" + e.getMessage() + "'.")); + } + + try (Socket socket = new Socket()) { + socket.connect(new InetSocketAddress(url.getHost(), url.getPort()), 300); + + } catch (IOException e) { + return Mono.just( + new DatasourceTestResult("Failed to reach API endpoint: '" + e.getMessage() + "'.") + ); + + } + + return Mono.just(new DatasourceTestResult()); + } + private boolean addHeadersToRequestAndAscertainContentType(WebClient.Builder webClientBuilder, List headers, boolean isContentTypeJson) { diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/controllers/DatasourceController.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/controllers/DatasourceController.java index 12d80e4e8e..586b212733 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/controllers/DatasourceController.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/controllers/DatasourceController.java @@ -1,11 +1,17 @@ package com.appsmith.server.controllers; +import com.appsmith.external.models.DatasourceTestResult; import com.appsmith.server.constants.Url; import com.appsmith.server.domains.Datasource; +import com.appsmith.server.dtos.ResponseDTO; import com.appsmith.server.services.DatasourceService; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.util.CollectionUtils; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; +import reactor.core.publisher.Mono; @RestController @RequestMapping(Url.DATASOURCE_URL) @@ -15,4 +21,30 @@ public class DatasourceController extends BaseController> testDatasource(@RequestBody Datasource datasource) { + Mono datasourceMono; + + if (datasource.getId() != null) { + datasourceMono = service.getById(datasource.getId()); + } else { + datasourceMono = Mono.just(datasource); + } + + return datasourceMono + .flatMap(service::validateDatasource) + .flatMap(datasource1 -> { + if (CollectionUtils.isEmpty(datasource1.getInvalids())) { + return service.testDatasource(datasource1); + } else { + return Mono.just(new DatasourceTestResult(datasource1.getInvalids())); + } + }) + .map(testResult -> { + ResponseDTO response = new ResponseDTO<>(); + response.setData(testResult); + return response; + }); + } } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/DatasourceContextServiceImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/DatasourceContextServiceImpl.java index c0459dc584..be82f86e4a 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/DatasourceContextServiceImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/DatasourceContextServiceImpl.java @@ -55,18 +55,34 @@ public class DatasourceContextServiceImpl implements DatasourceContextService { Mono pluginMono = datasourceMono .flatMap(resource -> pluginService.findById(resource.getPluginId())); - //Datasource Context has not been created for this resource on this machine. Create one now. + // Datasource Context has not been created for this resource on this machine. Create one now. Mono pluginExecutorMono = pluginExecutorHelper.getPluginExecutor(pluginMono); - return Mono.zip(datasourceMono, pluginExecutorMono, ((datasource1, pluginExecutor) -> { - Object connection = pluginExecutor.datasourceCreate(datasource1.getDatasourceConfiguration()); - DatasourceContext datasourceContext = new DatasourceContext(); - datasourceContext.setConnection(connection); - if (datasource1.getId() != null) { - datasourceContextMap.put(datasourceId, datasourceContext); - } - return datasourceContext; - })); + return Mono.zip(datasourceMono, pluginExecutorMono) + .flatMap(objects -> { + Datasource datasource1 = objects.getT1(); + PluginExecutor pluginExecutor = objects.getT2(); + + DatasourceContext datasourceContext = new DatasourceContext(); + + if (datasource1.getId() != null) { + datasourceContextMap.put(datasourceId, datasourceContext); + } + + Mono connectionMono = pluginExecutor.datasourceCreate(datasource1.getDatasourceConfiguration()); + return connectionMono + .map(connection -> { + // When a connection object exists and makes sense for the plugin, we put it in the + // context. Example, DB plugins. + datasourceContext.setConnection(connection); + return datasourceContext; + }) + .defaultIfEmpty( + // When a connection object doesn't make sense for the plugin, we get an empty mono + // and we just return the context object as is. + datasourceContext + ); + }); } @Override diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/DatasourceService.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/DatasourceService.java index 07ea0c7fd9..9931afd5fe 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/DatasourceService.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/DatasourceService.java @@ -1,5 +1,6 @@ package com.appsmith.server.services; +import com.appsmith.external.models.DatasourceTestResult; import com.appsmith.server.domains.Datasource; import reactor.core.publisher.Mono; @@ -7,6 +8,8 @@ import java.util.Set; public interface DatasourceService extends CrudService { + Mono testDatasource(Datasource datasource); + Mono findByName(String name); Mono findById(String id); diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/DatasourceServiceImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/DatasourceServiceImpl.java index edd25ba220..4eb848847a 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/DatasourceServiceImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/DatasourceServiceImpl.java @@ -1,6 +1,7 @@ package com.appsmith.server.services; import com.appsmith.external.models.DatasourceConfiguration; +import com.appsmith.external.models.DatasourceTestResult; import com.appsmith.external.plugins.PluginExecutor; import com.appsmith.server.constants.FieldName; import com.appsmith.server.domains.Datasource; @@ -157,6 +158,15 @@ public class DatasourceServiceImpl extends BaseService testDatasource(Datasource datasource) { + Mono pluginExecutorMono = pluginExecutorHelper.getPluginExecutor(pluginService.findById(datasource.getPluginId())) + .switchIfEmpty(Mono.error(new AppsmithException(AppsmithError.NO_RESOURCE_FOUND, FieldName.PLUGIN, datasource.getPluginId()))); + + return pluginExecutorMono + .flatMap(pluginExecutor -> pluginExecutor.testDatasource(datasource.getDatasourceConfiguration())); + } + @Override public Mono findByName(String name) { return repository.findByName(name); diff --git a/app/server/appsmith-server/src/test/java/com/appsmith/server/services/DatasourceServiceTest.java b/app/server/appsmith-server/src/test/java/com/appsmith/server/services/DatasourceServiceTest.java index fba3686864..28668dfe51 100644 --- a/app/server/appsmith-server/src/test/java/com/appsmith/server/services/DatasourceServiceTest.java +++ b/app/server/appsmith-server/src/test/java/com/appsmith/server/services/DatasourceServiceTest.java @@ -51,9 +51,9 @@ public class DatasourceServiceTest { } @Override - public Object datasourceCreate(DatasourceConfiguration datasourceConfiguration) { + public Mono datasourceCreate(DatasourceConfiguration datasourceConfiguration) { System.out.println("In the datasourceCreate"); - return null; + return Mono.empty(); } @Override @@ -67,6 +67,11 @@ public class DatasourceServiceTest { System.out.println("In the datasourceValidate"); return new HashSet<>(); } + + @Override + public Mono testDatasource(DatasourceConfiguration datasourceConfiguration) { + return Mono.just(new DatasourceTestResult()); + } } @Before