Protei/PromucFlow_constructor
2
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Test API for data sources

Browse Source
This commit is contained in:
Shrikant Kandula 2020-04-15 10:02:09 +00:00
parent 72540715e3
commit 2dbf9d1c6b
11 changed files with 226 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/DatasourceTestResult.java vendored Normal file
View File

@ -0,0 +1,35 @@
package com.appsmith.external.models;
import lombok.Getter;
import lombok.Setter;
import lombok.ToString;
import org.springframework.util.CollectionUtils;
import java.util.Set;
@Getter
@Setter
@ToString
public class DatasourceTestResult {
Set<String> invalids;
/**
* Convenience constructor to create a result object with one or more error messages. This constructor also ensures
* that the `invalids` field is never null.
* @param invalids String messages that explain why the test failed.
*/
public DatasourceTestResult(String... invalids) {
this.invalids = Set.of(invalids);
}
public DatasourceTestResult(Set<String> invalids) {
this.invalids = invalids;
}
public boolean isSuccess() {
// This method exists so that a `"success"` boolean key is present in the JSON response to the frontend.
return CollectionUtils.isEmpty(invalids);
}
}

5
app/server/appsmith-interfaces/src/main/java/com/appsmith/external/plugins/PluginExecutor.java vendored
View File

@ -2,6 +2,7 @@ package com.appsmith.external.plugins;
import com.appsmith.external.models.ActionConfiguration;
import com.appsmith.external.models.DatasourceConfiguration;
import com.appsmith.external.models.DatasourceTestResult;
import org.pf4j.ExtensionPoint;
import org.springframework.util.CollectionUtils;
import reactor.core.publisher.Mono;
@ -28,7 +29,7 @@ public interface PluginExecutor extends ExtensionPoint {
* @param datasourceConfiguration
* @return Connection object
*/
Object datasourceCreate(DatasourceConfiguration datasourceConfiguration);
Mono<Object> datasourceCreate(DatasourceConfiguration datasourceConfiguration);
/**
* This function is used to bring down/destroy the connection to the data source.
@ -42,4 +43,6 @@ public interface PluginExecutor extends ExtensionPoint {
}
Set<String> validateDatasource(DatasourceConfiguration datasourceConfiguration);
Mono<DatasourceTestResult> testDatasource(DatasourceConfiguration datasourceConfiguration);
}

11
app/server/appsmith-plugins/mongoPlugin/src/main/java/com/external/plugins/MongoPlugin.java vendored
View File

@ -3,6 +3,7 @@ package com.external.plugins;
import com.appsmith.external.models.ActionConfiguration;
import com.appsmith.external.models.ActionExecutionResult;
import com.appsmith.external.models.DatasourceConfiguration;
import com.appsmith.external.models.DatasourceTestResult;
import com.appsmith.external.plugins.BasePlugin;
import com.appsmith.external.plugins.PluginExecutor;
import com.fasterxml.jackson.databind.ObjectMapper;
@ -114,10 +115,9 @@ public class MongoPlugin extends BasePlugin {
}
@Override
public Object datasourceCreate(DatasourceConfiguration datasourceConfiguration) {
public Mono<Object> datasourceCreate(DatasourceConfiguration datasourceConfiguration) {
MongoClientURI mongoClientURI = new MongoClientURI(datasourceConfiguration.getUrl());
return new MongoClient(mongoClientURI);
return Mono.just(new MongoClient(mongoClientURI));
}
@Override
@ -134,6 +134,11 @@ public class MongoPlugin extends BasePlugin {
return new HashSet<>();
}
@Override
public Mono<DatasourceTestResult> testDatasource(DatasourceConfiguration datasourceConfiguration) {
return Mono.just(new DatasourceTestResult());
}
}
}

77
app/server/appsmith-plugins/postgresPlugin/src/main/java/com/external/plugins/PostgresPlugin.java vendored
View File

@ -1,6 +1,11 @@
package com.external.plugins;
import com.appsmith.external.models.*;
import com.appsmith.external.models.ActionConfiguration;
import com.appsmith.external.models.ActionExecutionResult;
import com.appsmith.external.models.AuthenticationDTO;
import com.appsmith.external.models.DatasourceConfiguration;
import com.appsmith.external.models.DatasourceTestResult;
import com.appsmith.external.models.Endpoint;
import com.appsmith.external.pluginExceptions.AppsmithPluginError;
import com.appsmith.external.pluginExceptions.AppsmithPluginException;
import com.appsmith.external.plugins.BasePlugin;
@ -16,9 +21,18 @@ import org.springframework.util.StringUtils;
import reactor.core.publisher.Mono;
import java.sql.Connection;
import java.sql.*;
import java.util.*;
import java.util.stream.Stream;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import static com.appsmith.external.models.Connection.Mode.READ_ONLY;
@ -64,9 +78,11 @@ public class PostgresPlugin extends BasePlugin {
List<Map<String, Object>> rowsList = new ArrayList<>(50);
Statement statement = null;
ResultSet resultSet = null;
try {
Statement statement = conn.createStatement();
ResultSet resultSet = statement.executeQuery(query);
statement = conn.createStatement();
resultSet = statement.executeQuery(query);
ResultSetMetaData metaData = resultSet.getMetaData();
int colCount = metaData.getColumnCount();
while (resultSet.next()) {
@ -80,12 +96,29 @@ public class PostgresPlugin extends BasePlugin {
} catch (SQLException e) {
return pluginErrorMono(e);
} finally {
if (resultSet != null) {
try {
resultSet.close();
} catch (SQLException e) {
log.warn("Error closing Postgres ResultSet", e);
}
}
if (statement != null) {
try {
statement.close();
} catch (SQLException e) {
log.warn("Error closing Postgres Statement", e);
}
}
}
ActionExecutionResult result = new ActionExecutionResult();
result.setBody(objectMapper.valueToTree(rowsList));
result.setShouldCacheResponse(true);
System.out.println("In the PostgresPlugin, got action execution result: " + result.toString());
log.debug("In the PostgresPlugin, got action execution result: " + result.toString());
return Mono.just(result);
}
@ -94,7 +127,7 @@ public class PostgresPlugin extends BasePlugin {
}
@Override
public Object datasourceCreate(DatasourceConfiguration datasourceConfiguration) {
public Mono<Object> datasourceCreate(DatasourceConfiguration datasourceConfiguration) {
try {
Class.forName(JDBC_DRIVER);
} catch (ClassNotFoundException e) {
@ -104,12 +137,14 @@ public class PostgresPlugin extends BasePlugin {
String url;
AuthenticationDTO authentication = datasourceConfiguration.getAuthentication();
com.appsmith.external.models.Connection configurationConnection = datasourceConfiguration.getConnection();
Properties properties = new Properties();
properties.putAll(Map.of(
USER, authentication.getUsername(),
PASSWORD, authentication.getPassword(),
// TODO: Set SSL connection parameters.
SSL, datasourceConfiguration.getConnection().getSsl() != null
SSL, configurationConnection != null && configurationConnection.getSsl() != null
));
if (CollectionUtils.isEmpty(datasourceConfiguration.getEndpoints())) {
@ -131,11 +166,12 @@ public class PostgresPlugin extends BasePlugin {
try {
Connection connection = DriverManager.getConnection(url, properties);
connection.setReadOnly(READ_ONLY.equals(datasourceConfiguration.getConnection().getMode()));
return connection;
connection.setReadOnly(
configurationConnection != null && READ_ONLY.equals(configurationConnection.getMode()));
return Mono.just(connection);
} catch (SQLException e) {
return pluginErrorMono("Error connecting to Postgres.");
return pluginErrorMono("Error connecting to Postgres.", e);
}
}
@ -186,6 +222,23 @@ public class PostgresPlugin extends BasePlugin {
return invalids;
}
@Override
public Mono<DatasourceTestResult> testDatasource(DatasourceConfiguration datasourceConfiguration) {
return datasourceCreate(datasourceConfiguration)
.map(connection -> {
try {
if (connection != null) {
((Connection) connection).close();
}
} catch (SQLException e) {
log.warn("Error closing Postgres connection that was made for testing.", e);
}
return new DatasourceTestResult();
})
.onErrorResume(error -> Mono.just(new DatasourceTestResult(error.getMessage())));
}
}
}

14
app/server/appsmith-plugins/rapidApiPlugin/src/main/java/com/external/plugins/RapidApiPlugin.java vendored
View File

@ -1,9 +1,6 @@
package com.external.plugins;
import com.appsmith.external.models.ActionConfiguration;
import com.appsmith.external.models.ActionExecutionResult;
import com.appsmith.external.models.DatasourceConfiguration;
import com.appsmith.external.models.Property;
import com.appsmith.external.models.*;
import com.appsmith.external.pluginExceptions.AppsmithPluginError;
import com.appsmith.external.pluginExceptions.AppsmithPluginException;
import com.appsmith.external.plugins.BasePlugin;
@ -238,8 +235,8 @@ public class RapidApiPlugin extends BasePlugin {
}
@Override
public Object datasourceCreate(DatasourceConfiguration datasourceConfiguration) {
return null;
public Mono<Object> datasourceCreate(DatasourceConfiguration datasourceConfiguration) {
return Mono.empty();
}
@Override
@ -254,6 +251,11 @@ public class RapidApiPlugin extends BasePlugin {
return new HashSet<>();
}
@Override
public Mono<DatasourceTestResult> testDatasource(DatasourceConfiguration datasourceConfiguration) {
return Mono.just(new DatasourceTestResult());
}
private void addHeadersToRequest(WebClient.Builder webClientBuilder, List<Property> headers) {
for (Property header : headers) {
if (header.getKey() != null && !header.getKey().isEmpty()) {

30
app/server/appsmith-plugins/restApiPlugin/src/main/java/com/external/plugins/RestApiPlugin.java vendored
View File

@ -3,6 +3,7 @@ package com.external.plugins;
import com.appsmith.external.models.ActionConfiguration;
import com.appsmith.external.models.ActionExecutionResult;
import com.appsmith.external.models.DatasourceConfiguration;
import com.appsmith.external.models.DatasourceTestResult;
import com.appsmith.external.models.Property;
import com.appsmith.external.pluginExceptions.AppsmithPluginError;
import com.appsmith.external.pluginExceptions.AppsmithPluginException;
@ -27,6 +28,9 @@ import org.springframework.web.util.UriComponentsBuilder;
import reactor.core.publisher.Mono;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.MalformedURLException;
import java.net.Socket;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
@ -208,8 +212,8 @@ public class RestApiPlugin extends BasePlugin {
}
@Override
public Object datasourceCreate(DatasourceConfiguration datasourceConfiguration) {
return null;
public Mono<Object> datasourceCreate(DatasourceConfiguration datasourceConfiguration) {
return Mono.empty();
}
@Override
@ -235,6 +239,28 @@ public class RestApiPlugin extends BasePlugin {
return invalids;
}
@Override
public Mono<DatasourceTestResult> testDatasource(DatasourceConfiguration datasourceConfiguration) {
URL url;
try {
url = new URL(datasourceConfiguration.getUrl());
} catch (MalformedURLException e) {
return Mono.just(new DatasourceTestResult("Invalid URL: '" + e.getMessage() + "'."));
}
try (Socket socket = new Socket()) {
socket.connect(new InetSocketAddress(url.getHost(), url.getPort()), 300);
} catch (IOException e) {
return Mono.just(
new DatasourceTestResult("Failed to reach API endpoint: '" + e.getMessage() + "'.")
);
}
return Mono.just(new DatasourceTestResult());
}
private boolean addHeadersToRequestAndAscertainContentType(WebClient.Builder webClientBuilder,
List<Property> headers,
boolean isContentTypeJson) {

32
app/server/appsmith-server/src/main/java/com/appsmith/server/controllers/DatasourceController.java
View File

@ -1,11 +1,17 @@
package com.appsmith.server.controllers;
import com.appsmith.external.models.DatasourceTestResult;
import com.appsmith.server.constants.Url;
import com.appsmith.server.domains.Datasource;
import com.appsmith.server.dtos.ResponseDTO;
import com.appsmith.server.services.DatasourceService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.util.CollectionUtils;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import reactor.core.publisher.Mono;
@RestController
@RequestMapping(Url.DATASOURCE_URL)
@ -15,4 +21,30 @@ public class DatasourceController extends BaseController<DatasourceService, Data
public DatasourceController(DatasourceService service) {
super(service);
}
@PostMapping("/test")
public Mono<ResponseDTO<DatasourceTestResult>> testDatasource(@RequestBody Datasource datasource) {
Mono<Datasource> datasourceMono;
if (datasource.getId() != null) {
datasourceMono = service.getById(datasource.getId());
} else {
datasourceMono = Mono.just(datasource);
}
return datasourceMono
.flatMap(service::validateDatasource)
.flatMap(datasource1 -> {
if (CollectionUtils.isEmpty(datasource1.getInvalids())) {
return service.testDatasource(datasource1);
} else {
return Mono.just(new DatasourceTestResult(datasource1.getInvalids()));
}
})
.map(testResult -> {
ResponseDTO<DatasourceTestResult> response = new ResponseDTO<>();
response.setData(testResult);
return response;
});
}
}

36
app/server/appsmith-server/src/main/java/com/appsmith/server/services/DatasourceContextServiceImpl.java
View File

@ -55,18 +55,34 @@ public class DatasourceContextServiceImpl implements DatasourceContextService {
Mono<Plugin> pluginMono = datasourceMono
.flatMap(resource -> pluginService.findById(resource.getPluginId()));
//Datasource Context has not been created for this resource on this machine. Create one now.
// Datasource Context has not been created for this resource on this machine. Create one now.
Mono<PluginExecutor> pluginExecutorMono = pluginExecutorHelper.getPluginExecutor(pluginMono);
return Mono.zip(datasourceMono, pluginExecutorMono, ((datasource1, pluginExecutor) -> {
Object connection = pluginExecutor.datasourceCreate(datasource1.getDatasourceConfiguration());
DatasourceContext datasourceContext = new DatasourceContext();
datasourceContext.setConnection(connection);
if (datasource1.getId() != null) {
datasourceContextMap.put(datasourceId, datasourceContext);
}
return datasourceContext;
}));
return Mono.zip(datasourceMono, pluginExecutorMono)
.flatMap(objects -> {
Datasource datasource1 = objects.getT1();
PluginExecutor pluginExecutor = objects.getT2();
DatasourceContext datasourceContext = new DatasourceContext();
if (datasource1.getId() != null) {
datasourceContextMap.put(datasourceId, datasourceContext);
}
Mono<Object> connectionMono = pluginExecutor.datasourceCreate(datasource1.getDatasourceConfiguration());
return connectionMono
.map(connection -> {
// When a connection object exists and makes sense for the plugin, we put it in the
// context. Example, DB plugins.
datasourceContext.setConnection(connection);
return datasourceContext;
})
.defaultIfEmpty(
// When a connection object doesn't make sense for the plugin, we get an empty mono
// and we just return the context object as is.
datasourceContext
);
});
}
@Override

3
app/server/appsmith-server/src/main/java/com/appsmith/server/services/DatasourceService.java
View File

@ -1,5 +1,6 @@
package com.appsmith.server.services;
import com.appsmith.external.models.DatasourceTestResult;
import com.appsmith.server.domains.Datasource;
import reactor.core.publisher.Mono;
@ -7,6 +8,8 @@ import java.util.Set;
public interface DatasourceService extends CrudService<Datasource, String> {
Mono<DatasourceTestResult> testDatasource(Datasource datasource);
Mono<Datasource> findByName(String name);
Mono<Datasource> findById(String id);

10
app/server/appsmith-server/src/main/java/com/appsmith/server/services/DatasourceServiceImpl.java
View File

@ -1,6 +1,7 @@
package com.appsmith.server.services;
import com.appsmith.external.models.DatasourceConfiguration;
import com.appsmith.external.models.DatasourceTestResult;
import com.appsmith.external.plugins.PluginExecutor;
import com.appsmith.server.constants.FieldName;
import com.appsmith.server.domains.Datasource;
@ -157,6 +158,15 @@ public class DatasourceServiceImpl extends BaseService<DatasourceRepository, Dat
.flatMap(repository::save);
}
@Override
public Mono<DatasourceTestResult> testDatasource(Datasource datasource) {
Mono<PluginExecutor> pluginExecutorMono = pluginExecutorHelper.getPluginExecutor(pluginService.findById(datasource.getPluginId()))
.switchIfEmpty(Mono.error(new AppsmithException(AppsmithError.NO_RESOURCE_FOUND, FieldName.PLUGIN, datasource.getPluginId())));
return pluginExecutorMono
.flatMap(pluginExecutor -> pluginExecutor.testDatasource(datasource.getDatasourceConfiguration()));
}
@Override
public Mono<Datasource> findByName(String name) {
return repository.findByName(name);

9
app/server/appsmith-server/src/test/java/com/appsmith/server/services/DatasourceServiceTest.java
View File

@ -51,9 +51,9 @@ public class DatasourceServiceTest {
}
@Override
public Object datasourceCreate(DatasourceConfiguration datasourceConfiguration) {
public Mono<Object> datasourceCreate(DatasourceConfiguration datasourceConfiguration) {
System.out.println("In the datasourceCreate");
return null;
return Mono.empty();
}
@Override
@ -67,6 +67,11 @@ public class DatasourceServiceTest {
System.out.println("In the datasourceValidate");
return new HashSet<>();
}
@Override
public Mono<DatasourceTestResult> testDatasource(DatasourceConfiguration datasourceConfiguration) {
return Mono.just(new DatasourceTestResult());
}
}
@Before