Protei/PromucFlow_constructor
2
0
Fork 0
Code Issues Pull Requests Actions 4 Packages Projects Releases Wiki Activity

Feature: connect to mongo db via connection string URI (#4131)

Browse Source 
allow user to connect using a mongo connection string URI directly.
This commit is contained in:
Sumit Kumar 2021-05-10 10:35:45 +05:30 committed by GitHub
parent 3573ea3913
commit 0ec3604eac
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 401 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

226
app/server/appsmith-plugins/mongoPlugin/src/main/java/com/external/plugins/MongoPlugin.java vendored
View File

@ -62,6 +62,8 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeoutException;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import static com.appsmith.external.constants.ActionConstants.ACTION_CONFIGURATION_BODY;
@ -89,6 +91,40 @@ public class MongoPlugin extends BasePlugin {
private static final int SMART_BSON_SUBSTITUTION_INDEX = 0;
/*
* - The regex matches the following two pattern types:
* - mongodb+srv://user:pass@some-url/some-db....
* - mongodb://user:pass@some-url:port,some-url:port,../some-db....
* - It has been grouped like this: (mongodb+srv://)((user):(pass))(@some-url/(some-db....))
*/
private static final String MONGO_URI_REGEX = "^(mongodb(\\+srv)?:\\/\\/)((.+):(.+))(@.+\\/(.+))$";
private static final int REGEX_GROUP_HEAD = 1;
private static final int REGEX_GROUP_USERNAME = 4;
private static final int REGEX_GROUP_PASSWORD = 5;
private static final int REGEX_GROUP_TAIL = 6;
private static final int REGEX_GROUP_DBNAME = 7;
private static final String KEY_USERNAME = "username";
private static final String KEY_PASSWORD = "password";
private static final String KEY_URI_HEAD = "uriHead";
private static final String KEY_URI_TAIL = "uriTail";
private static final String KEY_URI_DBNAME = "dbName";
private static final String YES = "Yes";
private static final int DATASOURCE_CONFIG_USE_MONGO_URI_PROPERTY_INDEX = 0;
private static final int DATASOURCE_CONFIG_MONGO_URI_PROPERTY_INDEX = 1;
private static final Integer MONGO_COMMAND_EXCEPTION_UNAUTHORIZED_ERROR_CODE = 13;
public MongoPlugin(PluginWrapper wrapper) {
@ -367,9 +403,83 @@ public class MongoPlugin extends BasePlugin {
.subscribeOn(scheduler);
}
public static String buildClientURI(DatasourceConfiguration datasourceConfiguration) throws AppsmithPluginException {
StringBuilder builder = new StringBuilder();
private boolean isUsingURI(DatasourceConfiguration datasourceConfiguration) {
List<Property> properties = datasourceConfiguration.getProperties();
if (properties != null && properties.size() > DATASOURCE_CONFIG_USE_MONGO_URI_PROPERTY_INDEX
&& properties.get(DATASOURCE_CONFIG_USE_MONGO_URI_PROPERTY_INDEX) != null
&& YES.equals(properties.get(DATASOURCE_CONFIG_USE_MONGO_URI_PROPERTY_INDEX).getValue())) {
return true;
}
return false;
}
private boolean hasNonEmptyURI(DatasourceConfiguration datasourceConfiguration) {
List<Property> properties = datasourceConfiguration.getProperties();
if (properties != null && properties.size() > DATASOURCE_CONFIG_MONGO_URI_PROPERTY_INDEX
&& properties.get(DATASOURCE_CONFIG_MONGO_URI_PROPERTY_INDEX) != null
&& !StringUtils.isEmpty(properties.get(DATASOURCE_CONFIG_MONGO_URI_PROPERTY_INDEX).getValue())) {
return true;
}
return false;
}
private Map extractInfoFromConnectionStringURI(String uri, String regex) {
if (uri.matches(regex)) {
Pattern pattern = Pattern.compile(regex);
Matcher matcher = pattern.matcher(uri);
if (matcher.find()) {
Map extractedInfoMap = new HashMap();
String username = matcher.group(REGEX_GROUP_USERNAME);
extractedInfoMap.put(KEY_USERNAME, username == null ? "" : username);
String password = matcher.group(REGEX_GROUP_PASSWORD);
extractedInfoMap.put(KEY_PASSWORD, password == null ? "" : password);
extractedInfoMap.put(KEY_URI_HEAD, matcher.group(REGEX_GROUP_HEAD));
extractedInfoMap.put(KEY_URI_TAIL, matcher.group(REGEX_GROUP_TAIL));
extractedInfoMap.put(KEY_URI_DBNAME, matcher.group(REGEX_GROUP_DBNAME).split("\\?")[0]);
return extractedInfoMap;
}
}
return null;
}
private String buildURIfromExtractedInfo(Map extractedInfo, String password) {
return extractedInfo.get(KEY_URI_HEAD) + (extractedInfo.get(KEY_USERNAME) == null ? "" :
extractedInfo.get(KEY_USERNAME) + ":") + (password == null ? "" : password)
+ extractedInfo.get(KEY_URI_TAIL);
}
public String buildClientURI(DatasourceConfiguration datasourceConfiguration) throws AppsmithPluginException {
List<Property> properties = datasourceConfiguration.getProperties();
if (isUsingURI(datasourceConfiguration)) {
if (hasNonEmptyURI(datasourceConfiguration)) {
String uriWithHiddenPassword =
(String)properties.get(DATASOURCE_CONFIG_MONGO_URI_PROPERTY_INDEX).getValue();
Map extractedInfo = extractInfoFromConnectionStringURI(uriWithHiddenPassword, MONGO_URI_REGEX);
if (extractedInfo != null) {
String password = ((DBAuth)datasourceConfiguration.getAuthentication()).getPassword();
return buildURIfromExtractedInfo(extractedInfo, password);
}
else {
throw new AppsmithPluginException(
AppsmithPluginError.PLUGIN_DATASOURCE_ARGUMENT_ERROR,
"Appsmith server has failed to parse the Mongo connection string URI. Please check " +
"if the URI has the correct format."
);
}
}
else {
throw new AppsmithPluginException(
AppsmithPluginError.PLUGIN_DATASOURCE_ARGUMENT_ERROR,
"Could not find any Mongo connection string URI. Please edit the 'Mongo Connection String" +
" URI' field to provide the URI to connect to."
);
}
}
StringBuilder builder = new StringBuilder();
final Connection connection = datasourceConfiguration.getConnection();
final List<Endpoint> endpoints = datasourceConfiguration.getEndpoints();
@ -483,52 +593,89 @@ public class MongoPlugin extends BasePlugin {
@Override
public Set<String> validateDatasource(DatasourceConfiguration datasourceConfiguration) {
Set<String> invalids = new HashSet<>();
List<Property> properties = datasourceConfiguration.getProperties();
if (isUsingURI(datasourceConfiguration)) {
if (!hasNonEmptyURI(datasourceConfiguration)) {
invalids.add("'Mongo Connection String URI' field is empty. Please edit the 'Mongo Connection " +
"URI' field to provide a connection uri to connect with.");
} else {
String mongoUri = (String)properties.get(DATASOURCE_CONFIG_MONGO_URI_PROPERTY_INDEX).getValue();
if (!mongoUri.matches(MONGO_URI_REGEX)) {
invalids.add("Mongo Connection String URI does not seem to be in the correct format. Please " +
"check the URI once.");
} else {
Map extractedInfo = extractInfoFromConnectionStringURI(mongoUri, MONGO_URI_REGEX);
if (extractedInfo == null) {
invalids.add("Mongo Connection String URI does not seem to be in the correct format. " +
"Please check the URI once.");
} else {
String mongoUriWithHiddenPassword = buildURIfromExtractedInfo(extractedInfo, "****");
properties.get(DATASOURCE_CONFIG_MONGO_URI_PROPERTY_INDEX).setValue(mongoUriWithHiddenPassword);
DBAuth authentication = datasourceConfiguration.getAuthentication() == null ?
new DBAuth() : (DBAuth) datasourceConfiguration.getAuthentication();
authentication.setUsername((String) extractedInfo.get(KEY_USERNAME));
authentication.setPassword((String) extractedInfo.get(KEY_PASSWORD));
authentication.setDatabaseName((String) extractedInfo.get(KEY_URI_DBNAME));
datasourceConfiguration.setAuthentication(authentication);
List<Endpoint> endpoints = datasourceConfiguration.getEndpoints();
if (CollectionUtils.isEmpty(endpoints)) {
invalids.add("Missing endpoint(s).");
// remove any default db set via form auto-fill via browser
if (datasourceConfiguration.getConnection() != null) {
datasourceConfiguration.getConnection().setDefaultDatabaseName(null);
}
}
}
}
} else {
List<Endpoint> endpoints = datasourceConfiguration.getEndpoints();
if (CollectionUtils.isEmpty(endpoints)) {
invalids.add("Missing endpoint(s).");
} else if (Connection.Type.REPLICA_SET.equals(datasourceConfiguration.getConnection().getType())) {
if (endpoints.size() == 1 && endpoints.get(0).getPort() != null) {
invalids.add("REPLICA_SET connections should not be given a port." +
" If you are trying to specify all the shards, please add more than one.");
}
} else if (Connection.Type.REPLICA_SET.equals(datasourceConfiguration.getConnection().getType())) {
if (endpoints.size() == 1 && endpoints.get(0).getPort() != null) {
invalids.add("REPLICA_SET connections should not be given a port." +
" If you are trying to specify all the shards, please add more than one.");
}
}
if (!CollectionUtils.isEmpty(endpoints)) {
boolean usingUri = endpoints
.stream()
.anyMatch(endPoint -> endPoint.getHost().matches(MONGO_URI_REGEX));
if (!CollectionUtils.isEmpty(endpoints)) {
boolean usingSrvUrl = endpoints
.stream()
.anyMatch(endPoint -> endPoint.getHost().contains("mongodb+srv"));
if (usingSrvUrl) {
invalids.add("MongoDb SRV URLs are not yet supported. Please extract the individual fields from " +
"the SRV URL into the datasource configuration form.");
}
}
DBAuth authentication = (DBAuth) datasourceConfiguration.getAuthentication();
if (authentication != null) {
DBAuth.Type authType = authentication.getAuthType();
if (authType == null || !VALID_AUTH_TYPES.contains(authType)) {
invalids.add("Invalid authType. Must be one of " + VALID_AUTH_TYPES_STR);
if (usingUri) {
invalids.add("It seems that you are trying to use a mongo connection string URI. Please " +
"extract relevant fields and fill the form with extracted values. For " +
"details, please check out the Appsmith's documentation for Mongo database. " +
"Alternatively, you may use 'Import from Connection String URI' option from the " +
"dropdown labelled 'Use Mongo Connection String URI' to use the URI connection string" +
" directly.");
}
}
if (StringUtils.isEmpty(authentication.getDatabaseName())) {
invalids.add("Missing database name.");
DBAuth authentication = (DBAuth) datasourceConfiguration.getAuthentication();
if (authentication != null) {
DBAuth.Type authType = authentication.getAuthType();
if (authType == null || !VALID_AUTH_TYPES.contains(authType)) {
invalids.add("Invalid authType. Must be one of " + VALID_AUTH_TYPES_STR);
}
if (StringUtils.isEmpty(authentication.getDatabaseName())) {
invalids.add("Missing database name.");
}
}
}
/*
* - Ideally, it is never expected to be null because the SSL dropdown is set to a initial value.
*/
if (datasourceConfiguration.getConnection() == null
|| datasourceConfiguration.getConnection().getSsl() == null
|| datasourceConfiguration.getConnection().getSsl().getAuthType() == null) {
invalids.add("Appsmith server has failed to fetch SSL configuration from datasource configuration " +
"form. Please reach out to Appsmith customer support to resolve this.");
/*
* - Ideally, it is never expected to be null because the SSL dropdown is set to a initial value.
*/
if (datasourceConfiguration.getConnection() == null
|| datasourceConfiguration.getConnection().getSsl() == null
|| datasourceConfiguration.getConnection().getSsl().getAuthType() == null) {
invalids.add("Appsmith server has failed to fetch SSL configuration from datasource configuration " +
"form. Please reach out to Appsmith customer support to resolve this.");
}
}
return invalids;
@ -581,6 +728,7 @@ public class MongoPlugin extends BasePlugin {
final DatasourceStructure structure = new DatasourceStructure();
List<DatasourceStructure.Table> tables = new ArrayList<>();
structure.setTables(tables);
final MongoDatabase database = mongoClient.getDatabase(getDatabaseName(datasourceConfiguration));
return Flux.from(database.listCollectionNames())

88
app/server/appsmith-plugins/mongoPlugin/src/main/resources/form.json
View File

@ -3,6 +3,47 @@
{
"sectionName": "Connection",
"children": [
{
"label": "Use Mongo Connection String URI Key",
"configProperty": "datasourceConfiguration.properties[0].key",
"controlType": "INPUT_TEXT",
"initialValue": "Use Mongo Connection String URI",
"hidden": true
},
{
"label": "Use Mongo Connection String URI",
"configProperty": "datasourceConfiguration.properties[0].value",
"controlType": "DROP_DOWN",
"initialValue": "No",
"options": [
{
"label": "Yes",
"value": "Yes"
},
{
"label": "No",
"value": "No"
}
]
},
{
"label": "Connection String URI Key",
"configProperty": "datasourceConfiguration.properties[1].key",
"controlType": "INPUT_TEXT",
"initialValue": "Connection String URI",
"hidden": true
},
{
"label": "Connection String URI",
"placeholderText": "mongodb+srv://<username>:<password>@test-db.swrsq.mongodb.net/myDatabase",
"configProperty": "datasourceConfiguration.properties[1].value",
"controlType": "INPUT_TEXT",
"hidden": {
"path": "datasourceConfiguration.properties[0].value",
"comparison": "NOT_EQUALS",
"value": "Yes"
}
},
{
"label": "Connection Mode",
"configProperty": "datasourceConfiguration.connection.mode",
@ -17,7 +58,12 @@
"label": "Read / Write",
"value": "READ_WRITE"
}
]
],
"hidden": {
"path": "datasourceConfiguration.properties[0].value",
"comparison": "EQUALS",
"value": "Yes"
}
},
{
"label": "Connection Type",
@ -33,7 +79,12 @@
"label": "Replica set",
"value": "REPLICA_SET"
}
]
],
"hidden": {
"path": "datasourceConfiguration.properties[0].value",
"comparison": "EQUALS",
"value": "Yes"
}
},
{
"sectionName": null,
@ -44,13 +95,23 @@
"controlType": "KEYVALUE_ARRAY",
"validationMessage": "Please enter a valid host",
"validationRegex": "^((?![/:]).)*$",
"placeholderText": "myapp.abcde.mongodb.net"
"placeholderText": "myapp.abcde.mongodb.net",
"hidden": {
"path": "datasourceConfiguration.properties[0].value",
"comparison": "EQUALS",
"value": "Yes"
}
},
{
"label": "Port",
"configProperty": "datasourceConfiguration.endpoints[*].port",
"dataType": "NUMBER",
"controlType": "KEYVALUE_ARRAY"
"controlType": "KEYVALUE_ARRAY",
"hidden": {
"path": "datasourceConfiguration.properties[0].value",
"comparison": "EQUALS",
"value": "Yes"
}
}
]
},
@ -58,12 +119,22 @@
"label": "Default Database Name",
"placeholderText": "(Optional)",
"configProperty": "datasourceConfiguration.connection.defaultDatabaseName",
"controlType": "INPUT_TEXT"
"controlType": "INPUT_TEXT",
"hidden": {
"path": "datasourceConfiguration.properties[0].value",
"comparison": "EQUALS",
"value": "Yes"
}
}
]
},
{
"sectionName": "Authentication",
"hidden": {
"path": "datasourceConfiguration.properties[0].value",
"comparison": "EQUALS",
"value": "Yes"
},
"children": [
{
"label": "Database Name",
@ -108,13 +179,18 @@
"controlType": "INPUT_TEXT",
"placeholderText": "Password",
"encrypted": true
}
}
]
}
]
},
{
"sectionName": "SSL (optional)",
"hidden": {
"path": "datasourceConfiguration.properties[0].value",
"comparison": "EQUALS",
"value": "Yes"
},
"children": [
{
"label": "SSL Mode",

119
app/server/appsmith-plugins/mongoPlugin/src/test/java/com/external/plugins/MongoPluginTest.java vendored
View File

@ -473,16 +473,129 @@ public class MongoPluginTest {
}
@Test
public void testErrorMessageOnSrvUrl() {
public void testErrorMessageOnSrvUriWithFormInterface() {
DatasourceConfiguration dsConfig = createDatasourceConfiguration();
dsConfig.getEndpoints().get(0).setHost("mongodb+srv:://url.net");
dsConfig.getEndpoints().get(0).setHost("mongodb+srv://user:pass@url.net/dbName");
dsConfig.setProperties(List.of(new Property("Import from URI", "No")));
Mono<Set<String>> invalidsMono = Mono.just(pluginExecutor.validateDatasource(dsConfig));
StepVerifier.create(invalidsMono)
.assertNext(invalids -> {
assertTrue(invalids
.stream()
.anyMatch(error -> error.contains("MongoDb SRV URLs are not yet supported")));
.anyMatch(error -> error.contains("It seems that you are trying to use a mongo connection" +
" string URI. Please extract relevant fields and fill the form with extracted " +
"values. For details, please check out the Appsmith's documentation for Mongo " +
"database. Alternatively, you may use 'Import from Connection String URI' option " +
"from the dropdown labelled 'Use Mongo Connection String URI' to use the URI " +
"connection string directly.")));
})
.verifyComplete();
}
@Test
public void testErrorMessageOnNonSrvUri() {
DatasourceConfiguration dsConfig = createDatasourceConfiguration();
dsConfig.getEndpoints().get(0).setHost("mongodb://user:pass@url.net:1234,url.net:1234/dbName");
dsConfig.setProperties(List.of(new Property("Import from URI", "No")));
Mono<Set<String>> invalidsMono = Mono.just(pluginExecutor.validateDatasource(dsConfig));
StepVerifier.create(invalidsMono)
.assertNext(invalids -> {
assertTrue(invalids
.stream()
.anyMatch(error -> error.contains("It seems that you are trying to use a mongo connection" +
" string URI. Please extract relevant fields and fill the form with extracted " +
"values. For details, please check out the Appsmith's documentation for Mongo " +
"database. Alternatively, you may use 'Import from Connection String URI' option " +
"from the dropdown labelled 'Use Mongo Connection String URI' to use the URI " +
"connection string directly.")));
})
.verifyComplete();
}
@Test
public void testInvalidsOnMissingUri() {
DatasourceConfiguration dsConfig = createDatasourceConfiguration();
dsConfig.setProperties(List.of(new Property("Import from URI", "Yes")));
Mono<Set<String>> invalidsMono = Mono.just(pluginExecutor.validateDatasource(dsConfig));
StepVerifier.create(invalidsMono)
.assertNext(invalids -> {
assertTrue(invalids
.stream()
.anyMatch(error -> error.contains("'Mongo Connection String URI' field is empty. Please " +
"edit the 'Mongo Connection URI' field to provide a connection uri to connect with.")));
})
.verifyComplete();
}
@Test
public void testInvalidsOnBadSrvUriFormat() {
DatasourceConfiguration dsConfig = createDatasourceConfiguration();
List<Property> properties = new ArrayList<>();
properties.add(new Property("Import from URI", "Yes"));
properties.add(new Property("Srv Url", "mongodb+srv::username:password//url.net"));
dsConfig.setProperties(properties);
Mono<Set<String>> invalidsMono = Mono.just(pluginExecutor.validateDatasource(dsConfig));
StepVerifier.create(invalidsMono)
.assertNext(invalids -> {
assertTrue(invalids
.stream()
.anyMatch(error -> error.contains("Mongo Connection String URI does not seem to be in the" +
" correct format. Please check the URI once.")));
})
.verifyComplete();
}
@Test
public void testInvalidsOnBadNonSrvUriFormat() {
DatasourceConfiguration dsConfig = createDatasourceConfiguration();
List<Property> properties = new ArrayList<>();
properties.add(new Property("Import from URI", "Yes"));
properties.add(new Property("Srv Url", "mongodb::username:password//url.net"));
dsConfig.setProperties(properties);
Mono<Set<String>> invalidsMono = Mono.just(pluginExecutor.validateDatasource(dsConfig));
StepVerifier.create(invalidsMono)
.assertNext(invalids -> {
assertTrue(invalids
.stream()
.anyMatch(error -> error.contains("Mongo Connection String URI does not seem to be in the" +
" correct format. Please check the URI once.")));
})
.verifyComplete();
}
@Test
public void testInvalidsEmptyOnCorrectSrvUriFormat() {
DatasourceConfiguration dsConfig = createDatasourceConfiguration();
List<Property> properties = new ArrayList<>();
properties.add(new Property("Import from URI", "Yes"));
properties.add(new Property("Srv Url", "mongodb+srv://username:password@url.net/dbname"));
dsConfig.setProperties(properties);
Mono<Set<String>> invalidsMono = Mono.just(pluginExecutor.validateDatasource(dsConfig));
StepVerifier.create(invalidsMono)
.assertNext(invalids -> {
assertTrue(invalids.isEmpty());
})
.verifyComplete();
}
@Test
public void testInvalidsEmptyOnCorrectNonSrvUriFormat() {
DatasourceConfiguration dsConfig = createDatasourceConfiguration();
List<Property> properties = new ArrayList<>();
properties.add(new Property("Import from URI", "Yes"));
properties.add(new Property("Srv Url", "mongodb://username:password@url-1.net:1234,url-2:1234/dbname"));
dsConfig.setProperties(properties);
Mono<Set<String>> invalidsMono = Mono.just(pluginExecutor.validateDatasource(dsConfig));
StepVerifier.create(invalidsMono)
.assertNext(invalids -> {
assertTrue(invalids.isEmpty());
})
.verifyComplete();
}

16
app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/DatabaseChangelog.java
View File

@ -2242,4 +2242,20 @@ public class DatabaseChangelog {
NewAction.class
);
}
@ChangeSet(order = "067", id = "update-mongo-import-from-srv-field", author = "")
public void updateMongoImportFromSrvField(MongoTemplate mongoTemplate) {
Plugin mongoPlugin = mongoTemplate
.findOne(query(where("packageName").is("mongo-plugin")), Plugin.class);
List<Datasource> mongoDatasources = mongoTemplate
.find(query(where("pluginId").is(mongoPlugin.getId())), Datasource.class);
mongoDatasources.stream()
.forEach(datasource -> {
datasource.getDatasourceConfiguration().setProperties(List.of(new Property("Use Mongo Connection " +
"String URI", "No")));
mongoTemplate.save(datasource);
});
}
}