Protei/PromucFlow_constructor
2
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

fix: add connection pool to MySQL plugin and fix issues due to Spring upgrade (#17873)

Browse Source 
- Add connection pool to MySQL
- Fix JUnit TC failures due to Spring upgrade
- Fix Cypress TC failures due to change in the MySQL plugin code 
- Remove `Preferred` SSL option
This commit is contained in:
Sumit Kumar 2023-01-09 14:04:51 +05:30 committed by GitHub
parent 6c11c9334a
commit 03f8b2a523
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 1977 additions and 2040 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
app/client/cypress/integration/Smoke_TestSuite/ServerSideTests/Datasources/MySQLNoiseTest_spec.js
View File

@ -63,10 +63,8 @@ describe("MySQL noise test", function() {
expect(response.body.data.statusCode).to.eq("200 OK");
});
cy.wait("@postExecute").then(({ response }) => {
expect(response.body.data.statusCode).to.eq("5004");
expect(response.body.data.title).to.eq(
"Datasource configuration is invalid",
);
expect(response.body.data.statusCode).to.eq("5000");
expect(response.body.data.title).to.eq("Query execution error");
});
});
});

50
app/server/appsmith-plugins/mysqlPlugin/pom.xml
View File

@ -24,27 +24,6 @@
</properties>
<dependencies>
<dependency>
<groupId>org.springframework.data</groupId>
<artifactId>spring-data-r2dbc</artifactId>
<version>1.1.5.RELEASE</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</exclusion>
<exclusion>
<groupId>io.projectreactor</groupId>
<artifactId>reactor-core</artifactId>
</exclusion>
<exclusion>
<groupId>org.reactivestreams</groupId>
<artifactId>reactive-streams</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.mariadb</groupId>
<artifactId>r2dbc-mariadb</artifactId>
@ -85,6 +64,35 @@
<version>4.1.75.Final</version>
</dependency>
<dependency>
<groupId>io.r2dbc</groupId>
<artifactId>r2dbc-pool</artifactId>
<!--
Please be careful when upgrading package version from 0.8.x to 0.9.x or higher since 0.9.x version
seems incompatible with r2dbc-mysql version 0.8.2.RELEASE because 0.9.x version contains a higher
version of one of the dependent packages (not able to remember the name of the package at the moment)
-->
<version>0.8.8.RELEASE</version>
<exclusions>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</exclusion>
<exclusion>
<groupId>io.projectreactor</groupId>
<artifactId>reactor-core</artifactId>
</exclusion>
<exclusion>
<groupId>org.reactivestreams</groupId>
<artifactId>reactive-streams</artifactId>
</exclusion>
</exclusions>
</dependency>
<!-- Test Dependencies -->
<dependency>
<groupId>mysql</groupId>

606
app/server/appsmith-plugins/mysqlPlugin/src/main/java/com/external/plugins/MySqlPlugin.java vendored
View File

@ -10,26 +10,24 @@ import com.appsmith.external.helpers.MustacheHelper;
import com.appsmith.external.models.ActionConfiguration;
import com.appsmith.external.models.ActionExecutionRequest;
import com.appsmith.external.models.ActionExecutionResult;
import com.appsmith.external.models.DBAuth;
import com.appsmith.external.models.DatasourceConfiguration;
import com.appsmith.external.models.DatasourceStructure;
import com.appsmith.external.models.Endpoint;
import com.appsmith.external.models.DatasourceTestResult;
import com.appsmith.external.models.MustacheBindingToken;
import com.appsmith.external.models.Param;
import com.appsmith.external.models.Property;
import com.appsmith.external.models.PsParameterDTO;
import com.appsmith.external.models.RequestParamDTO;
import com.appsmith.external.models.SSLDetails;
import com.appsmith.external.plugins.BasePlugin;
import com.appsmith.external.plugins.PluginExecutor;
import com.appsmith.external.plugins.SmartSubstitutionInterface;
import com.external.plugins.datatypes.MySQLSpecificDataTypes;
import com.external.utils.MySqlDatasourceUtils;
import com.external.utils.QueryUtils;
import io.r2dbc.pool.ConnectionPool;
import io.r2dbc.spi.ColumnMetadata;
import io.r2dbc.spi.Connection;
import io.r2dbc.spi.ConnectionFactories;
import io.r2dbc.spi.ConnectionFactoryOptions;
import io.r2dbc.spi.Option;
import io.r2dbc.spi.R2dbcNonTransientResourceException;
import io.r2dbc.spi.Result;
import io.r2dbc.spi.Row;
import io.r2dbc.spi.RowMetadata;
@ -37,15 +35,14 @@ import io.r2dbc.spi.Statement;
import io.r2dbc.spi.ValidationDepth;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang.ObjectUtils;
import org.mariadb.r2dbc.MariadbConnectionFactoryProvider;
import org.pf4j.Extension;
import org.pf4j.PluginWrapper;
import org.springframework.util.CollectionUtils;
import org.springframework.util.StringUtils;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import reactor.core.scheduler.Scheduler;
import reactor.core.scheduler.Schedulers;
import reactor.pool.PoolShutdownException;
import java.time.Duration;
import java.time.LocalDate;
@ -64,7 +61,6 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeoutException;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import static com.appsmith.external.constants.ActionConstants.ACTION_CONFIGURATION_BODY;
@ -72,16 +68,16 @@ import static com.appsmith.external.helpers.PluginUtils.MATCH_QUOTED_WORDS_REGEX
import static com.appsmith.external.helpers.PluginUtils.getIdenticalColumns;
import static com.appsmith.external.helpers.PluginUtils.getPSParamLabel;
import static com.appsmith.external.helpers.SmartSubstitutionHelper.replaceQuestionMarkWithDollarIndex;
import static io.r2dbc.spi.ConnectionFactoryOptions.SSL;
import static com.external.utils.MySqlDatasourceUtils.getNewConnectionPool;
import static com.external.utils.MySqlGetStructureUtils.getKeyInfo;
import static com.external.utils.MySqlGetStructureUtils.getTableInfo;
import static com.external.utils.MySqlGetStructureUtils.getTemplates;
import static java.lang.Boolean.FALSE;
import static java.lang.Boolean.TRUE;
@Slf4j
public class MySqlPlugin extends BasePlugin {
private static final String DATE_COLUMN_TYPE_NAME = "date";
private static final String DATETIME_COLUMN_TYPE_NAME = "datetime";
private static final String TIMESTAMP_COLUMN_TYPE_NAME = "timestamp";
private static final int VALIDATION_CHECK_TIMEOUT = 4; // seconds
private static final String IS_KEY = "is";
@ -142,7 +138,7 @@ public class MySqlPlugin extends BasePlugin {
}
@Extension
public static class MySqlPluginExecutor implements PluginExecutor<Connection>, SmartSubstitutionInterface {
public static class MySqlPluginExecutor implements PluginExecutor<ConnectionPool>, SmartSubstitutionInterface {
private static final int PREPARED_STATEMENT_INDEX = 0;
private final Scheduler scheduler = Schedulers.boundedElastic();
@ -162,7 +158,7 @@ public class MySqlPlugin extends BasePlugin {
* @return
*/
@Override
public Mono<ActionExecutionResult> executeParameterized(Connection connection,
public Mono<ActionExecutionResult> executeParameterized(ConnectionPool connection,
ExecuteActionDTO executeActionDTO,
DatasourceConfiguration datasourceConfiguration,
ActionConfiguration actionConfiguration) {
@ -223,7 +219,7 @@ public class MySqlPlugin extends BasePlugin {
return executeCommon(connection, actionConfiguration, TRUE, mustacheKeysInOrder, executeActionDTO, requestData);
}
public Mono<ActionExecutionResult> executeCommon(Connection connection,
public Mono<ActionExecutionResult> executeCommon(ConnectionPool connectionPool,
ActionConfiguration actionConfiguration,
Boolean preparedStatement,
List<MustacheBindingToken> mustacheValuesInOrder,
@ -233,6 +229,7 @@ public class MySqlPlugin extends BasePlugin {
String query = actionConfiguration.getBody();
/**
* TBD: check if this comment is resolved with the new MariaDB driver.
* - MySQL r2dbc driver is not able to substitute the `True/False` value properly after the IS keyword.
* Converting `True/False` to integer 1 or 0 also does not work in this case as MySQL syntax does not support
* integers with IS keyword.
@ -260,87 +257,96 @@ public class MySqlPlugin extends BasePlugin {
List<RequestParamDTO> requestParams = List.of(new RequestParamDTO(ACTION_CONFIGURATION_BODY,
transformedQuery, null, null, psParams));
// TODO: need to write a JUnit TC for VALIDATION_CHECK_TIMEOUT
Flux<Result> resultFlux = Mono.from(connection.validate(ValidationDepth.REMOTE))
.timeout(Duration.ofSeconds(VALIDATION_CHECK_TIMEOUT))
.onErrorMap(TimeoutException.class, error -> new StaleConnectionException())
.flatMapMany(isValid -> {
if (isValid) {
return createAndExecuteQueryFromConnection(finalQuery,
connection,
preparedStatement,
mustacheValuesInOrder,
executeActionDTO,
requestData,
psParams);
}
return Flux.error(new StaleConnectionException());
});
return Mono.usingWhen(
connectionPool.create(),
connection -> {
// TODO: add JUnit TC for the `connection.validate` check. Not sure how to do it at the moment.
Flux<Result> resultFlux = Mono.from(connection.validate(ValidationDepth.REMOTE))
.timeout(Duration.ofSeconds(VALIDATION_CHECK_TIMEOUT))
.onErrorMap(TimeoutException.class, error -> new StaleConnectionException())
.flatMapMany(isValid -> {
if (isValid) {
return createAndExecuteQueryFromConnection(finalQuery,
connection,
preparedStatement,
mustacheValuesInOrder,
executeActionDTO,
requestData,
psParams);
}
return Flux.error(new StaleConnectionException());
});
Mono<List<Map<String, Object>>> resultMono;
Mono<List<Map<String, Object>>> resultMono;
if (isSelectOrShowOrDescQuery) {
resultMono = resultFlux
.flatMap(result ->
result.map((row, meta) -> {
rowsList.add(getRow(row, meta));
if (isSelectOrShowOrDescQuery) {
resultMono = resultFlux
.flatMap(result ->
result.map((row, meta) -> {
rowsList.add(getRow(row, meta));
if (columnsList.isEmpty()) {
meta.getColumnMetadatas().stream().forEach(columnMetadata -> columnsList.add(columnMetadata.getName()));
}
if (columnsList.isEmpty()) {
meta.getColumnMetadatas().stream().forEach(columnMetadata -> columnsList.add(columnMetadata.getName()));
}
return result;
}
)
)
.collectList()
.thenReturn(rowsList);
} else {
resultMono = resultFlux
.flatMap(Result::getRowsUpdated)
.collectList()
.map(list -> list.get(list.size() - 1))
.map(rowsUpdated -> {
rowsList.add(
Map.of(
"affectedRows",
ObjectUtils.defaultIfNull(rowsUpdated, 0)
return result;
}
)
)
);
return rowsList;
});
}
return resultMono
.map(res -> {
ActionExecutionResult result = new ActionExecutionResult();
result.setBody(objectMapper.valueToTree(rowsList));
result.setMessages(populateHintMessages(columnsList));
result.setIsExecutionSuccess(true);
log.debug("In the MySqlPlugin, got action execution result");
return result;
})
.onErrorResume(error -> {
if (error instanceof StaleConnectionException) {
return Mono.error(error);
.collectList()
.thenReturn(rowsList);
} else {
resultMono = resultFlux
.flatMap(Result::getRowsUpdated)
.collectList()
.map(list -> list.get(list.size() - 1))
.map(rowsUpdated -> {
rowsList.add(
Map.of(
"affectedRows",
ObjectUtils.defaultIfNull(rowsUpdated, 0)
)
);
return rowsList;
});
}
ActionExecutionResult result = new ActionExecutionResult();
result.setIsExecutionSuccess(false);
result.setErrorInfo(error);
return Mono.just(result);
})
// Now set the request in the result to be returned back to the server
.map(actionExecutionResult -> {
ActionExecutionRequest request = new ActionExecutionRequest();
request.setQuery(finalQuery);
request.setProperties(requestData);
request.setRequestParams(requestParams);
ActionExecutionResult result = actionExecutionResult;
result.setRequest(request);
return result;
})
.subscribeOn(scheduler);
return resultMono
.map(res -> {
ActionExecutionResult result = new ActionExecutionResult();
result.setBody(objectMapper.valueToTree(rowsList));
result.setMessages(populateHintMessages(columnsList));
result.setIsExecutionSuccess(true);
log.debug("In the MySqlPlugin, got action execution result");
return result;
})
.onErrorResume(error -> {
if (error instanceof StaleConnectionException) {
return Mono.error(error);
}
ActionExecutionResult result = new ActionExecutionResult();
result.setIsExecutionSuccess(false);
result.setErrorInfo(error);
return Mono.just(result);
})
// Now set the request in the result to be returned back to the server
.map(actionExecutionResult -> {
ActionExecutionRequest request = new ActionExecutionRequest();
request.setQuery(finalQuery);
request.setProperties(requestData);
request.setRequestParams(requestParams);
ActionExecutionResult result = actionExecutionResult;
result.setRequest(request);
return result;
});
},
Connection::close
)
.timeout(Duration.ofSeconds(VALIDATION_CHECK_TIMEOUT))
.onErrorMap(TimeoutException.class, error -> new StaleConnectionException())
.onErrorMap(PoolShutdownException.class, error -> new StaleConnectionException())
.onErrorMap(R2dbcNonTransientResourceException.class, error -> new StaleConnectionException())
.subscribeOn(scheduler);
}
boolean isIsOperatorUsed(String query) {
@ -388,6 +394,14 @@ public class MySqlPlugin extends BasePlugin {
}
@Override
public Mono<DatasourceTestResult> testDatasource(ConnectionPool pool) {
return Mono.just(pool)
.flatMap(p -> p.create())
.flatMap(conn -> Mono.from(conn.close()))
.then(Mono.just(new DatasourceTestResult()));
}
@Override
public Object substituteValueInInput(int index,
String binding,
@ -500,112 +514,26 @@ public class MySqlPlugin extends BasePlugin {
}
@Override
public Mono<ActionExecutionResult> execute(Connection connection, DatasourceConfiguration datasourceConfiguration, ActionConfiguration actionConfiguration) {
public Mono<ActionExecutionResult> execute(ConnectionPool connection,
DatasourceConfiguration datasourceConfiguration, ActionConfiguration actionConfiguration) {
// Unused function
return Mono.error(new AppsmithPluginException(AppsmithPluginError.PLUGIN_ERROR, "Unsupported Operation"));
}
@Override
public Mono<Connection> datasourceCreate(DatasourceConfiguration datasourceConfiguration) {
DBAuth authentication = (DBAuth) datasourceConfiguration.getAuthentication();
StringBuilder urlBuilder = new StringBuilder();
if (CollectionUtils.isEmpty(datasourceConfiguration.getEndpoints())) {
urlBuilder.append(datasourceConfiguration.getUrl());
} else {
urlBuilder.append("r2dbc:mysql://");
final List<String> hosts = new ArrayList<>();
for (Endpoint endpoint : datasourceConfiguration.getEndpoints()) {
hosts.add(endpoint.getHost() + ":" + ObjectUtils.defaultIfNull(endpoint.getPort(), 3306L));
}
urlBuilder.append(String.join(",", hosts)).append("/");
if (StringUtils.hasLength(authentication.getDatabaseName())) {
urlBuilder.append(authentication.getDatabaseName());
}
public Mono<ConnectionPool> datasourceCreate(DatasourceConfiguration datasourceConfiguration) {
ConnectionPool pool = null;
try {
pool = getNewConnectionPool(datasourceConfiguration);
} catch (AppsmithPluginException e) {
return Mono.error(e);
}
urlBuilder.append("?zeroDateTimeBehavior=convertToNull");
final List<Property> dsProperties = datasourceConfiguration.getProperties();
if (dsProperties != null) {
for (Property property : dsProperties) {
if ("serverTimezone".equals(property.getKey()) && !StringUtils.isEmpty(property.getValue())) {
urlBuilder.append("&serverTimezone=").append(property.getValue());
break;
}
}
}
ConnectionFactoryOptions baseOptions = ConnectionFactoryOptions.parse(urlBuilder.toString());
ConnectionFactoryOptions.Builder ob = ConnectionFactoryOptions.builder().from(baseOptions)
.option(ConnectionFactoryOptions.DRIVER, MariadbConnectionFactoryProvider.MARIADB_DRIVER)
.option(MariadbConnectionFactoryProvider.ALLOW_MULTI_QUERIES, true)
.option(ConnectionFactoryOptions.USER, authentication.getUsername())
.option(ConnectionFactoryOptions.PASSWORD, authentication.getPassword());
/*
* - Ideally, it is never expected to be null because the SSL dropdown is set to a initial value.
*/
if (datasourceConfiguration.getConnection() == null
|| datasourceConfiguration.getConnection().getSsl() == null
|| datasourceConfiguration.getConnection().getSsl().getAuthType() == null) {
return Mono.error(
new AppsmithPluginException(
AppsmithPluginError.PLUGIN_ERROR,
"Appsmith server has failed to fetch SSL configuration from datasource configuration form. " +
"Please reach out to Appsmith customer support to resolve this."
)
);
}
/*
* - By default, the driver configures SSL in the preferred mode.
*/
SSLDetails.AuthType sslAuthType = datasourceConfiguration.getConnection().getSsl().getAuthType();
switch (sslAuthType) {
case PREFERRED:
case REQUIRED:
ob = ob
.option(SSL, true)
.option(Option.valueOf("sslMode"), sslAuthType.toString().toLowerCase());
break;
case DISABLED:
ob = ob.option(SSL, false);
break;
case DEFAULT:
/* do nothing - accept default driver setting*/
break;
default:
return Mono.error(
new AppsmithPluginException(
AppsmithPluginError.PLUGIN_ERROR,
"Appsmith server has found an unexpected SSL option: " + sslAuthType + ". Please reach out to" +
" Appsmith customer support to resolve this."
)
);
}
return (Mono<Connection>) Mono.from(ConnectionFactories.get(ob.build()).create())
.onErrorResume(exception -> Mono.error(new AppsmithPluginException(
AppsmithPluginError.PLUGIN_DATASOURCE_ARGUMENT_ERROR,
exception
)))
.subscribeOn(scheduler);
return Mono.just(pool);
}
@Override
public void datasourceDestroy(Connection connection) {
if (connection != null) {
Mono.from(connection.close())
public void datasourceDestroy(ConnectionPool connectionPool) {
if (connectionPool != null) {
Mono.just(connectionPool.disposeLater())
.onErrorResume(exception -> {
log.debug("In datasourceDestroy function error mode.", exception);
return Mono.empty();
@ -617,258 +545,72 @@ public class MySqlPlugin extends BasePlugin {
@Override
public Set<String> validateDatasource(DatasourceConfiguration datasourceConfiguration) {
Set<String> invalids = new HashSet<>();
if (datasourceConfiguration.getConnection() != null
&& datasourceConfiguration.getConnection().getMode() == null) {
invalids.add("Missing Connection Mode.");
}
if (StringUtils.isEmpty(datasourceConfiguration.getUrl()) &&
CollectionUtils.isEmpty(datasourceConfiguration.getEndpoints())) {
invalids.add("Missing endpoint and url");
} else if (!CollectionUtils.isEmpty(datasourceConfiguration.getEndpoints())) {
for (final Endpoint endpoint : datasourceConfiguration.getEndpoints()) {
if (endpoint.getHost() == null || endpoint.getHost().isBlank()) {
invalids.add("Host value cannot be empty");
} else if (endpoint.getHost().contains("/") || endpoint.getHost().contains(":")) {
invalids.add("Host value cannot contain `/` or `:` characters. Found `" + endpoint.getHost() + "`.");
}
}
}
if (datasourceConfiguration.getAuthentication() == null) {
invalids.add("Missing authentication details.");
} else {
DBAuth authentication = (DBAuth) datasourceConfiguration.getAuthentication();
if (StringUtils.isEmpty(authentication.getUsername())) {
invalids.add("Missing username for authentication.");
}
if (StringUtils.isEmpty(authentication.getPassword()) && StringUtils.isEmpty(authentication.getUsername())) {
invalids.add("Missing password for authentication.");
} else if (StringUtils.isEmpty(authentication.getPassword())) {
// it is valid if it has the username but not the password
authentication.setPassword("");
}
if (StringUtils.isEmpty(authentication.getDatabaseName())) {
invalids.add("Missing database name.");
}
}
/*
* - Ideally, it is never expected to be null because the SSL dropdown is set to a initial value.
*/
if (datasourceConfiguration.getConnection() == null
|| datasourceConfiguration.getConnection().getSsl() == null
|| datasourceConfiguration.getConnection().getSsl().getAuthType() == null) {
invalids.add("Appsmith server has failed to fetch SSL configuration from datasource configuration form. " +
"Please reach out to Appsmith customer support to resolve this.");
}
return invalids;
}
/**
* 1. Parse results obtained by running COLUMNS_QUERY defined on top of the page.
* 2. A sample mysql output for the query is also given near COLUMNS_QUERY definition on top of the page.
*/
private void getTableInfo(Row row, RowMetadata meta, Map<String, DatasourceStructure.Table> tablesByName) {
final String tableName = row.get("table_name", String.class);
if (!tablesByName.containsKey(tableName)) {
tablesByName.put(tableName, new DatasourceStructure.Table(
DatasourceStructure.TableType.TABLE,
null,
tableName,
new ArrayList<>(),
new ArrayList<>(),
new ArrayList<>()
));
}
final DatasourceStructure.Table table = tablesByName.get(tableName);
table.getColumns().add(new DatasourceStructure.Column(
row.get("column_name", String.class),
row.get("column_type", String.class),
null,
row.get("extra", String.class).contains("auto_increment")
));
return;
}
/**
* 1. Parse results obtained by running KEYS_QUERY defined on top of the page.
* 2. A sample mysql output for the query is also given near KEYS_QUERY definition on top of the page.
*/
private void getKeyInfo(Row row, RowMetadata meta, Map<String, DatasourceStructure.Table> tablesByName,
Map<String, DatasourceStructure.Key> keyRegistry) {
final String constraintName = row.get("constraint_name", String.class);
final char constraintType = row.get("constraint_type", String.class).charAt(0);
final String selfSchema = row.get("self_schema", String.class);
final String tableName = row.get("self_table", String.class);
if (!tablesByName.containsKey(tableName)) {
/* do nothing */
return;
}
final DatasourceStructure.Table table = tablesByName.get(tableName);
final String keyFullName = tableName + "." + row.get("constraint_name", String.class);
if (constraintType == 'p') {
if (!keyRegistry.containsKey(keyFullName)) {
final DatasourceStructure.PrimaryKey key = new DatasourceStructure.PrimaryKey(
constraintName,
new ArrayList<>()
);
keyRegistry.put(keyFullName, key);
table.getKeys().add(key);
}
((DatasourceStructure.PrimaryKey) keyRegistry.get(keyFullName)).getColumnNames()
.add(row.get("self_column", String.class));
} else if (constraintType == 'f') {
final String foreignSchema = row.get("foreign_schema", String.class);
final String prefix = (foreignSchema.equalsIgnoreCase(selfSchema) ? "" : foreignSchema + ".")
+ row.get("foreign_table", String.class) + ".";
if (!keyRegistry.containsKey(keyFullName)) {
final DatasourceStructure.ForeignKey key = new DatasourceStructure.ForeignKey(
constraintName,
new ArrayList<>(),
new ArrayList<>()
);
keyRegistry.put(keyFullName, key);
table.getKeys().add(key);
}
((DatasourceStructure.ForeignKey) keyRegistry.get(keyFullName)).getFromColumns()
.add(row.get("self_column", String.class));
((DatasourceStructure.ForeignKey) keyRegistry.get(keyFullName)).getToColumns()
.add(prefix + row.get("foreign_column", String.class));
}
return;
}
/**
* 1. Generate template for all tables in the database.
*/
private void getTemplates(Map<String, DatasourceStructure.Table> tablesByName) {
for (DatasourceStructure.Table table : tablesByName.values()) {
final List<DatasourceStructure.Column> columnsWithoutDefault = table.getColumns()
.stream()
.filter(column -> column.getDefaultValue() == null)
.collect(Collectors.toList());
final List<String> columnNames = new ArrayList<>();
final List<String> columnValues = new ArrayList<>();
final StringBuilder setFragments = new StringBuilder();
for (DatasourceStructure.Column column : columnsWithoutDefault) {
final String name = column.getName();
final String type = column.getType();
String value;
if (type == null) {
value = "null";
} else if ("text".equals(type) || "varchar".equals(type)) {
value = "''";
} else if (type.startsWith("int")) {
value = "1";
} else if (type.startsWith("double")) {
value = "1.0";
} else if (DATE_COLUMN_TYPE_NAME.equals(type)) {
value = "'2019-07-01'";
} else if (DATETIME_COLUMN_TYPE_NAME.equals(type)
|| TIMESTAMP_COLUMN_TYPE_NAME.equals(type)) {
value = "'2019-07-01 10:00:00'";
} else {
value = "''";
}
columnNames.add(name);
columnValues.add(value);
setFragments.append("\n ").append(name).append(" = ").append(value).append(",");
}
// Delete the last comma
if (setFragments.length() > 0) {
setFragments.deleteCharAt(setFragments.length() - 1);
}
final String tableName = table.getName();
table.getTemplates().addAll(List.of(
new DatasourceStructure.Template("SELECT", "SELECT * FROM " + tableName + " LIMIT 10;"),
new DatasourceStructure.Template("INSERT", "INSERT INTO " + tableName
+ " (" + String.join(", ", columnNames) + ")\n"
+ " VALUES (" + String.join(", ", columnValues) + ");"),
new DatasourceStructure.Template("UPDATE", "UPDATE " + tableName + " SET"
+ setFragments + "\n"
+ " WHERE 1 = 0; -- Specify a valid condition here. Removing the condition may update every row in the table!"),
new DatasourceStructure.Template("DELETE", "DELETE FROM " + tableName
+ "\n WHERE 1 = 0; -- Specify a valid condition here. Removing the condition may delete everything in the table!")
));
}
return MySqlDatasourceUtils.validateDatasource(datasourceConfiguration);
}
@Override
public Mono<DatasourceStructure> getStructure(Connection connection, DatasourceConfiguration datasourceConfiguration) {
public Mono<DatasourceStructure> getStructure(ConnectionPool connectionPool,
DatasourceConfiguration datasourceConfiguration) {
final DatasourceStructure structure = new DatasourceStructure();
final Map<String, DatasourceStructure.Table> tablesByName = new LinkedHashMap<>();
final Map<String, DatasourceStructure.Key> keyRegistry = new HashMap<>();
return Mono.from(connection.validate(ValidationDepth.REMOTE))
return Mono.usingWhen(
connectionPool.create(),
connection -> {
return Mono.from(connection.validate(ValidationDepth.REMOTE))
.timeout(Duration.ofSeconds(VALIDATION_CHECK_TIMEOUT))
.onErrorMap(TimeoutException.class, error -> new StaleConnectionException())
.flatMapMany(isValid -> {
if (isValid) {
return connection.createStatement(COLUMNS_QUERY).execute();
} else {
return Flux.error(new StaleConnectionException());
}
})
.flatMap(result -> {
return result.map((row, meta) -> {
getTableInfo(row, meta, tablesByName);
return result;
});
})
.collectList()
.thenMany(Flux.from(connection.createStatement(KEYS_QUERY).execute()))
.flatMap(result -> {
return result.map((row, meta) -> {
getKeyInfo(row, meta, tablesByName, keyRegistry);
return result;
});
})
.collectList()
.map(list -> {
/* Get templates for each table and put those in. */
getTemplates(tablesByName);
structure.setTables(new ArrayList<>(tablesByName.values()));
for (DatasourceStructure.Table table : structure.getTables()) {
table.getKeys().sort(Comparator.naturalOrder());
}
return structure;
})
.onErrorMap(e -> {
if (!(e instanceof AppsmithPluginException) && !(e instanceof StaleConnectionException)) {
return new AppsmithPluginException(
AppsmithPluginError.PLUGIN_ERROR,
e.getMessage()
);
}
return e;
});
},
Connection::close
)
.timeout(Duration.ofSeconds(VALIDATION_CHECK_TIMEOUT))
.onErrorMap(TimeoutException.class, error -> new StaleConnectionException())
.flatMapMany(isValid -> {
if (isValid) {
return connection.createStatement(COLUMNS_QUERY).execute();
} else {
return Flux.error(new StaleConnectionException());
}
})
.flatMap(result -> {
return result.map((row, meta) -> {
getTableInfo(row, meta, tablesByName);
return result;
});
})
.collectList()
.thenMany(Flux.from(connection.createStatement(KEYS_QUERY).execute()))
.flatMap(result -> {
return result.map((row, meta) -> {
getKeyInfo(row, meta, tablesByName, keyRegistry);
return result;
});
})
.collectList()
.map(list -> {
/* Get templates for each table and put those in. */
getTemplates(tablesByName);
structure.setTables(new ArrayList<>(tablesByName.values()));
for (DatasourceStructure.Table table : structure.getTables()) {
table.getKeys().sort(Comparator.naturalOrder());
}
return structure;
})
.onErrorMap(e -> {
if (!(e instanceof AppsmithPluginException) && !(e instanceof StaleConnectionException)) {
return new AppsmithPluginException(
AppsmithPluginError.PLUGIN_ERROR,
e.getMessage()
);
}
return e;
})
.onErrorMap(PoolShutdownException.class, error -> new StaleConnectionException())
.subscribeOn(scheduler);
}
}

193
app/server/appsmith-plugins/mysqlPlugin/src/main/java/com/external/utils/MySqlDatasourceUtils.java vendored Normal file
View File

@ -0,0 +1,193 @@
package com.external.utils;
import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginError;
import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginException;
import com.appsmith.external.models.DBAuth;
import com.appsmith.external.models.DatasourceConfiguration;
import com.appsmith.external.models.Endpoint;
import com.appsmith.external.models.Property;
import com.appsmith.external.models.SSLDetails;
import io.r2dbc.pool.ConnectionPool;
import io.r2dbc.pool.ConnectionPoolConfiguration;
import io.r2dbc.spi.ConnectionFactoryOptions;
import io.r2dbc.spi.Option;
import org.apache.commons.lang.ObjectUtils;
import org.mariadb.r2dbc.MariadbConnectionConfiguration;
import org.mariadb.r2dbc.MariadbConnectionFactory;
import org.springframework.util.CollectionUtils;
import org.springframework.util.StringUtils;
import java.time.Duration;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import static io.r2dbc.pool.PoolingConnectionFactoryProvider.MAX_SIZE;
import static io.r2dbc.spi.ConnectionFactoryOptions.SSL;
public class MySqlDatasourceUtils {
public static int MAX_CONNECTION_POOL_SIZE = 5;
private static final Duration MAX_IDLE_TIME = Duration.ofMinutes(10);
public static ConnectionFactoryOptions.Builder getBuilder(DatasourceConfiguration datasourceConfiguration) {
DBAuth authentication = (DBAuth) datasourceConfiguration.getAuthentication();
StringBuilder urlBuilder = new StringBuilder();
if (CollectionUtils.isEmpty(datasourceConfiguration.getEndpoints())) {
urlBuilder.append(datasourceConfiguration.getUrl());
} else {
urlBuilder.append("r2dbc:pool:mariadb://");
final List<String> hosts = new ArrayList<>();
for (Endpoint endpoint : datasourceConfiguration.getEndpoints()) {
hosts.add(endpoint.getHost() + ":" + ObjectUtils.defaultIfNull(endpoint.getPort(), 3306L));
}
urlBuilder.append(String.join(",", hosts)).append("/");
if (!StringUtils.isEmpty(authentication.getDatabaseName())) {
urlBuilder.append(authentication.getDatabaseName());
}
}
urlBuilder.append("?zeroDateTimeBehavior=convertToNull&allowMultiQueries=true");
final List<Property> dsProperties = datasourceConfiguration.getProperties();
if (dsProperties != null) {
for (Property property : dsProperties) {
if ("serverTimezone".equals(property.getKey()) && !StringUtils.isEmpty(property.getValue())) {
urlBuilder.append("&serverTimezone=").append(property.getValue());
break;
}
}
}
ConnectionFactoryOptions baseOptions = ConnectionFactoryOptions.parse(urlBuilder.toString());
ConnectionFactoryOptions.Builder ob = ConnectionFactoryOptions.builder().from(baseOptions)
.option(ConnectionFactoryOptions.USER, authentication.getUsername())
.option(ConnectionFactoryOptions.PASSWORD, authentication.getPassword());
return ob;
}
public static ConnectionFactoryOptions.Builder addSslOptionsToBuilder(DatasourceConfiguration datasourceConfiguration,
ConnectionFactoryOptions.Builder ob) throws AppsmithPluginException {
/*
* - Ideally, it is never expected to be null because the SSL dropdown is set to a initial value.
*/
if (datasourceConfiguration.getConnection() == null
|| datasourceConfiguration.getConnection().getSsl() == null
|| datasourceConfiguration.getConnection().getSsl().getAuthType() == null) {
throw new AppsmithPluginException(
AppsmithPluginError.PLUGIN_ERROR,
"Appsmith server has failed to fetch SSL configuration from datasource configuration form. " +
"Please reach out to Appsmith customer support to resolve this.");
}
/*
* - By default, the driver configures SSL in the preferred mode.
*/
SSLDetails.AuthType sslAuthType = datasourceConfiguration.getConnection().getSsl().getAuthType();
switch (sslAuthType) {
case REQUIRED:
ob = ob
.option(SSL, true)
.option(Option.valueOf("sslMode"), sslAuthType.toString().toLowerCase());
break;
case DISABLED:
ob = ob.option(SSL, false);
break;
case DEFAULT:
/* do nothing - accept default driver setting*/
break;
default:
throw new AppsmithPluginException(
AppsmithPluginError.PLUGIN_ERROR,
"Appsmith server has found an unexpected SSL option: " + sslAuthType + ". Please reach out to" +
" Appsmith customer support to resolve this.");
}
return ob;
}
public static Set<String> validateDatasource(DatasourceConfiguration datasourceConfiguration) {
Set<String> invalids = new HashSet<>();
if (datasourceConfiguration.getConnection() != null
&& datasourceConfiguration.getConnection().getMode() == null) {
invalids.add("Missing Connection Mode.");
}
if (StringUtils.isEmpty(datasourceConfiguration.getUrl()) &&
CollectionUtils.isEmpty(datasourceConfiguration.getEndpoints())) {
invalids.add("Missing endpoint and url");
} else if (!CollectionUtils.isEmpty(datasourceConfiguration.getEndpoints())) {
for (final Endpoint endpoint : datasourceConfiguration.getEndpoints()) {
if (endpoint.getHost() == null || endpoint.getHost().isBlank()) {
invalids.add("Host value cannot be empty");
} else if (endpoint.getHost().contains("/") || endpoint.getHost().contains(":")) {
invalids.add("Host value cannot contain `/` or `:` characters. Found `" + endpoint.getHost() + "`.");
}
}
}
if (datasourceConfiguration.getAuthentication() == null) {
invalids.add("Missing authentication details.");
} else {
DBAuth authentication = (DBAuth) datasourceConfiguration.getAuthentication();
if (StringUtils.isEmpty(authentication.getUsername())) {
invalids.add("Missing username for authentication.");
}
if (StringUtils.isEmpty(authentication.getPassword()) && StringUtils.isEmpty(authentication.getUsername())) {
invalids.add("Missing password for authentication.");
} else if (StringUtils.isEmpty(authentication.getPassword())) {
// it is valid if it has the username but not the password
authentication.setPassword("");
}
if (StringUtils.isEmpty(authentication.getDatabaseName())) {
invalids.add("Missing database name.");
}
}
/*
* - Ideally, it is never expected to be null because the SSL dropdown is set to a initial value.
*/
if (datasourceConfiguration.getConnection() == null
|| datasourceConfiguration.getConnection().getSsl() == null
|| datasourceConfiguration.getConnection().getSsl().getAuthType() == null) {
invalids.add("Appsmith server has failed to fetch SSL configuration from datasource configuration form. " +
"Please reach out to Appsmith customer support to resolve this.");
}
return invalids;
}
public static ConnectionPool getNewConnectionPool(DatasourceConfiguration datasourceConfiguration) throws AppsmithPluginException {
ConnectionFactoryOptions.Builder ob = getBuilder(datasourceConfiguration);
ob = addSslOptionsToBuilder(datasourceConfiguration, ob);
MariadbConnectionFactory connectionFactory =
MariadbConnectionFactory.from(
MariadbConnectionConfiguration.fromOptions(ob.build())
.allowPublicKeyRetrieval(true).build()
);
/**
* The pool configuration object does not seem to have any option to set the minimum pool size, hence could
* not configure the minimum pool size.
*/
ConnectionPoolConfiguration configuration = ConnectionPoolConfiguration.builder(connectionFactory)
.maxIdleTime(MAX_IDLE_TIME)
.maxSize(MAX_CONNECTION_POOL_SIZE)
.build();
return new ConnectionPool(configuration);
}
}

162
app/server/appsmith-plugins/mysqlPlugin/src/main/java/com/external/utils/MySqlGetStructureUtils.java vendored Normal file
View File

@ -0,0 +1,162 @@
package com.external.utils;
import com.appsmith.external.models.DatasourceStructure;
import io.r2dbc.spi.Row;
import io.r2dbc.spi.RowMetadata;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
public class MySqlGetStructureUtils {
private static final String DATE_COLUMN_TYPE_NAME = "date";
private static final String DATETIME_COLUMN_TYPE_NAME = "datetime";
private static final String TIMESTAMP_COLUMN_TYPE_NAME = "timestamp";
/**
* 1. Parse results obtained by running COLUMNS_QUERY defined on top of the page.
* 2. A sample mysql output for the query is also given near COLUMNS_QUERY definition on top of the page.
*/
public static void getTableInfo(Row row, RowMetadata meta, Map<String, DatasourceStructure.Table> tablesByName) {
final String tableName = row.get("table_name", String.class);
if (!tablesByName.containsKey(tableName)) {
tablesByName.put(tableName, new DatasourceStructure.Table(
DatasourceStructure.TableType.TABLE,
null,
tableName,
new ArrayList<>(),
new ArrayList<>(),
new ArrayList<>()
));
}
final DatasourceStructure.Table table = tablesByName.get(tableName);
table.getColumns().add(new DatasourceStructure.Column(
row.get("column_name", String.class),
row.get("column_type", String.class),
null,
row.get("extra", String.class).contains("auto_increment")
));
return;
}
/**
* 1. Parse results obtained by running KEYS_QUERY defined on top of the page.
* 2. A sample mysql output for the query is also given near KEYS_QUERY definition on top of the page.
*/
public static void getKeyInfo(Row row, RowMetadata meta, Map<String, DatasourceStructure.Table> tablesByName,
Map<String, DatasourceStructure.Key> keyRegistry) {
final String constraintName = row.get("constraint_name", String.class);
final char constraintType = row.get("constraint_type", String.class).charAt(0);
final String selfSchema = row.get("self_schema", String.class);
final String tableName = row.get("self_table", String.class);
if (!tablesByName.containsKey(tableName)) {
/* do nothing */
return;
}
final DatasourceStructure.Table table = tablesByName.get(tableName);
final String keyFullName = tableName + "." + row.get("constraint_name", String.class);
if (constraintType == 'p') {
if (!keyRegistry.containsKey(keyFullName)) {
final DatasourceStructure.PrimaryKey key = new DatasourceStructure.PrimaryKey(
constraintName,
new ArrayList<>()
);
keyRegistry.put(keyFullName, key);
table.getKeys().add(key);
}
((DatasourceStructure.PrimaryKey) keyRegistry.get(keyFullName)).getColumnNames()
.add(row.get("self_column", String.class));
} else if (constraintType == 'f') {
final String foreignSchema = row.get("foreign_schema", String.class);
final String prefix = (foreignSchema.equalsIgnoreCase(selfSchema) ? "" : foreignSchema + ".")
+ row.get("foreign_table", String.class) + ".";
if (!keyRegistry.containsKey(keyFullName)) {
final DatasourceStructure.ForeignKey key = new DatasourceStructure.ForeignKey(
constraintName,
new ArrayList<>(),
new ArrayList<>()
);
keyRegistry.put(keyFullName, key);
table.getKeys().add(key);
}
((DatasourceStructure.ForeignKey) keyRegistry.get(keyFullName)).getFromColumns()
.add(row.get("self_column", String.class));
((DatasourceStructure.ForeignKey) keyRegistry.get(keyFullName)).getToColumns()
.add(prefix + row.get("foreign_column", String.class));
}
return;
}
/**
* 1. Generate template for all tables in the database.
*/
public static void getTemplates(Map<String, DatasourceStructure.Table> tablesByName) {
for (DatasourceStructure.Table table : tablesByName.values()) {
final List<DatasourceStructure.Column> columnsWithoutDefault = table.getColumns()
.stream()
.filter(column -> column.getDefaultValue() == null)
.collect(Collectors.toList());
final List<String> columnNames = new ArrayList<>();
final List<String> columnValues = new ArrayList<>();
final StringBuilder setFragments = new StringBuilder();
for (DatasourceStructure.Column column : columnsWithoutDefault) {
final String name = column.getName();
final String type = column.getType();
String value;
if (type == null) {
value = "null";
} else if ("text".equals(type) || "varchar".equals(type)) {
value = "''";
} else if (type.startsWith("int")) {
value = "1";
} else if (type.startsWith("double")) {
value = "1.0";
} else if (DATE_COLUMN_TYPE_NAME.equals(type)) {
value = "'2019-07-01'";
} else if (DATETIME_COLUMN_TYPE_NAME.equals(type)
|| TIMESTAMP_COLUMN_TYPE_NAME.equals(type)) {
value = "'2019-07-01 10:00:00'";
} else {
value = "''";
}
columnNames.add(name);
columnValues.add(value);
setFragments.append("\n ").append(name).append(" = ").append(value).append(",");
}
// Delete the last comma
if (setFragments.length() > 0) {
setFragments.deleteCharAt(setFragments.length() - 1);
}
final String tableName = table.getName();
table.getTemplates().addAll(List.of(
new DatasourceStructure.Template("SELECT", "SELECT * FROM " + tableName + " LIMIT 10;"),
new DatasourceStructure.Template("INSERT", "INSERT INTO " + tableName
+ " (" + String.join(", ", columnNames) + ")\n"
+ " VALUES (" + String.join(", ", columnValues) + ");"),
new DatasourceStructure.Template("UPDATE", "UPDATE " + tableName + " SET"
+ setFragments + "\n"
+ " WHERE 1 = 0; -- Specify a valid condition here. Removing the condition may update every row in the table!"),
new DatasourceStructure.Template("DELETE", "DELETE FROM " + tableName
+ "\n WHERE 1 = 0; -- Specify a valid condition here. Removing the condition may delete everything in the table!")
));
}
}
}

4
app/server/appsmith-plugins/mysqlPlugin/src/main/resources/form.json
View File

@ -87,10 +87,6 @@
"label": "Default",
"value": "DEFAULT"
},
{
"label": "Preferred",
"value": "PREFERRED"
},
{
"label": "Required",
"value": "REQUIRED"

2818
app/server/appsmith-plugins/mysqlPlugin/src/test/java/com/external/plugins/MySqlPluginTest.java vendored
View File

File diff suppressed because it is too large Load Diff

11
app/server/appsmith-server/pom.xml
View File

@ -41,12 +41,13 @@
</repositories>
<dependencies>
<!--
Ideally this dependency should have been added in the pom.xml file of GraphQLPlugin module, but it is
causing 'java.lang.NoClassDefFoundError' error. Hence, adding it here after many attempts of fixing it the right
way. Somehow adding it here makes it work. GraphQLPlugin module's pom.xml file also has this dependency
defined with 'provided' scope
-->
Ideally this dependency should have been added in the pom.xml file of GraphQLPlugin module, but it is
causing 'java.lang.NoClassDefFoundError' error. Hence, adding it here after many attempts of fixing it the right
way. Somehow adding it here makes it work. GraphQLPlugin module's pom.xml file also has this dependency
defined with 'provided' scope
-->
<dependency>
<groupId>com.graphql-java</groupId>
<artifactId>graphql-java</artifactId>

165
app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/DatabaseChangelog2.java
View File

@ -66,6 +66,7 @@ import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.querydsl.core.types.Path;
import io.changock.migration.api.annotations.NonLockGuarded;
import io.mongock.api.annotations.ChangeUnit;
import lombok.extern.slf4j.Slf4j;
import net.minidev.json.JSONObject;
import org.bson.types.ObjectId;
@ -2774,154 +2775,20 @@ public class DatabaseChangelog2 {
ensureIndexes(mongoTemplate, CustomJSLib.class, uidStringUniqueness);
}
// @ChangeSet(order = "039", id = "deprecate-queryabletext-encryption", author = "")
// public void deprecateQueryableTextEncryption(MongockTemplate mongockTemplate,
// @NonLockGuarded EncryptionConfig encryptionConfig,
// EncryptionService encryptionService) {
// Stopwatch stopwatch = new Stopwatch("Instance Schema migration to v2");
/**
* Since MySQL plugin's underlying driver has been changed to MariaDB driver, the `ssl-mode=preferred` is no
* longer supported. Hence, any such usage is being updated to `ssl-mode=default` by this method.
*/
@ChangeSet(order = "039", id = "remove-preferred-ssl-mode-from-mysql", author = "")
public void changeSSLModeFromPreferredToDefaultForMySQLPlugin(MongoTemplate mongoTemplate) {
Plugin mySQLPlugin = mongoTemplate.findOne(query(where("packageName").is("mysql-plugin")),
Plugin.class);
Query queryToGetDatasources = getQueryToFetchAllDomainObjectsWhichAreNotDeletedUsingPluginId(mySQLPlugin);
queryToGetDatasources.addCriteria(Criteria.where("datasourceConfiguration.connection.ssl.authType").is(
"PREFERRED"));
// Config encryptionVersion = mongockTemplate.findOne(
// query(where(fieldName(QConfig.config1.name)).is(Appsmith.INSTANCE_SCHEMA_VERSION)),
// Config.class);
// if (encryptionVersion != null && (Integer) encryptionVersion.getConfig().get("value") < 2) {
// String saltInHex = Hex.encodeHexString(encryptionConfig.getSalt().getBytes());
// TextEncryptor textEncryptor = Encryptors.queryableText(encryptionConfig.getPassword(), saltInHex);
// /**
// * - List of attributes in datasources that need to be encoded.
// * - Each path represents where the attribute exists in mongo db document.
// */
// List<String> datasourcePathList = new ArrayList<>();
// datasourcePathList.add("datasourceConfiguration.connection.ssl.keyFile.base64Content");
// datasourcePathList.add("datasourceConfiguration.connection.ssl.certificateFile.base64Content");
// datasourcePathList.add("datasourceConfiguration.connection.ssl.caCertificateFile.base64Content");
// datasourcePathList.add("datasourceConfiguration.connection.ssl.pemCertificate.file.base64Content");
// datasourcePathList.add("datasourceConfiguration.connection.ssl.pemCertificate.password");
// datasourcePathList.add("datasourceConfiguration.sshProxy.privateKey.keyFile.base64Content");
// datasourcePathList.add("datasourceConfiguration.sshProxy.privateKey.password");
// datasourcePathList.add("datasourceConfiguration.authentication.value");
// datasourcePathList.add("datasourceConfiguration.authentication.password");
// datasourcePathList.add("datasourceConfiguration.authentication.bearerToken");
// datasourcePathList.add("datasourceConfiguration.authentication.clientSecret");
// datasourcePathList.add("datasourceConfiguration.authentication.authenticationResponse.token");
// datasourcePathList.add("datasourceConfiguration.authentication.authenticationResponse.refreshToken");
// datasourcePathList.add("datasourceConfiguration.authentication.authenticationResponse.tokenResponse");
// List<Bson> datasourcePathListExists = datasourcePathList
// .stream()
// .map(Filters::exists)
// .collect(Collectors.toList());
// List<Bson> gitDeployKeysPathListExists = new ArrayList<>();
// ArrayList<String> gitDeployKeysPathList = new ArrayList<>();
// gitDeployKeysPathList.add("gitAuth.privateKey");
// gitDeployKeysPathListExists.add(Filters.exists("gitAuth.privateKey"));
// List<Bson> applicationPathListExists = new ArrayList<>();
// ArrayList<String> applicationPathList = new ArrayList<>();
// applicationPathList.add("gitApplicationMetadata.gitAuth.privateKey");
// applicationPathListExists.add(Filters.exists("gitApplicationMetadata.gitAuth.privateKey"));
// mongockTemplate.execute("datasource", getNewEncryptionCallback(textEncryptor, encryptionService, datasourcePathListExists, datasourcePathList, stopwatch));
// mongockTemplate.execute("gitDeployKeys", getNewEncryptionCallback(textEncryptor, encryptionService, gitDeployKeysPathListExists, gitDeployKeysPathList, stopwatch));
// mongockTemplate.execute("application", getNewEncryptionCallback(textEncryptor, encryptionService, applicationPathListExists, applicationPathList, stopwatch));
// mongockTemplate.upsert(
// query(where(fieldName(QConfig.config1.name)).is(Appsmith.INSTANCE_SCHEMA_VERSION)),
// update("config.value", 2),
// Config.class);
// }
// stopwatch.stopAndLogTimeInMillis();
// }
// private CollectionCallback<String> getNewEncryptionCallback(
// TextEncryptor textEncryptor,
// EncryptionService encryptionService,
// Iterable<Bson> collectionFilterIterable,
// List<String> pathList,
// Stopwatch stopwatch) {
// return new CollectionCallback<String>() {
// @Override
// public String doInCollection(MongoCollection<Document> collection) {
// MongoCursor<Document> cursor = collection
// .find(
// Filters.and(
// Filters.or(collectionFilterIterable),
// Filters.not(Filters.exists("encryptionVersion"))))
// .cursor();
// log.debug("collection callback start: {}ms", stopwatch.getExecutionTime());
// List<List<Bson>> documentPairList = new ArrayList<>();
// while (cursor.hasNext()) {
// Document old = cursor.next();
// BasicDBObject query = new BasicDBObject();
// query.put("_id", old.getObjectId("_id"));
// // This document will have the encrypted values.
// BasicDBObject updated = new BasicDBObject();
// updated.put("$set", new BasicDBObject("encryptionVersion", 2));
// updated.put("$unset", new BasicDBObject());
// // Encrypt attributes
// pathList.stream()
// .forEach(path -> reapplyNewEncryptionToPathValueIfExists(old, updated, path, encryptionService, textEncryptor));
// // Since empty unset values are only allowed since Mongo v5+,
// // Remove the operation if there is nothing to unset
// if (((BasicDBObject) updated.get("$unset")).isEmpty()) {
// updated.remove("$unset");
// }
// documentPairList.add(List.of(query, updated));
// }
// log.debug("collection callback processing end: {}ms", stopwatch.getExecutionTime());
// log.debug("update will be run for {} documents", documentPairList.size());
// /**
// * - Replace old document with the updated document that has encrypted values.
// * - Replacing here instead of the while loop above makes sure that we attempt replacement only if
// * the encryption step succeeded without error for each selected document.
// */
// documentPairList.stream().parallel()
// .forEach(docPair -> collection.updateOne(docPair.get(0), docPair.get(1)));
// log.debug("collection callback update end: {}ms", stopwatch.getExecutionTime());
// return null;
// }
// };
// }
// private void reapplyNewEncryptionToPathValueIfExists(Document document, BasicDBObject update, String path,
// EncryptionService encryptionService,
// TextEncryptor textEncryptor) {
// String[] pathKeys = path.split("\\.");
// /**
// * - For attribute path "datasourceConfiguration.connection.ssl.keyFile.base64Content", first get the parent
// * document that contains the attribute 'base64Content' i.e. fetch the document corresponding to path
// * "datasourceConfiguration.connection.ssl.keyFile"
// */
// String parentDocumentPath = org.apache.commons.lang.StringUtils.join(ArrayUtils.subarray(pathKeys, 0, pathKeys.length - 1), ".");
// Document parentDocument = DatabaseChangelog1.getDocumentFromPath(document, parentDocumentPath);
// if (parentDocument != null) {
// if (parentDocument.containsKey(pathKeys[pathKeys.length - 1])) {
// String oldEncryptedValue = parentDocument.getString(pathKeys[pathKeys.length - 1]);
// if (StringUtils.hasLength(String.valueOf(oldEncryptedValue))) {
// String decryptedValue = null;
// try {
// decryptedValue = textEncryptor.decrypt(String.valueOf(oldEncryptedValue));
// } catch (IllegalArgumentException e) {
// // This happens on release DB for some creds that are malformed
// if ("Hex-encoded string must have an even number of characters".equals(e.getMessage())) {
// decryptedValue = String.valueOf(oldEncryptedValue);
// }
// }
// String newEncryptedValue = encryptionService.encryptString(decryptedValue);
// ((BasicDBObject) update.get("$set")).put(path, newEncryptedValue);
// if (path.startsWith("datasourceConfiguration.authentication")) {
// ((BasicDBObject) update.get("$unset")).put("datasourceConfiguration.authentication.isEncrypted", 1);
// }
// }
// }
// }
// }
Update update = new Update();
update.set("datasourceConfiguration.connection.ssl.authType", "DEFAULT");
mongoTemplate.updateMulti(queryToGetDatasources, update, Datasource.class);
}
}

2
app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/BaseAppsmithRepositoryImpl.java
View File

@ -15,4 +15,4 @@ public abstract class BaseAppsmithRepositoryImpl<T extends BaseDomain> extends B
super(mongoOperations, mongoConverter, cacheableRepositoryHelper);
}
}
}