Protei/PromucFlow_constructor
2
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Mysql plugin integration (#53)

Browse Source 
This commit adds the Mysql plugin to the Appsmith server. We also add a migration to ensure that this plugin is installed by default for all existing organizations. The migration also adds the plugin details into the DB.

Also adding the test cases for mysql plugin.

Co-authored-by: Arpit Mohan <arpit@appsmith.com>
Co-authored-by: Hetu Nandu <hetu@appsmith.com>
Co-authored-by: Arpit Mohan <me@arpitmohan.com>
Co-authored-by: Nupur Singhal <nupursinghal@Nupurs-MacBook-Air.local>
This commit is contained in:
nupur 2020-07-21 16:01:42 +05:30 committed by GitHub
parent 1fc582af08
commit d0e60a1890
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 683 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
app/server/appsmith-plugins/mysqlPlugin/plugin.properties Normal file
View File

@ -0,0 +1,5 @@
plugin.id=mysql-plugin
plugin.class=com.external.plugins.MysqlPlugin
plugin.version=1.0-SNAPSHOT
plugin.provider=tech@appsmith.com
plugin.dependencies=

121
app/server/appsmith-plugins/mysqlPlugin/pom.xml Normal file
View File

@ -0,0 +1,121 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.external.plugins</groupId>
<artifactId>mysqlPlugin</artifactId>
<version>1.0-SNAPSHOT</version>
<name>mysqlPlugin</name>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<java.version>11</java.version>
<maven.compiler.source>${java.version}</maven.compiler.source>
<maven.compiler.target>${java.version}</maven.compiler.target>
<plugin.id>mysql-plugin</plugin.id>
<plugin.class>com.external.plugins.MySqlPlugin</plugin.class>
<plugin.version>1.0-SNAPSHOT</plugin.version>
<plugin.provider>tech@appsmith.com</plugin.provider>
<plugin.dependencies/>
</properties>
<dependencies>
<dependency>
<groupId>org.pf4j</groupId>
<artifactId>pf4j-spring</artifactId>
<version>0.6.0</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.appsmith</groupId>
<artifactId>interfaces</artifactId>
<version>1.0-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>1.18.8</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>8.0.20</version>
<scope>runtime</scope>
</dependency>
<!-- Test Dependencies -->
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.11</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>testcontainers</artifactId>
<version>1.13.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>mysql</artifactId>
<version>1.14.1</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.projectreactor</groupId>
<artifactId>reactor-test</artifactId>
<version>3.2.11.RELEASE</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<version>3.1.0</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>3.2.4</version>
<configuration>
<minimizeJar>false</minimizeJar>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<manifestEntries>
<Plugin-Id>${plugin.id}</Plugin-Id>
<Plugin-Class>${plugin.class}</Plugin-Class>
<Plugin-Version>${plugin.version}</Plugin-Version>
<Plugin-Provider>${plugin.provider}</Plugin-Provider>
</manifestEntries>
</transformer>
</transformers>
</configuration>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

219
app/server/appsmith-plugins/mysqlPlugin/src/main/java/com/external/plugins/MySqlPlugin.java vendored Normal file
View File

@ -0,0 +1,219 @@
package com.external.plugins;
import com.appsmith.external.models.*;
import com.appsmith.external.pluginExceptions.AppsmithPluginError;
import com.appsmith.external.pluginExceptions.AppsmithPluginException;
import com.appsmith.external.plugins.BasePlugin;
import com.appsmith.external.plugins.PluginExecutor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang.ObjectUtils;
import org.pf4j.Extension;
import org.pf4j.PluginWrapper;
import org.springframework.util.CollectionUtils;
import org.springframework.util.StringUtils;
import reactor.core.publisher.Mono;
import java.sql.*;
import java.sql.Connection;
import java.util.*;
import static com.appsmith.external.models.Connection.Mode.READ_ONLY;
public class MySqlPlugin extends BasePlugin {
static final String JDBC_DRIVER = "com.mysql.cj.jdbc.Driver";
private static final String USER = "user";
private static final String PASSWORD = "password";
public MySqlPlugin(PluginWrapper wrapper) {
super(wrapper);
}
@Slf4j
@Extension
public static class MySqlPluginExecutor implements PluginExecutor {
private Mono<Object> pluginErrorMono(Object... args) {
return Mono.error(new AppsmithPluginException(AppsmithPluginError.PLUGIN_ERROR, args));
}
@Override
public Mono<Object> execute(Object connection, DatasourceConfiguration datasourceConfiguration,
ActionConfiguration actionConfiguration) {
Connection conn = (Connection) connection;
String query = actionConfiguration.getBody();
if (query == null) {
return pluginErrorMono("Missing required parameter: Query.");
}
List<Map<String, Object>> rowsList = new ArrayList<>(50);
Statement statement = null;
ResultSet resultSet = null;
try {
statement = conn.createStatement();
boolean isResultSet = statement.execute(query);
if (isResultSet) {
resultSet = statement.getResultSet();
ResultSetMetaData metaData = resultSet.getMetaData();
int colCount = metaData.getColumnCount();
while (resultSet.next()) {
Map<String, Object> row = new HashMap<>(colCount);
for (int i = 1; i <= colCount; i++) {
row.put(metaData.getColumnName(i), resultSet.getObject(i));
}
rowsList.add(row);
}
} else {
rowsList.add(Map.of("affectedRows", statement.getUpdateCount()));
}
} catch (SQLException e) {
return pluginErrorMono(e.getMessage());
} finally {
if (resultSet != null) {
try {
resultSet.close();
} catch (SQLException e) {
log.warn("Error closing MySql ResultSet", e);
}
}
if (statement != null) {
try {
statement.close();
} catch (SQLException e) {
log.warn("Error closing MySql Statement", e);
}
}
}
ActionExecutionResult result = new ActionExecutionResult();
result.setBody(objectMapper.valueToTree(rowsList));
result.setIsExecutionSuccess(true);
log.debug("In the MySqlPlugin, got action execution result: " + result.toString());
return Mono.just(result);
}
@Override
public Mono<Object> datasourceCreate(DatasourceConfiguration datasourceConfiguration) {
try {
Class.forName(JDBC_DRIVER);
} catch (ClassNotFoundException e) {
return pluginErrorMono("Error loading MySql JDBC Driver class.");
}
String url;
AuthenticationDTO authentication = datasourceConfiguration.getAuthentication();
com.appsmith.external.models.Connection configurationConnection = datasourceConfiguration.getConnection();
Properties properties = new Properties();
properties.putAll(Map.of(
USER, authentication.getUsername(),
PASSWORD, authentication.getPassword()
// TODO: Set SSL connection parameters.
));
if (CollectionUtils.isEmpty(datasourceConfiguration.getEndpoints())) {
url = datasourceConfiguration.getUrl();
} else {
StringBuilder urlBuilder = new StringBuilder("jdbc:mysql://");
for (Endpoint endpoint : datasourceConfiguration.getEndpoints()) {
urlBuilder
.append(endpoint.getHost())
.append(':')
.append(ObjectUtils.defaultIfNull(endpoint.getPort(), 3306L))
.append('/');
if (!StringUtils.isEmpty(authentication.getDatabaseName())) {
urlBuilder.append(authentication.getDatabaseName());
}
}
url = urlBuilder.toString();
}
try {
Connection connection = DriverManager.getConnection(url, properties);
connection.setReadOnly(
configurationConnection != null && READ_ONLY.equals(configurationConnection.getMode()));
return Mono.just(connection);
} catch (SQLException e) {
return pluginErrorMono("Error connecting to MySql.", e);
}
}
@Override
public void datasourceDestroy(Object connection) {
Connection conn = (Connection) connection;
try {
if (conn != null) {
conn.close();
}
} catch (SQLException e) {
log.error("Error closing MySQL Connection.", e);
}
}
@Override
public Set<String> validateDatasource(DatasourceConfiguration datasourceConfiguration) {
Set<String> invalids = new HashSet<>();
if (datasourceConfiguration.getConnection() != null
&& datasourceConfiguration.getConnection().getMode() == null) {
invalids.add("Missing Connection Mode.");
}
if (StringUtils.isEmpty(datasourceConfiguration.getUrl()) &&
CollectionUtils.isEmpty(datasourceConfiguration.getEndpoints())) {
invalids.add("Missing endpoint and url");
}
if (datasourceConfiguration.getAuthentication() == null) {
invalids.add("Missing authentication details.");
} else {
if (StringUtils.isEmpty(datasourceConfiguration.getAuthentication().getUsername())) {
invalids.add("Missing username for authentication.");
}
if (StringUtils.isEmpty(datasourceConfiguration.getAuthentication().getPassword())) {
invalids.add("Missing password for authentication.");
}
if (StringUtils.isEmpty(datasourceConfiguration.getAuthentication().getDatabaseName())) {
invalids.add("Missing database name");
}
}
return invalids;
}
@Override
public Mono<DatasourceTestResult> testDatasource(DatasourceConfiguration datasourceConfiguration) {
return datasourceCreate(datasourceConfiguration)
.map(connection -> {
try {
if (connection != null) {
((Connection) connection).close();
}
} catch (SQLException e) {
log.warn("Error closing MySQL connection that was made for testing.", e);
}
return new DatasourceTestResult();
})
.onErrorResume(error -> Mono.just(new DatasourceTestResult(error.getMessage())));
}
}
}

152
app/server/appsmith-plugins/mysqlPlugin/src/main/resources/form.json Normal file
View File

@ -0,0 +1,152 @@
{
"form": [
{
"sectionName": "Connection",
"id": 1,
"children": [
{
"label": "Connection Mode",
"configProperty": "datasourceConfiguration.connection.mode",
"controlType": "DROP_DOWN",
"isRequired": true,
"initialValue": "READ_WRITE",
"options": [
{
"label": "Read Only",
"value": "READ_ONLY"
},
{
"label": "Read / Write",
"value": "READ_WRITE"
}
]
},
{
"sectionName": null,
"children": [
{
"label": "Host Address",
"configProperty": "datasourceConfiguration.endpoints[*].host",
"controlType": "KEYVALUE_ARRAY"
},
{
"label": "Port",
"configProperty": "datasourceConfiguration.endpoints[*].port",
"dataType": "NUMBER",
"controlType": "KEYVALUE_ARRAY"
}
]
},
{
"label": "Database Name",
"configProperty": "datasourceConfiguration.authentication.databaseName",
"controlType": "INPUT_TEXT",
"placeholderText": "Database name",
"initialValue": "admin"
}
]
},
{
"sectionName": "Authentication",
"id": 2,
"children": [
{
"sectionName": null,
"children": [
{
"label": "Username",
"configProperty": "datasourceConfiguration.authentication.username",
"controlType": "INPUT_TEXT",
"placeholderText": "Username"
},
{
"label": "Password",
"configProperty": "datasourceConfiguration.authentication.password",
"dataType": "PASSWORD",
"controlType": "INPUT_TEXT",
"placeholderText": "Password"
}
]
}
]
},
{
"id": 3,
"sectionName": "SSL (optional)",
"children": [
{
"label": "SSL Mode",
"configProperty": "datasourceConfiguration.connection.ssl.authType",
"controlType": "DROP_DOWN",
"options": [
{
"label": "No SSL",
"value": "NO_SSL"
},
{
"label": "Allow",
"value": "ALLOW"
},
{
"label": "Prefer",
"value": "PREFER"
},
{
"label": "Require",
"value": "REQUIRE"
},
{
"label": "Disable",
"value": "DISABLE"
},
{
"label": "Verify-CA",
"value": "VERIFY_CA"
},
{
"label": "Verify-Full",
"value": "VERIFY_FULL"
}
]
},
{
"sectionName": null,
"children": [
{
"label": "Key File",
"configProperty": "datasourceConfiguration.connection.ssl.keyFile",
"controlType": "FILE_PICKER"
},
{
"label": "Certificate",
"configProperty": "datasourceConfiguration.connection.ssl.certificateFile",
"controlType": "FILE_PICKER"
}
]
},
{
"sectionName": null,
"children": [
{
"label": "CA Certificate",
"configProperty": "datasourceConfiguration.connection.ssl.caCertificateFile",
"controlType": "FILE_PICKER"
},
{
"label": "PEM Certificate",
"configProperty": "datasourceConfiguration.connection.ssl.pemCertificate.file",
"controlType": "FILE_PICKER"
},
{
"label": "PEM Passphrase",
"configProperty": "datasourceConfiguration.connection.ssl.pemCertificate.password",
"dataType": "PASSWORD",
"controlType": "INPUT_TEXT",
"placeholderText": "PEM Passphrase"
}
]
}
]
}
]
}

4
app/server/appsmith-plugins/mysqlPlugin/src/main/resources/templates/CREATE.sql Normal file
View File

@ -0,0 +1,4 @@
INSERT INTO users
(id, name, gender, avatar, email, address, role)
VALUES
(?, ?, ?, ?, ?, ?, ?);

1
app/server/appsmith-plugins/mysqlPlugin/src/main/resources/templates/DELETE.sql Normal file
View File

@ -0,0 +1 @@
DELETE FROM users WHERE id = ?;

1
app/server/appsmith-plugins/mysqlPlugin/src/main/resources/templates/SELECT.sql Normal file
View File

@ -0,0 +1 @@
SELECT * FROM users ORDER BY id LIMIT 10;

3
app/server/appsmith-plugins/mysqlPlugin/src/main/resources/templates/UPDATE.sql Normal file
View File

@ -0,0 +1,3 @@
UPDATE users
SET status = 'APPROVED'
WHERE id = 1;

143
app/server/appsmith-plugins/mysqlPlugin/src/test/java/com/external/plugins/MySqlPluginTest.java vendored Normal file
View File

@ -0,0 +1,143 @@
package com.external.plugins;
import com.appsmith.external.models.*;
import lombok.extern.log4j.Log4j;
import org.junit.Before;
import org.junit.ClassRule;
import org.junit.Test;
import org.testcontainers.containers.MySQLContainer;
import reactor.core.publisher.Mono;
import reactor.test.StepVerifier;
import java.sql.Connection;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import static org.junit.Assert.*;
@Log4j
public class MySqlPluginTest {
MySqlPlugin.MySqlPluginExecutor pluginExecutor = new MySqlPlugin.MySqlPluginExecutor();
@ClassRule
public static MySQLContainer mySQLContainer = new MySQLContainer()
.withUsername("mysql").withPassword("password").withDatabaseName("mysql");
String address;
Integer port;
String username, password;
DatasourceConfiguration dsConfig;
@Before
public void setUp() {
address = mySQLContainer.getContainerIpAddress();
port = mySQLContainer.getFirstMappedPort();
username = mySQLContainer.getUsername();
password = mySQLContainer.getPassword();
createDatasourceConfiguration();
}
private DatasourceConfiguration createDatasourceConfiguration() {
AuthenticationDTO authDTO = new AuthenticationDTO();
authDTO.setAuthType(AuthenticationDTO.Type.USERNAME_PASSWORD);
authDTO.setUsername(username);
authDTO.setPassword(password);
authDTO.setDatabaseName("mysql");
Endpoint endpoint = new Endpoint();
endpoint.setHost(address);
endpoint.setPort(port.longValue());
dsConfig = new DatasourceConfiguration();
dsConfig.setAuthentication(authDTO);
dsConfig.setEndpoints(List.of(endpoint));
return dsConfig;
}
@Test
public void testConnectMySQLContainer() {
Mono<Object> dsConnectionMono = pluginExecutor.datasourceCreate(dsConfig);
StepVerifier.create(dsConnectionMono)
.assertNext(connection -> {
java.sql.Connection conn = (Connection) connection;
assertNotNull(conn);
})
.verifyComplete();
}
@Test
public void testExecute() {
Mono<Object> dsConnectionMono = pluginExecutor.datasourceCreate(dsConfig);
ActionConfiguration actionConfiguration = new ActionConfiguration();
actionConfiguration.setBody("show databases");
Mono<Object> executeMono = dsConnectionMono.flatMap(conn -> pluginExecutor.execute(conn, dsConfig, actionConfiguration));
StepVerifier.create(executeMono)
.assertNext(obj -> {
ActionExecutionResult result = (ActionExecutionResult) obj;
System.out.println(result);
assertNotNull(result);
assertTrue(result.getIsExecutionSuccess());
assertNotNull(result.getBody());
})
.verifyComplete();
}
@Test
public void testValidateDatasourceNullCredentials() {
dsConfig.setConnection(new com.appsmith.external.models.Connection());
dsConfig.getAuthentication().setUsername(null);
dsConfig.getAuthentication().setPassword(null);
dsConfig.getAuthentication().setDatabaseName("someDbName");
Set<String> output = pluginExecutor.validateDatasource(dsConfig);
assertTrue(output.contains("Missing username for authentication."));
assertTrue(output.contains("Missing password for authentication."));
}
@Test
public void testValidateDatasourceMissingDBName() {
dsConfig.getAuthentication().setDatabaseName("");
Set<String> output = pluginExecutor.validateDatasource(dsConfig);
assertEquals(output.size(), 1);
assertTrue(output.contains("Missing database name"));
}
@Test
public void testValidateDatasourceNullEndpoint() {
dsConfig.setEndpoints(null);
Set<String> output = pluginExecutor.validateDatasource(dsConfig);
assertEquals(output.size(), 1);
assertTrue(output.contains("Missing endpoint and url"));
}
/* checking that the connection is being closed after the datadourceDestroy method is being called
NOT : this test case will fail in case of a SQL Exception
*/
@Test
public void testDatasourceDestroy() {
Mono<Object> connectionMono = pluginExecutor.datasourceCreate(dsConfig);
StepVerifier.create(connectionMono)
.assertNext(connection -> {
java.sql.Connection conn = (Connection) connection;
pluginExecutor.datasourceDestroy(conn);
try {
assertEquals(conn.isClosed(), true);
} catch (SQLException e) {
e.printStackTrace();
}
})
.verifyComplete();
}
}

1
app/server/appsmith-plugins/pom.xml
View File

@ -19,6 +19,7 @@
<module>restApiPlugin</module>
<module>mongoPlugin</module>
<module>rapidApiPlugin</module>
<module>mysqlPlugin</module>
</modules>
</project>

2
app/server/appsmith-plugins/postgresPlugin/plugin.properties
View File

@ -2,4 +2,4 @@ plugin.id=postgres-plugin
plugin.class=com.external.plugins.PostgresPlugin
plugin.version=1.0-SNAPSHOT
plugin.provider=tech@appsmith.com
plugin.dependencies=
plugin.dependencies=

31
app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/DatabaseChangelog.java
View File

@ -491,4 +491,35 @@ public class DatabaseChangelog {
mongoTemplate.save(datasource);
}
}
@ChangeSet(order = "018", id = "install-mysql-plugins", author = "")
public void mysqlPlugin(MongoTemplate mongoTemplate) {
Plugin plugin1 = new Plugin();
plugin1.setName("Mysql");
plugin1.setType(PluginType.DB);
plugin1.setPackageName("mysql-plugin");
plugin1.setUiComponent("DbEditorForm");
plugin1.setIconLocation("https://s3.us-east-2.amazonaws.com/assets.appsmith.com/Mysql.png");
plugin1.setDefaultInstall(true);
try {
mongoTemplate.insert(plugin1);
} catch (DuplicateKeyException e) {
log.warn("mysql-plugin already present in database.");
}
for (Organization organization : mongoTemplate.findAll(Organization.class)) {
if (CollectionUtils.isEmpty(organization.getPlugins())) {
organization.setPlugins(new ArrayList<>());
}
final Set<String> installedPlugins = organization.getPlugins()
.stream().map(OrganizationPlugin::getPluginId).collect(Collectors.toSet());
if (!installedPlugins.contains(plugin1.getId())) {
organization.getPlugins()
.add(new OrganizationPlugin(plugin1.getId(), OrganizationPluginStatus.FREE));
}
mongoTemplate.save(organization);
}
}
}

2
app/server/scripts/start-dev-server.sh
View File

@ -10,4 +10,4 @@ if [[ -f .env ]]; then
source .env
fi
exec java -jar dist/server-*.jar
(cd dist && exec java -jar server-*.jar)