diff --git a/app/server/appsmith-plugins/elasticSearchPlugin/plugin.properties b/app/server/appsmith-plugins/elasticSearchPlugin/plugin.properties new file mode 100644 index 0000000000..2d0f796852 --- /dev/null +++ b/app/server/appsmith-plugins/elasticSearchPlugin/plugin.properties @@ -0,0 +1,5 @@ +plugin.id=elasticsearch-plugin +plugin.class=com.external.plugins.ElasticSearchPlugin +plugin.version=1.0-SNAPSHOT +plugin.provider=tech@appsmith.com +plugin.dependencies= diff --git a/app/server/appsmith-plugins/elasticSearchPlugin/pom.xml b/app/server/appsmith-plugins/elasticSearchPlugin/pom.xml new file mode 100644 index 0000000000..8f71ee9fe4 --- /dev/null +++ b/app/server/appsmith-plugins/elasticSearchPlugin/pom.xml @@ -0,0 +1,132 @@ + + + + 4.0.0 + + com.external.plugins + elasticSearchPlugin + 1.0-SNAPSHOT + + elasticSearchPlugin + + + UTF-8 + 11 + ${java.version} + ${java.version} + elasticsearch-plugin + com.external.plugins.ElasticSearchPlugin + 1.0-SNAPSHOT + tech@appsmith.com + + + + + + + org.pf4j + pf4j-spring + 0.6.0 + provided + + + + com.appsmith + interfaces + 1.0-SNAPSHOT + provided + + + + org.projectlombok + lombok + 1.18.8 + provided + + + + org.elasticsearch.client + elasticsearch-rest-client + 7.9.2 + + + + + junit + junit + 4.11 + test + + + + io.projectreactor + reactor-test + 3.3.5.RELEASE + test + + + + org.testcontainers + testcontainers + 1.15.0-rc2 + test + + + + org.testcontainers + elasticsearch + 1.15.0-rc2 + test + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.2.4 + + false + + + + ${plugin.id} + ${plugin.class} + ${plugin.version} + ${plugin.provider} + + + + + + + package + + shade + + + + + + maven-dependency-plugin + + + copy-dependencies + package + + copy-dependencies + + + runtime + ${project.build.directory}/lib + + + + + + + + diff --git a/app/server/appsmith-plugins/elasticSearchPlugin/src/main/java/com/external/plugins/ElasticSearchPlugin.java b/app/server/appsmith-plugins/elasticSearchPlugin/src/main/java/com/external/plugins/ElasticSearchPlugin.java new file mode 100644 index 0000000000..82ef9d9a76 --- /dev/null +++ b/app/server/appsmith-plugins/elasticSearchPlugin/src/main/java/com/external/plugins/ElasticSearchPlugin.java @@ -0,0 +1,202 @@ +package com.external.plugins; + +import com.appsmith.external.models.ActionConfiguration; +import com.appsmith.external.models.ActionExecutionResult; +import com.appsmith.external.models.AuthenticationDTO; +import com.appsmith.external.models.DatasourceConfiguration; +import com.appsmith.external.models.DatasourceTestResult; +import com.appsmith.external.models.Endpoint; +import com.appsmith.external.pluginExceptions.AppsmithPluginError; +import com.appsmith.external.pluginExceptions.AppsmithPluginException; +import com.appsmith.external.plugins.BasePlugin; +import com.appsmith.external.plugins.PluginExecutor; +import lombok.extern.slf4j.Slf4j; +import org.apache.http.Header; +import org.apache.http.HttpHost; +import org.apache.http.StatusLine; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.entity.ContentType; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.message.BasicHeader; +import org.apache.http.nio.entity.NStringEntity; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestClientBuilder; +import org.pf4j.Extension; +import org.pf4j.PluginWrapper; +import org.springframework.util.CollectionUtils; +import org.springframework.util.StringUtils; +import reactor.core.publisher.Mono; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +public class ElasticSearchPlugin extends BasePlugin { + + public ElasticSearchPlugin(PluginWrapper wrapper) { + super(wrapper); + } + + @Slf4j + @Extension + public static class ElasticSearchPluginExecutor implements PluginExecutor { + + @Override + public Mono execute(RestClient client, + DatasourceConfiguration datasourceConfiguration, + ActionConfiguration actionConfiguration) { + final ActionExecutionResult result = new ActionExecutionResult(); + + String body = actionConfiguration.getBody(); + + final String path = actionConfiguration.getPath(); + final Request request = new Request(actionConfiguration.getHttpMethod().toString(), path); + ContentType contentType = ContentType.APPLICATION_JSON; + + if (isBulkQuery(path)) { + contentType = ContentType.create("application/x-ndjson"); + + // If body is a JSON Array, convert it to an ND-JSON string. + if (body != null && body.trim().startsWith("[")) { + final StringBuilder ndJsonBuilder = new StringBuilder(); + try { + List commands = objectMapper.readValue(body, ArrayList.class); + for (Object object : commands) { + ndJsonBuilder.append(objectMapper.writeValueAsString(object)).append("\n"); + } + } catch (IOException e) { + final String message = "Error converting array to ND-JSON: " + e.getMessage(); + log.warn(message, e); + return Mono.error(new AppsmithPluginException(AppsmithPluginError.PLUGIN_ERROR, message)); + } + body = ndJsonBuilder.toString(); + } + } + + if (body != null) { + request.setEntity(new NStringEntity(body, contentType)); + } + + try { + final String responseBody = new String( + client.performRequest(request).getEntity().getContent().readAllBytes()); + result.setBody(objectMapper.readValue(responseBody, HashMap.class)); + } catch (IOException e) { + final String message = "Error performing request: " + e.getMessage(); + log.warn(message, e); + return Mono.error(new AppsmithPluginException(AppsmithPluginError.PLUGIN_ERROR, message)); + } + + result.setIsExecutionSuccess(true); + return Mono.just(result); + } + + private static boolean isBulkQuery(String path) { + return path.split("\\?", 1)[0].matches(".*\\b_bulk$"); + } + + @Override + public Mono datasourceCreate(DatasourceConfiguration datasourceConfiguration) { + final List hosts = new ArrayList<>(); + + for (Endpoint endpoint : datasourceConfiguration.getEndpoints()) { + hosts.add(new HttpHost(endpoint.getHost(), endpoint.getPort().intValue(), "http")); + } + + final RestClientBuilder clientBuilder = RestClient.builder(hosts.toArray(new HttpHost[]{})); + + final AuthenticationDTO authentication = datasourceConfiguration.getAuthentication(); + if (authentication != null + && !StringUtils.isEmpty(authentication.getUsername()) + && !StringUtils.isEmpty(authentication.getPassword())) { + final CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials( + AuthScope.ANY, + new UsernamePasswordCredentials(authentication.getUsername(), authentication.getPassword()) + ); + + clientBuilder + .setHttpClientConfigCallback( + httpClientBuilder -> httpClientBuilder + .setDefaultCredentialsProvider(credentialsProvider) + ); + } + + if (!CollectionUtils.isEmpty(datasourceConfiguration.getHeaders())) { + clientBuilder.setDefaultHeaders( + (Header[]) datasourceConfiguration.getHeaders() + .stream() + .map(h -> new BasicHeader(h.getKey(), h.getValue())) + .toArray() + ); + } + + return Mono.just(clientBuilder.build()); + } + + @Override + public void datasourceDestroy(RestClient client) { + try { + client.close(); + } catch (IOException e) { + log.warn("Error closing connection to ElasticSearch.", e); + } + } + + @Override + public Set validateDatasource(DatasourceConfiguration datasourceConfiguration) { + Set invalids = new HashSet<>(); + + if (CollectionUtils.isEmpty(datasourceConfiguration.getEndpoints())) { + invalids.add("No endpoint provided. Please provide a host:port where ElasticSearch is reachable."); + } + + return invalids; + } + + @Override + public Mono testDatasource(DatasourceConfiguration datasourceConfiguration) { + return datasourceCreate(datasourceConfiguration) + .map(client -> { + if (client == null) { + return new DatasourceTestResult("Null client object to ElasticSearch."); + } + + // This HEAD request is to check if an index exists. It response with 200 if the index exists, + // 404 if it doesn't. We just check for either of these two. + // Ref: https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-exists.html + Request request = new Request("HEAD", "/potentially-missing-index?local=true"); + + final Response response; + try { + response = client.performRequest(request); + } catch (IOException e) { + return new DatasourceTestResult("Error running HEAD request: " + e.getMessage()); + } + + final StatusLine statusLine = response.getStatusLine(); + + try { + client.close(); + } catch (IOException e) { + log.warn("Error closing ElasticSearch client that was made for testing.", e); + } + + if (statusLine.getStatusCode() != 404 && statusLine.getStatusCode() != 200) { + return new DatasourceTestResult( + "Unexpected response from ElasticSearch: " + statusLine); + } + + return new DatasourceTestResult(); + }) + .onErrorResume(error -> Mono.just(new DatasourceTestResult(error.getMessage()))); + } + } +} diff --git a/app/server/appsmith-plugins/elasticSearchPlugin/src/main/resources/editor.json b/app/server/appsmith-plugins/elasticSearchPlugin/src/main/resources/editor.json new file mode 100644 index 0000000000..ee3cfc1645 --- /dev/null +++ b/app/server/appsmith-plugins/elasticSearchPlugin/src/main/resources/editor.json @@ -0,0 +1,45 @@ +{ + "editor": [ + { + "sectionName": "", + "id": 1, + "children": [ + { + "label": "Method", + "configProperty": "actionConfiguration.httpMethod", + "controlType": "DROP_DOWN", + "isRequired": true, + "initialValue": "GET", + "options": [ + { + "label": "GET", + "value": "GET" + }, + { + "label": "POST", + "value": "POST" + }, + { + "label": "PUT", + "value": "PUT" + }, + { + "label": "DELETE", + "value": "DELETE" + } + ] + }, + { + "label": "Path", + "configProperty": "actionConfiguration.path", + "controlType": "INPUT_TEXT" + }, + { + "label": "Body", + "configProperty": "actionConfiguration.body", + "controlType": "QUERY_DYNAMIC_TEXT" + } + ] + } + ] +} diff --git a/app/server/appsmith-plugins/elasticSearchPlugin/src/main/resources/form.json b/app/server/appsmith-plugins/elasticSearchPlugin/src/main/resources/form.json new file mode 100644 index 0000000000..8efcc6f90e --- /dev/null +++ b/app/server/appsmith-plugins/elasticSearchPlugin/src/main/resources/form.json @@ -0,0 +1,52 @@ +{ + "form": [ + { + "sectionName": "Connection", + "children": [ + { + "sectionName": null, + "children": [ + { + "label": "Host Address", + "configProperty": "datasourceConfiguration.endpoints[*].host", + "controlType": "KEYVALUE_ARRAY", + "validationMessage": "Please enter a valid host", + "validationRegex": "^((?![/:]).)*$" + }, + { + "label": "Port", + "configProperty": "datasourceConfiguration.endpoints[*].port", + "dataType": "NUMBER", + "controlType": "KEYVALUE_ARRAY" + } + ] + } + ] + }, + { + "sectionName": "Authentication", + "children": [ + { + "label": "Username for Basic Auth", + "configProperty": "datasourceConfiguration.authentication.username", + "controlType": "INPUT_TEXT", + "placeholderText": "Username" + }, + { + "label": "Password for Basic Auth", + "configProperty": "datasourceConfiguration.authentication.password", + "dataType": "PASSWORD", + "controlType": "INPUT_TEXT", + "placeholderText": "Password" + }, + { + "label": "Authorization Header (if username, password are not set)", + "configProperty": "datasourceConfiguration.headers[0]", + "controlType": "FIXED_KEY_INPUT", + "fixedKey": "Authorization", + "placeholderText": "Authorization Header" + } + ] + } + ] +} diff --git a/app/server/appsmith-plugins/elasticSearchPlugin/src/test/java/com/external/plugins/ElasticSearchPluginTest.java b/app/server/appsmith-plugins/elasticSearchPlugin/src/test/java/com/external/plugins/ElasticSearchPluginTest.java new file mode 100644 index 0000000000..921c09150f --- /dev/null +++ b/app/server/appsmith-plugins/elasticSearchPlugin/src/test/java/com/external/plugins/ElasticSearchPluginTest.java @@ -0,0 +1,206 @@ +package com.external.plugins; + +import com.appsmith.external.models.ActionConfiguration; +import com.appsmith.external.models.ActionExecutionResult; +import com.appsmith.external.models.DatasourceConfiguration; +import com.appsmith.external.models.Endpoint; +import lombok.extern.slf4j.Slf4j; +import org.apache.http.HttpHost; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.RestClient; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.springframework.http.HttpMethod; +import org.testcontainers.elasticsearch.ElasticsearchContainer; +import reactor.core.publisher.Mono; +import reactor.test.StepVerifier; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +@Slf4j +public class ElasticSearchPluginTest { + ElasticSearchPlugin.ElasticSearchPluginExecutor pluginExecutor = new ElasticSearchPlugin.ElasticSearchPluginExecutor(); + + @ClassRule + public static final ElasticsearchContainer container = + new ElasticsearchContainer("docker.elastic.co/elasticsearch/elasticsearch:6.4.1") + .withEnv("discovery.type", "single-node"); + + private static final DatasourceConfiguration dsConfig = new DatasourceConfiguration(); + + @BeforeClass + public static void setUp() throws IOException { + final Integer port = container.getMappedPort(9200); + + final RestClient client = RestClient.builder( + new HttpHost("localhost", port, "http") + ).build(); + + Request request; + + request = new Request("PUT", "/planets/doc/id1"); + request.setJsonEntity("{\"name\": \"Mercury\"}"); + client.performRequest(request); + + request = new Request("PUT", "/planets/doc/id2"); + request.setJsonEntity("{\"name\": \"Venus\"}"); + client.performRequest(request); + + request = new Request("PUT", "/planets/doc/id3"); + request.setJsonEntity("{\"name\": \"Earth\"}"); + client.performRequest(request); + + client.close(); + + dsConfig.setEndpoints(List.of(new Endpoint("localhost", port.longValue()))); + } + + private Mono execute(HttpMethod method, String path, String body) { + final ActionConfiguration actionConfiguration = new ActionConfiguration(); + actionConfiguration.setHttpMethod(method); + actionConfiguration.setPath(path); + actionConfiguration.setBody(body); + + return pluginExecutor + .datasourceCreate(dsConfig) + .flatMap(conn -> pluginExecutor.execute(conn, dsConfig, actionConfiguration)); + } + + @Test + public void testGet() { + StepVerifier.create(execute(HttpMethod.GET, "/planets/doc/id1", null)) + .assertNext(result -> { + assertNotNull(result); + assertTrue(result.getIsExecutionSuccess()); + assertNotNull(result.getBody()); + final Map resultBody = (Map) result.getBody(); + assertEquals("Mercury", ((Map) resultBody.get("_source")).get("name")); + }) + .verifyComplete(); + } + + @Test + public void testMultiGet() { + final String contentJson = "{\n" + + " \"docs\": [\n" + + " {\n" + + " \"_index\": \"planets\",\n" + + " \"_id\": \"id1\"\n" + + " },\n" + + " {\n" + + " \"_index\": \"planets\",\n" + + " \"_id\": \"id2\"\n" + + " }\n" + + " ]\n" + + "}"; + StepVerifier.create(execute(HttpMethod.GET, "/planets/_mget", contentJson)) + .assertNext(result -> { + assertNotNull(result); + assertTrue(result.getIsExecutionSuccess()); + assertNotNull(result.getBody()); + final List docs = ((Map>) result.getBody()).get("docs"); + assertEquals(2, docs.size()); + }) + .verifyComplete(); + } + + @Test + public void testPutCreate() { + final String contentJson = "{\"name\": \"Pluto\"}"; + StepVerifier.create(execute(HttpMethod.PUT, "/planets/doc/id9", contentJson)) + .assertNext(result -> { + assertNotNull(result); + assertTrue(result.getIsExecutionSuccess()); + assertNotNull(result.getBody()); + final Map resultBody = (Map) result.getBody(); + assertEquals("created", resultBody.get("result")); + assertEquals("id9", resultBody.get("_id")); + }) + .verifyComplete(); + } + + @Test + public void testPutUpdate() { + final String contentJson = "{\"name\": \"New Venus\"}"; + StepVerifier.create(execute(HttpMethod.PUT, "/planets/doc/id2", contentJson)) + .assertNext(result -> { + assertNotNull(result); + assertTrue(result.getIsExecutionSuccess()); + assertNotNull(result.getBody()); + final Map resultBody = (Map) result.getBody(); + assertEquals("updated", resultBody.get("result")); + assertEquals("id2", resultBody.get("_id")); + }) + .verifyComplete(); + } + + @Test + public void testDelete() { + StepVerifier.create(execute(HttpMethod.DELETE, "/planets/doc/id3", null)) + .assertNext(result -> { + assertNotNull(result); + assertTrue(result.getIsExecutionSuccess()); + assertNotNull(result.getBody()); + final Map resultBody = (Map) result.getBody(); + assertEquals("deleted", resultBody.get("result")); + assertEquals("id3", resultBody.get("_id")); + }) + .verifyComplete(); + } + + @Test + public void testBulkWithArrayBody() { + final String contentJson = "[\n" + + " { \"index\" : { \"_index\" : \"test1\", \"_type\": \"doc\", \"_id\" : \"1\" } },\n" + + " { \"field1\" : \"value1\" },\n" + + " { \"delete\" : { \"_index\" : \"test1\", \"_type\": \"doc\", \"_id\" : \"2\" } },\n" + + " { \"create\" : { \"_index\" : \"test1\", \"_type\": \"doc\", \"_id\" : \"3\" } },\n" + + " { \"field1\" : \"value3\" },\n" + + " { \"update\" : {\"_id\" : \"1\", \"_type\": \"doc\", \"_index\" : \"test1\"} },\n" + + " { \"doc\" : {\"field2\" : \"value2\"} }\n" + + "]"; + + StepVerifier.create(execute(HttpMethod.POST, "/_bulk", contentJson)) + .assertNext(result -> { + assertNotNull(result); + assertTrue(result.getIsExecutionSuccess()); + assertNotNull(result.getBody()); + final Map resultBody = (Map) result.getBody(); + assertFalse((Boolean) resultBody.get("errors")); + assertEquals(4, ((List) resultBody.get("items")).size()); + }) + .verifyComplete(); + } + + @Test + public void testBulkWithDirectBody() { + final String contentJson = + "{ \"index\" : { \"_index\" : \"test2\", \"_type\": \"doc\", \"_id\" : \"1\" } }\n" + + "{ \"field1\" : \"value1\" }\n" + + "{ \"delete\" : { \"_index\" : \"test2\", \"_type\": \"doc\", \"_id\" : \"2\" } }\n" + + "{ \"create\" : { \"_index\" : \"test2\", \"_type\": \"doc\", \"_id\" : \"3\" } }\n" + + "{ \"field1\" : \"value3\" }\n" + + "{ \"update\" : {\"_id\" : \"1\", \"_type\": \"doc\", \"_index\" : \"test2\"} }\n" + + "{ \"doc\" : {\"field2\" : \"value2\"} }\n"; + + StepVerifier.create(execute(HttpMethod.POST, "/_bulk", contentJson)) + .assertNext(result -> { + assertNotNull(result); + assertTrue(result.getIsExecutionSuccess()); + assertNotNull(result.getBody()); + final Map resultBody = (Map) result.getBody(); + assertFalse((Boolean) resultBody.get("errors")); + assertEquals(4, ((List) resultBody.get("items")).size()); + }) + .verifyComplete(); + } + +} diff --git a/app/server/appsmith-plugins/pom.xml b/app/server/appsmith-plugins/pom.xml index 83f2d5f1bd..48bb4d74e3 100644 --- a/app/server/appsmith-plugins/pom.xml +++ b/app/server/appsmith-plugins/pom.xml @@ -20,6 +20,7 @@ mongoPlugin rapidApiPlugin mysqlPlugin + elasticSearchPlugin diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/DatabaseChangelog.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/DatabaseChangelog.java index 856884a298..f19d6c62b8 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/DatabaseChangelog.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/DatabaseChangelog.java @@ -936,6 +936,26 @@ public class DatabaseChangelog { ); } + @ChangeSet(order = "027", id = "add-elastic-search-plugin", author = "") + public void addElasticSearchPlugin(MongoTemplate mongoTemplate) { + Plugin plugin1 = new Plugin(); + plugin1.setName("ElasticSearch"); + plugin1.setType(PluginType.DB); + plugin1.setPackageName("elasticsearch-plugin"); + plugin1.setUiComponent("DbEditorForm"); + plugin1.setResponseType(Plugin.ResponseType.JSON); + plugin1.setIconLocation("https://s3.us-east-2.amazonaws.com/assets.appsmith.com/ElasticSearch.jpg"); + plugin1.setDocumentationLink("https://docs.appsmith.com/core-concepts/connecting-to-databases/querying-elasticsearch"); + plugin1.setDefaultInstall(true); + try { + mongoTemplate.insert(plugin1); + } catch (DuplicateKeyException e) { + log.warn(plugin1.getPackageName() + " already present in database."); + } + + installPluginToAllOrganizations(mongoTemplate, plugin1.getId()); + } + private void installPluginToAllOrganizations(MongoTemplate mongoTemplate, String pluginId) { for (Organization organization : mongoTemplate.findAll(Organization.class)) { if (CollectionUtils.isEmpty(organization.getPlugins())) { @@ -954,6 +974,4 @@ public class DatabaseChangelog { } } - - }