From 81d5cffc44f3db815fbeb68731d09e6ee67e8d9d Mon Sep 17 00:00:00 2001 From: Trisha Anand Date: Thu, 18 Nov 2021 15:45:43 +0530 Subject: [PATCH] feat: UQI where clause support (#9051) * Added parsing of where condition to Condition format * Refactoring the name of functions to denote old style before implementing UQI where clause * Updated the error message for data type problem for mixed data tyeps * Generating complex logical expression. * Working version of filtering happening without refactoring of code * Added where clause configuration in List files in a bucket command. Not able to render due to some configuration issue. * Untested code completion * To be reverted. Ayush's changes. * Tested where condition on S3 List * Made AND the default option for where clause * where clause working in case of no valid inputs provided. * Added parallel test cases as that were existing for the old where clause * Revert "To be reverted. Ayush's changes." This reverts commit a0f9b72e241f0688b7ef07cea8c3017473423512. * Making equality the default option in a new where clause for LIST command * Added test cases incorporating the review comments. * Updated the options for the where command in S3 plugin. Removed the comparison operators except equality, non equality and belonging (in and not in) operators. * Added catching of exception while parsing the operator into known appsmith condition types * Reusing objectmapper from BasePlugin instead of creating a new one here. --- .../constants/ConditionalOperator.java | 12 + .../external/constants/FieldName.java | 5 + .../pluginExceptions/AppsmithPluginError.java | 4 +- .../external/helpers/PluginUtils.java | 53 ++ .../appsmith/external/models/Condition.java | 30 +- .../external/services/FilterDataService.java | 198 ++++- .../external/helpers/PluginUtilsTest.java | 119 +++ .../services/FilterDataServiceTest.java | 825 ++++++++++++++++++ .../com/external/plugins/AmazonS3Plugin.java | 21 + .../external/plugins/constants/FieldName.java | 2 + .../src/main/resources/editor/list.json | 34 + 11 files changed, 1288 insertions(+), 15 deletions(-) create mode 100644 app/server/appsmith-interfaces/src/test/java/com/appsmith/external/helpers/PluginUtilsTest.java diff --git a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/constants/ConditionalOperator.java b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/constants/ConditionalOperator.java index 0764bc2a0c..97b0d1f1fc 100644 --- a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/constants/ConditionalOperator.java +++ b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/constants/ConditionalOperator.java @@ -61,4 +61,16 @@ public enum ConditionalOperator { return "not in"; } }, + AND { + @Override + public String toString() { + return "and"; + } + }, + OR { + @Override + public String toString() { + return "or"; + } + }, } diff --git a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/constants/FieldName.java b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/constants/FieldName.java index b474748221..4454a6a811 100644 --- a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/constants/FieldName.java +++ b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/constants/FieldName.java @@ -7,4 +7,9 @@ public class FieldName { public static final String TOKEN_RESPONSE = "tokenResponse"; public static final String PASSWORD = "password"; + + public static final String CHILDREN = "children"; + public static final String KEY = "key"; + public static final String CONDITION = "condition"; + public static final String VALUE = "value"; } diff --git a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/exceptions/pluginExceptions/AppsmithPluginError.java b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/exceptions/pluginExceptions/AppsmithPluginError.java index a6f6808743..d7c8282b7e 100644 --- a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/exceptions/pluginExceptions/AppsmithPluginError.java +++ b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/exceptions/pluginExceptions/AppsmithPluginError.java @@ -33,8 +33,10 @@ public enum AppsmithPluginError { "datasource", ErrorType.CONNECTIVITY_ERROR), PLUGIN_AUTHENTICATION_ERROR(401, 4000, "Invalid authentication credentials. Please check datasource configuration.", AppsmithErrorAction.DEFAULT, "Datasource authentication error", ErrorType.AUTHENTICATION_ERROR), - PLUGIN_IN_MEMORY_FILTERING_ERROR(500, 5009, "{0}", + PLUGIN_IN_MEMORY_FILTERING_ERROR(500, 5010, "{0}", AppsmithErrorAction.LOG_EXTERNALLY, "Appsmith In Memory Filtering Failed", ErrorType.INTERNAL_ERROR), + PLUGIN_UQI_WHERE_CONDITION_UNKNOWN(500, 5011, "{0} is not a known conditional operator. Please reach out to Appsmith customer support to report this", + AppsmithErrorAction.LOG_EXTERNALLY, "Where condition could not be parsed", ErrorType.INTERNAL_ERROR), ; private final Integer httpErrorCode; diff --git a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/helpers/PluginUtils.java b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/helpers/PluginUtils.java index 1ac6266b52..9b5450ef8c 100644 --- a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/helpers/PluginUtils.java +++ b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/helpers/PluginUtils.java @@ -1,7 +1,12 @@ package com.appsmith.external.helpers; +import com.appsmith.external.constants.ConditionalOperator; +import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginError; +import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginException; +import com.appsmith.external.models.Condition; import com.appsmith.external.models.DatasourceConfiguration; import com.appsmith.external.models.Endpoint; +import lombok.extern.slf4j.Slf4j; import org.springframework.util.CollectionUtils; import org.springframework.util.StringUtils; @@ -18,6 +23,12 @@ import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.IntStream; +import static com.appsmith.external.constants.FieldName.CHILDREN; +import static com.appsmith.external.constants.FieldName.CONDITION; +import static com.appsmith.external.constants.FieldName.KEY; +import static com.appsmith.external.constants.FieldName.VALUE; + +@Slf4j public class PluginUtils { /** @@ -197,4 +208,46 @@ public class PluginUtils { return message; } + + public static Condition parseWhereClause(Map whereClause) { + Condition condition = new Condition(); + + Object unparsedOperator = whereClause.getOrDefault(CONDITION, ConditionalOperator.EQ.toString()); + + ConditionalOperator operator; + try { + + operator = ConditionalOperator.valueOf(((String) unparsedOperator).trim().toUpperCase()); + + } catch (IllegalArgumentException e) { + // The operator could not be cast into a known type. Throw an exception + log.error(e.getMessage()); + throw new AppsmithPluginException(AppsmithPluginError.PLUGIN_UQI_WHERE_CONDITION_UNKNOWN, unparsedOperator); + } + + + if (operator != null) { + + condition.setOperator(operator); + + // For logical operators, we must walk all the children and add the same as values to this condition + if (operator.equals(ConditionalOperator.AND) || operator.equals(ConditionalOperator.OR)) { + List children = new ArrayList<>(); + List> conditionList = (List) whereClause.get(CHILDREN); + for (Map unparsedCondition : conditionList) { + Condition childCondition = parseWhereClause(unparsedCondition); + children.add(childCondition); + } + condition.setValue(children); + } else { + // This is a comparison operator. + String key = (String) whereClause.get(KEY); + String value = (String) whereClause.get(VALUE); + condition.setPath(key); + condition.setValue(value); + } + } + + return condition; + } } diff --git a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/Condition.java b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/Condition.java index 2c1478737e..9f48890138 100644 --- a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/Condition.java +++ b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/Condition.java @@ -29,7 +29,7 @@ public class Condition { ConditionalOperator operator; - String value; + Object value; @JsonIgnore DataType valueDataType; @@ -45,19 +45,39 @@ public class Condition { return conditionList .stream() .map(condition -> { - String value = condition.getValue(); - DataType dataType = stringToKnownDataTypeConverter(value); - condition.setValueDataType(dataType); + if (condition.getValue() instanceof String) { + String value = (String) condition.getValue(); + DataType dataType = stringToKnownDataTypeConverter(value); + condition.setValueDataType(dataType); + } return condition; }) .collect(Collectors.toList()); } + public static Condition addValueDataType(Condition condition) { + Object objValue = condition.getValue(); + + if (objValue instanceof String) { + String value = (String) condition.getValue(); + DataType dataType = stringToKnownDataTypeConverter(value); + condition.setValueDataType(dataType); + } else if (objValue instanceof List) { + List conditionList = (List) objValue; + List updatedConditions = conditionList + .stream() + .map(subCondition -> addValueDataType(subCondition)) + .collect(Collectors.toList()); + condition.setValue(updatedConditions); + } + return condition; + } + public static Boolean isValid(Condition condition) { if (StringUtils.isEmpty(condition.getPath()) || (condition.getOperator() == null) || - StringUtils.isEmpty(condition.getValue())) { + StringUtils.isEmpty((CharSequence) condition.getValue())) { return false; } diff --git a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/services/FilterDataService.java b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/services/FilterDataService.java index ffc99f3323..742dfed23e 100644 --- a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/services/FilterDataService.java +++ b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/services/FilterDataService.java @@ -52,7 +52,8 @@ public class FilterDataService { DataType.FLOAT, "REAL", DataType.DOUBLE, "DOUBLE", DataType.BOOLEAN, "BOOLEAN", - DataType.STRING, "VARCHAR" + DataType.STRING, "VARCHAR", + DataType.DATE, "DATE" ); private static final Map SQL_OPERATOR_MAP = Map.of( @@ -108,7 +109,7 @@ public class FilterDataService { insertAllData(tableName, items, schema); // Filter the data - List> finalResults = executeFilterQuery(tableName, conditions, schema); + List> finalResults = executeFilterQueryOldFormat(tableName, conditions, schema); // Now that the data has been filtered. Clean Up. Drop the table dropTable(tableName); @@ -118,14 +119,104 @@ public class FilterDataService { return finalResultsNode; } - public List> executeFilterQuery(String tableName, List conditions, Map schema) { + public ArrayNode filterDataNew(ArrayNode items, Condition condition) { + + if (items == null || items.size() == 0) { + return items; + } + + Map schema = generateSchema(items); + + Condition updatedCondition = addValueDataType(condition); + + String tableName = generateTable(schema); + + // insert the data + insertAllData(tableName, items, schema); + + // Filter the data + List> finalResults = executeFilterQueryNew(tableName, updatedCondition, schema); + + // Now that the data has been filtered. Clean Up. Drop the table + dropTable(tableName); + + ArrayNode finalResultsNode = objectMapper.valueToTree(finalResults); + + return finalResultsNode; + } + + private List> executeFilterQueryNew(String tableName, Condition condition, Map schema) { Connection conn = checkAndGetConnection(); StringBuilder sb = new StringBuilder("SELECT * FROM " + tableName); LinkedHashMap values = new LinkedHashMap<>(); - String whereClause = generateWhereClause(conditions, values, schema); + if (condition != null) { + ConditionalOperator operator = condition.getOperator(); + List conditions = (List) condition.getValue(); + + String whereClause = generateLogicalExpression(conditions, values, schema, operator); + + if (StringUtils.isNotEmpty(whereClause)) { + sb.append(" WHERE "); + sb.append(whereClause); + } + } + + sb.append(";"); + + List> rowsList = new ArrayList<>(50); + + String selectQuery = sb.toString(); + log.debug("{} : Executing Query on H2 : {}", Thread.currentThread().getName(), selectQuery); + + try { + PreparedStatement preparedStatement = conn.prepareStatement(selectQuery); + Set> valueEntries = values.entrySet(); + Iterator> iterator = valueEntries.iterator(); + for (int i = 0; iterator.hasNext(); i++) { + Map.Entry valueEntry = iterator.next(); + String value = valueEntry.getKey(); + DataType dataType = valueEntry.getValue(); + setValueInStatement(preparedStatement, i + 1, value, dataType); + } + + ResultSet resultSet = preparedStatement.executeQuery(); + ResultSetMetaData metaData = resultSet.getMetaData(); + int colCount = metaData.getColumnCount(); + + while (resultSet.next()) { + Map row = new LinkedHashMap<>(colCount); + for (int i = 1; i <= colCount; i++) { + Object resultValue = resultSet.getObject(i); + + // Set null values to empty strings + if (null == resultValue) { + resultValue = ""; + } + + row.put(metaData.getColumnName(i), resultValue); + } + rowsList.add(row); + } + } catch (SQLException e) { + // Getting a SQL Exception here means that our generated query is incorrect. Raise an alarm! + log.error(e.getMessage()); + throw new AppsmithPluginException(AppsmithPluginError.PLUGIN_IN_MEMORY_FILTERING_ERROR, "Filtering failure seen : " + e.getMessage()); + } + + return rowsList; + } + + public List> executeFilterQueryOldFormat(String tableName, List conditions, Map schema) { + Connection conn = checkAndGetConnection(); + + StringBuilder sb = new StringBuilder("SELECT * FROM " + tableName); + + LinkedHashMap values = new LinkedHashMap<>(); + + String whereClause = generateWhereClauseOldFormat(conditions, values, schema); sb.append(whereClause); @@ -174,7 +265,7 @@ public class FilterDataService { return rowsList; } - private String generateWhereClause(List conditions, LinkedHashMap values, Map schema) { + private String generateWhereClauseOldFormat(List conditions, LinkedHashMap values, Map schema) { StringBuilder sb = new StringBuilder(); @@ -192,7 +283,7 @@ public class FilterDataService { String path = condition.getPath(); ConditionalOperator operator = condition.getOperator(); - String value = condition.getValue(); + String value = (String) condition.getValue(); String sqlOp = SQL_OPERATOR_MAP.get(operator); if (sqlOp == null) { @@ -431,7 +522,7 @@ public class FilterDataService { } - private Map generateSchema(ArrayNode items) { + public Map generateSchema(ArrayNode items) { JsonNode item = items.get(0); @@ -513,6 +604,12 @@ public class FilterDataService { // Override datatype to null for empty values if (StringUtils.isEmpty(value)) { dataType = DataType.NULL; + } else { + // value is not empty. + DataType inputDataType = stringToKnownDataTypeConverter(value); + if (DataType.NULL.equals(inputDataType)) { + dataType = DataType.NULL; + } } try { @@ -538,17 +635,23 @@ public class FilterDataService { preparedStatement.setBoolean(index, Boolean.parseBoolean(value)); break; } - case STRING: + case STRING: default: preparedStatement.setString(index, value); break; } - } catch (SQLException | IllegalArgumentException e) { + } catch (SQLException e) { // Alarm! This should never fail since appsmith is the creator of the query and supporter of it. Raise // an alarm and fix quickly! throw new AppsmithPluginException(AppsmithPluginError.PLUGIN_IN_MEMORY_FILTERING_ERROR, "Error while interacting with value " + value + " : " + e.getMessage()); + } catch (IllegalArgumentException e) { + // The data type recognized does not match the data type of the value being set via Prepared Statement + // Add proper handling here. + throw new AppsmithPluginException(AppsmithPluginError.PLUGIN_IN_MEMORY_FILTERING_ERROR, + "Error while interacting with value " + value + " : " + e.getMessage() + + ". The data type value was being parsed to was : " + dataType); } return preparedStatement; @@ -582,5 +685,82 @@ public class FilterDataService { return true; } + public String generateLogicalExpression(List conditions, LinkedHashMap values, Map schema, ConditionalOperator logicOp) { + + StringBuilder sb = new StringBuilder(); + + Boolean firstCondition = true; + for (Condition condition : conditions) { + String path = condition.getPath(); + ConditionalOperator operator = condition.getOperator(); + Object objValue = condition.getValue(); + if (operator.equals(ConditionalOperator.AND) || operator.equals(ConditionalOperator.OR)) { + List subConditions = (List) objValue; + String logicalExpression = generateLogicalExpression(subConditions, values, schema, operator); + if (StringUtils.isNotEmpty(logicalExpression)) { + sb.append(" " + logicOp + " ( "); + sb.append(logicalExpression); + sb.append(" ) "); + } + } else { + String value = (String) objValue; + + if (StringUtils.isNotEmpty(path) && StringUtils.isNotEmpty(value)) { + if (firstCondition) { + firstCondition = false; + } else { + // This is not the first valid condition. Append the operator before adding the next condition + sb.append(" " + logicOp); + } + String sqlOp = SQL_OPERATOR_MAP.get(operator); + if (sqlOp == null) { + throw new AppsmithPluginException(AppsmithPluginError.PLUGIN_EXECUTE_ARGUMENT_ERROR, + operator + " is not supported currently for filtering."); + } + sb.append(" ( "); + sb.append("\"" + path + "\""); + sb.append(" "); + sb.append(sqlOp); + sb.append(" "); + + // These are array operations. Convert value into appropriate format and then append + if (operator == ConditionalOperator.IN || operator == ConditionalOperator.NOT_IN) { + + StringBuilder valueBuilder = new StringBuilder("("); + + try { + List arrayValues = objectMapper.readValue(value, List.class); + List updatedStringValues = arrayValues + .stream() + .map(fieldValue -> { + values.put(String.valueOf(fieldValue), schema.get(path)); + return "?"; + }) + .collect(Collectors.toList()); + String finalValues = String.join(",", updatedStringValues); + valueBuilder.append(finalValues); + } catch (IOException e) { + throw new AppsmithPluginException(AppsmithPluginError.PLUGIN_EXECUTE_ARGUMENT_ERROR, + value + " could not be parsed into an array"); + } + + valueBuilder.append(")"); + value = valueBuilder.toString(); + sb.append(value); + + } else { + // Not an array. Simply add a placeholder + sb.append("?"); + values.put(value, schema.get(path)); + } + + sb.append(" ) "); + } + } + + } + return sb.toString(); + } + } diff --git a/app/server/appsmith-interfaces/src/test/java/com/appsmith/external/helpers/PluginUtilsTest.java b/app/server/appsmith-interfaces/src/test/java/com/appsmith/external/helpers/PluginUtilsTest.java new file mode 100644 index 0000000000..ce7084c3ef --- /dev/null +++ b/app/server/appsmith-interfaces/src/test/java/com/appsmith/external/helpers/PluginUtilsTest.java @@ -0,0 +1,119 @@ +package com.appsmith.external.helpers; + +import com.appsmith.external.constants.ConditionalOperator; +import com.appsmith.external.models.Condition; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.Test; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static com.appsmith.external.helpers.PluginUtils.parseWhereClause; +import static org.assertj.core.api.Assertions.assertThat; + +public class PluginUtilsTest { + + private final ObjectMapper objectMapper = new ObjectMapper(); + + @Test + public void parseWhereClauseTest() { + String whereJson = "{\n" + + " \"where\": {\n" + + " \"children\": [\n" + + " {\n" + + " \"key\": \"i\",\n" + + " \"condition\": \"GTE\",\n" + + " \"value\": \"u\"\n" + + " },\n" + + " {\n" + + " \"condition\": \"AND\",\n" + + " \"children\": [\n" + + " {\n" + + " \"key\": \"d\",\n" + + " \"condition\": \"LTE\",\n" + + " \"value\": \"w\"\n" + + " },\n" + + " {\n" + + " \"condition\": \"AND\",\n" + + " \"children\": [\n" + + " {\n" + + " \"key\": \"a\",\n" + + " \"condition\": \"LTE\",\n" + + " \"value\": \"s\"\n" + + " }\n" + + " ]\n" + + " }\n" + + " ]\n" + + " },\n" + + " {\n" + + " \"condition\": \"AND\",\n" + + " \"children\": [\n" + + " {\n" + + " \"key\": \"u\",\n" + + " \"condition\": \"LTE\",\n" + + " \"value\": \"me\"\n" + + " }\n" + + " ]\n" + + " }\n" + + " ],\n" + + " \"condition\": \"AND\"\n" + + " }\n" + + "}"; + try { + Map whereClause = objectMapper.readValue(whereJson, HashMap.class); + Map unparsedWhereClause = (Map) whereClause.get("where"); + Condition condition = parseWhereClause(unparsedWhereClause); + + assertThat(condition.getOperator().equals(ConditionalOperator.AND)); + Object conditionValue = condition.getValue(); + assertThat(conditionValue).isNotNull(); + assertThat(conditionValue instanceof List); + List conditionList = (List) conditionValue; + assertThat(conditionList.size()).isEqualTo(3); + for (Condition conditionFromChildren : conditionList) { + ConditionalOperator operator = conditionFromChildren.getOperator(); + assertThat(operator).isNotNull(); + + String path = conditionFromChildren.getPath(); + Object value = conditionFromChildren.getValue(); + if (operator.equals(ConditionalOperator.AND)) { + assertThat(path).isNull(); + assertThat(value instanceof List); + } else { + assertThat(path).isNotNull(); + assertThat(value instanceof String); + } + } + + } catch (IOException e) { + e.printStackTrace(); + } + } + + @Test + public void parseWhereClauseEmptyChildrenArrayTest() { + String whereJson = "{\n" + + " \"where\": {\n" + + " \"children\": [],\n" + + " \"condition\": \"AND\"\n" + + " }\n" + + "}"; + try { + Map whereClause = objectMapper.readValue(whereJson, HashMap.class); + Map unparsedWhereClause = (Map) whereClause.get("where"); + Condition condition = parseWhereClause(unparsedWhereClause); + + assertThat(condition.getOperator().equals(ConditionalOperator.AND)); + Object conditionValue = condition.getValue(); + assertThat(conditionValue).isNotNull(); + assertThat(conditionValue instanceof List); + List conditionList = (List) conditionValue; + assertThat(conditionList.size()).isEqualTo(0); + + } catch (IOException e) { + e.printStackTrace(); + } + } +} diff --git a/app/server/appsmith-interfaces/src/test/java/com/appsmith/external/services/FilterDataServiceTest.java b/app/server/appsmith-interfaces/src/test/java/com/appsmith/external/services/FilterDataServiceTest.java index 2d7b433c97..a2dbdbd946 100644 --- a/app/server/appsmith-interfaces/src/test/java/com/appsmith/external/services/FilterDataServiceTest.java +++ b/app/server/appsmith-interfaces/src/test/java/com/appsmith/external/services/FilterDataServiceTest.java @@ -1,6 +1,8 @@ package com.appsmith.external.services; +import com.appsmith.external.constants.ConditionalOperator; import com.appsmith.external.constants.DataType; +import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginException; import com.appsmith.external.models.Condition; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; @@ -8,14 +10,18 @@ import org.junit.Test; import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; import java.util.stream.Stream; +import static com.appsmith.external.helpers.PluginUtils.parseWhereClause; import static org.assertj.core.api.AssertionsForClassTypes.assertThat; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; public class FilterDataServiceTest { @@ -431,4 +437,823 @@ public class FilterDataServiceTest { e.printStackTrace(); } } + + @Test + public void generateLogicalOperatorTest() { + + String data = "[\n" + + " {\n" + + " \"id\": 2381224,\n" + + " \"email id\": \"michael.lawson@reqres.in\",\n" + + " \"userName\": \"Michael Lawson\",\n" + + " \"productName\": \"Chicken Sandwich\",\n" + + " \"orderAmount\": 4.99,\n" + + " \"date\": \"2021-09-01\"\n" + + " },\n" + + " {\n" + + " \"id\": \"\",\n" + + " \"email id\": \"\",\n" + + " \"userName\": \"Lindsay Ferguson\",\n" + + " \"productName\": \"Tuna Salad\",\n" + + " \"orderAmount\": 9.99,\n" + + " \"date\": \"2021-09-01\"\n" + + " },\n" + + " {\n" + + " \"id\": \"\",\n" + + " \"email id\": \"\",\n" + + " \"userName\": \"Tobias Funke\",\n" + + " \"productName\": \"Beef steak\",\n" + + " \"orderAmount\": 19.99,\n" + + " \"date\": \"2021-09-01\"\n" + + " }\n" + + "]"; + + String whereJson = "{\n" + + " \"where\": {\n" + + " \"children\": [\n" + + " {\n" + + " \"key\": \"i\",\n" + + " \"condition\": \"GTE\",\n" + + " \"value\": \"u\"\n" + + " },\n" + + " {\n" + + " \"condition\": \"AND\",\n" + + " \"children\": [\n" + + " {\n" + + " \"key\": \"d\",\n" + + " \"condition\": \"LTE\",\n" + + " \"value\": \"w\"\n" + + " },\n" + + " {\n" + + " \"condition\": \"AND\",\n" + + " \"children\": [\n" + + " {\n" + + " \"key\": \"a\",\n" + + " \"condition\": \"LTE\",\n" + + " \"value\": \"s\"\n" + + " }\n" + + " ]\n" + + " }\n" + + " ]\n" + + " },\n" + + " {\n" + + " \"condition\": \"AND\",\n" + + " \"children\": [\n" + + " {\n" + + " \"key\": \"u\",\n" + + " \"condition\": \"LTE\",\n" + + " \"value\": \"me\"\n" + + " }\n" + + " ]\n" + + " }\n" + + " ],\n" + + " \"condition\": \"AND\"\n" + + " }\n" + + "}"; + + try { + + ArrayNode items = (ArrayNode) objectMapper.readTree(data); + Map schema = filterDataService.generateSchema(items); + + Map whereClause = objectMapper.readValue(whereJson, HashMap.class); + Map unparsedWhereClause = (Map) whereClause.get("where"); + Condition condition = parseWhereClause(unparsedWhereClause); + + ConditionalOperator operator = condition.getOperator(); + List conditions = (List) condition.getValue(); + + String expression = filterDataService.generateLogicalExpression(conditions, new LinkedHashMap<>(), schema, operator); + assertThat(expression.equals("( \"i\" >= ? ) and ( ( \"d\" <= ? ) and ( ( \"a\" <= ? ) ) ) and ( ( \"u\" <= ? ) ) ")); + + } catch (IOException e) { + e.printStackTrace(); + } + } + + @Test + public void testFilterSingleConditionWithWhereJson() { + String data = "[\n" + + " {\n" + + " \"id\": 2381224,\n" + + " \"email\": \"michael.lawson@reqres.in\",\n" + + " \"userName\": \"Michael Lawson\",\n" + + " \"productName\": \"Chicken Sandwich\",\n" + + " \"orderAmount\": 4.99,\n" + + " \"orderStatus\": \"READY\"\n" + + " },\n" + + " {\n" + + " \"id\": 2736212,\n" + + " \"email\": \"lindsay.ferguson@reqres.in\",\n" + + " \"userName\": \"Lindsay Ferguson\",\n" + + " \"productName\": \"Tuna Salad\",\n" + + " \"orderAmount\": 9.99,\n" + + " \"orderStatus\": \"READY\"\n" + + " },\n" + + " {\n" + + " \"id\": 6788734,\n" + + " \"email\": \"tobias.funke@reqres.in\",\n" + + " \"userName\": \"Tobias Funke\",\n" + + " \"productName\": \"Beef steak\",\n" + + " \"orderAmount\": 19.99,\n" + + " \"orderStatus\": \"READY\"\n" + + " }\n" + + "]"; + + String whereJson = "{\n" + + " \"where\": {\n" + + " \"children\": [\n" + + " {\n" + + " \"key\": \"orderAmount\",\n" + + " \"condition\": \"LT\",\n" + + " \"value\": \"15\"\n" + + " }\n" + + " ],\n" + + " \"condition\": \"AND\"\n" + + " }\n" + + "}"; + + try { + ArrayNode items = (ArrayNode) objectMapper.readTree(data); + + Map whereClause = objectMapper.readValue(whereJson, HashMap.class); + Map unparsedWhereClause = (Map) whereClause.get("where"); + Condition condition = parseWhereClause(unparsedWhereClause); + + ArrayNode filteredData = filterDataService.filterDataNew(items, condition); + + assertEquals(filteredData.size(), 2); + + } catch (IOException e) { + e.printStackTrace(); + } + } + + @Test + public void testFilterMultipleConditionsNew() { + String data = "[\n" + + " {\n" + + " \"id\": 2381224,\n" + + " \"email\": \"michael.lawson@reqres.in\",\n" + + " \"userName\": \"Michael Lawson\",\n" + + " \"productName\": \"Chicken Sandwich\",\n" + + " \"orderAmount\": 4.99,\n" + + " \"orderStatus\": \"READY\"\n" + + " },\n" + + " {\n" + + " \"id\": 2736212,\n" + + " \"email\": \"lindsay.ferguson@reqres.in\",\n" + + " \"userName\": \"Lindsay Ferguson\",\n" + + " \"productName\": \"Tuna Salad\",\n" + + " \"orderAmount\": 9.99,\n" + + " \"orderStatus\": \"NOT READY\"\n" + + " },\n" + + " {\n" + + " \"id\": 6788734,\n" + + " \"email\": \"tobias.funke@reqres.in\",\n" + + " \"userName\": \"Tobias Funke\",\n" + + " \"productName\": \"Beef steak\",\n" + + " \"orderAmount\": 19.99,\n" + + " \"orderStatus\": \"READY\"\n" + + " }\n" + + "]"; + + String whereJson = "{\n" + + " \"where\": {\n" + + " \"children\": [\n" + + " {\n" + + " \"key\": \"orderAmount\",\n" + + " \"condition\": \"LT\",\n" + + " \"value\": \"15\"\n" + + " },\n" + + " {\n" + + " \"key\": \"orderStatus\",\n" + + " \"condition\": \"EQ\",\n" + + " \"value\": \"READY\"\n" + + " }\n" + + " ],\n" + + " \"condition\": \"AND\"\n" + + " }\n" + + "}"; + + try { + ArrayNode items = (ArrayNode) objectMapper.readTree(data); + + Map whereClause = objectMapper.readValue(whereJson, HashMap.class); + Map unparsedWhereClause = (Map) whereClause.get("where"); + Condition condition = parseWhereClause(unparsedWhereClause); + + ArrayNode filteredData = filterDataService.filterDataNew(items, condition); + + assertEquals(filteredData.size(), 1); + + + } catch (IOException e) { + e.printStackTrace(); + } + } + + @Test + public void testFilterInConditionForStringsNew() { + String data = "[\n" + + " {\n" + + " \"id\": 2381224,\n" + + " \"email\": \"michael.lawson@reqres.in\",\n" + + " \"userName\": \"Michael Lawson\",\n" + + " \"productName\": \"Chicken Sandwich\",\n" + + " \"orderAmount\": 4.99,\n" + + " \"orderStatus\": \"READY\"\n" + + " },\n" + + " {\n" + + " \"id\": 2736212,\n" + + " \"email\": \"lindsay.ferguson@reqres.in\",\n" + + " \"userName\": \"Lindsay Ferguson\",\n" + + " \"productName\": \"Tuna Salad\",\n" + + " \"orderAmount\": 9.99,\n" + + " \"orderStatus\": \"NOT READY\"\n" + + " },\n" + + " {\n" + + " \"id\": 6788734,\n" + + " \"email\": \"tobias.funke@reqres.in\",\n" + + " \"userName\": \"Tobias Funke\",\n" + + " \"productName\": \"Beef steak\",\n" + + " \"orderAmount\": 19.99,\n" + + " \"orderStatus\": \"READY\"\n" + + " }\n" + + "]"; + + String whereJson = "{\n" + + " \"where\": {\n" + + " \"children\": [\n" + + " {\n" + + " \"key\": \"orderAmount\",\n" + + " \"condition\": \"LT\",\n" + + " \"value\": \"15\"\n" + + " },\n" + + " {\n" + + " \"key\": \"orderStatus\",\n" + + " \"condition\": \"IN\",\n" + + " \"value\": \"[\\\"READY\\\", \\\"NOT READY\\\"]\"\n" + + " }\n" + + " ],\n" + + " \"condition\": \"AND\"\n" + + " }\n" + + "}"; + + try { + ArrayNode items = (ArrayNode) objectMapper.readTree(data); + + Map whereClause = objectMapper.readValue(whereJson, HashMap.class); + Map unparsedWhereClause = (Map) whereClause.get("where"); + Condition condition = parseWhereClause(unparsedWhereClause); + + ArrayNode filteredData = filterDataService.filterDataNew(items, condition); + + assertEquals(filteredData.size(), 2); + + + } catch (IOException e) { + e.printStackTrace(); + } + } + + @Test + public void testFilterInConditionForNumbersNew() { + String data = "[\n" + + " {\n" + + " \"id\": 2381224,\n" + + " \"email\": \"michael.lawson@reqres.in\",\n" + + " \"userName\": \"Michael Lawson\",\n" + + " \"productName\": \"Chicken Sandwich\",\n" + + " \"orderAmount\": 4.99,\n" + + " \"orderStatus\": \"READY\"\n" + + " },\n" + + " {\n" + + " \"id\": 2736212,\n" + + " \"email\": \"lindsay.ferguson@reqres.in\",\n" + + " \"userName\": \"Lindsay Ferguson\",\n" + + " \"productName\": \"Tuna Salad\",\n" + + " \"orderAmount\": 9.99,\n" + + " \"orderStatus\": \"NOT READY\"\n" + + " },\n" + + " {\n" + + " \"id\": 6788734,\n" + + " \"email\": \"tobias.funke@reqres.in\",\n" + + " \"userName\": \"Tobias Funke\",\n" + + " \"productName\": \"Beef steak\",\n" + + " \"orderAmount\": 19.99,\n" + + " \"orderStatus\": \"READY\"\n" + + " }\n" + + "]"; + + String whereJson = "{\n" + + " \"where\": {\n" + + " \"children\": [\n" + + " {\n" + + " \"key\": \"orderAmount\",\n" + + " \"condition\": \"LT\",\n" + + " \"value\": \"15\"\n" + + " },\n" + + " {\n" + + " \"key\": \"orderAmount\",\n" + + " \"condition\": \"IN\",\n" + + " \"value\": \"[4.99, 19.99]\"\n" + + " }\n" + + " ],\n" + + " \"condition\": \"AND\"\n" + + " }\n" + + "}"; + + try { + ArrayNode items = (ArrayNode) objectMapper.readTree(data); + + Map whereClause = objectMapper.readValue(whereJson, HashMap.class); + Map unparsedWhereClause = (Map) whereClause.get("where"); + Condition condition = parseWhereClause(unparsedWhereClause); + + ArrayNode filteredData = filterDataService.filterDataNew(items, condition); + + assertEquals(filteredData.size(), 1); + + } catch (IOException e) { + e.printStackTrace(); + } + } + + @Test + public void testFilterNotInConditionForNumbersNew() { + String data = "[\n" + + " {\n" + + " \"id\": 2381224,\n" + + " \"email\": \"michael.lawson@reqres.in\",\n" + + " \"userName\": \"Michael Lawson\",\n" + + " \"productName\": \"Chicken Sandwich\",\n" + + " \"orderAmount\": 4.99,\n" + + " \"orderStatus\": \"READY\"\n" + + " },\n" + + " {\n" + + " \"id\": 2736212,\n" + + " \"email\": \"lindsay.ferguson@reqres.in\",\n" + + " \"userName\": \"Lindsay Ferguson\",\n" + + " \"productName\": \"Tuna Salad\",\n" + + " \"orderAmount\": 9.99,\n" + + " \"orderStatus\": \"NOT READY\"\n" + + " },\n" + + " {\n" + + " \"id\": 6788734,\n" + + " \"email\": \"tobias.funke@reqres.in\",\n" + + " \"userName\": \"Tobias Funke\",\n" + + " \"productName\": \"Beef steak\",\n" + + " \"orderAmount\": 19.99,\n" + + " \"orderStatus\": \"READY\"\n" + + " }\n" + + "]"; + + String whereJson = "{\n" + + " \"where\": {\n" + + " \"children\": [\n" + + " {\n" + + " \"key\": \"orderAmount\",\n" + + " \"condition\": \"LT\",\n" + + " \"value\": \"15\"\n" + + " },\n" + + " {\n" + + " \"key\": \"orderAmount\",\n" + + " \"condition\": \"NOT_IN\",\n" + + " \"value\": \"[5.99, 19.00]\"\n" + + " }\n" + + " ],\n" + + " \"condition\": \"AND\"\n" + + " }\n" + + "}"; + + try { + ArrayNode items = (ArrayNode) objectMapper.readTree(data); + + Map whereClause = objectMapper.readValue(whereJson, HashMap.class); + Map unparsedWhereClause = (Map) whereClause.get("where"); + Condition condition = parseWhereClause(unparsedWhereClause); + + ArrayNode filteredData = filterDataService.filterDataNew(items, condition); + + assertEquals(filteredData.size(), 2); + + + } catch (IOException e) { + e.printStackTrace(); + } + } + + @Test + public void testMultiWordColumnNamesNew() { + String data = "[\n" + + " {\n" + + " \"id\": 2381224,\n" + + " \"email id\": \"michael.lawson@reqres.in\",\n" + + " \"userName\": \"Michael Lawson\",\n" + + " \"productName\": \"Chicken Sandwich\",\n" + + " \"orderAmount\": 4.99,\n" + + " \"orderStatus\": \"READY\"\n" + + " },\n" + + " {\n" + + " \"id\": 2736212,\n" + + " \"email id\": \"lindsay.ferguson@reqres.in\",\n" + + " \"userName\": \"Lindsay Ferguson\",\n" + + " \"productName\": \"Tuna Salad\",\n" + + " \"orderAmount\": 9.99,\n" + + " \"orderStatus\": \"READY\"\n" + + " },\n" + + " {\n" + + " \"id\": 6788734,\n" + + " \"email id\": \"tobias.funke@reqres.in\",\n" + + " \"userName\": \"Tobias Funke\",\n" + + " \"productName\": \"Beef steak\",\n" + + " \"orderAmount\": 19.99,\n" + + " \"orderStatus\": \"READY\"\n" + + " }\n" + + "]"; + + String whereJson = "{\n" + + " \"where\": {\n" + + " \"children\": [\n" + + " {\n" + + " \"key\": \"orderAmount\",\n" + + " \"condition\": \"LT\",\n" + + " \"value\": \"15\"\n" + + " }\n" + + " ],\n" + + " \"condition\": \"AND\"\n" + + " }\n" + + "}"; + + try { + ArrayNode items = (ArrayNode) objectMapper.readTree(data); + + Map whereClause = objectMapper.readValue(whereJson, HashMap.class); + Map unparsedWhereClause = (Map) whereClause.get("where"); + Condition condition = parseWhereClause(unparsedWhereClause); + + ArrayNode filteredData = filterDataService.filterDataNew(items, condition); + + assertEquals(filteredData.size(), 2); + + Iterator fieldNamesIterator = filteredData.get(0).fieldNames(); + + List columnNames = Stream.generate(() -> null) + .takeWhile(x -> fieldNamesIterator.hasNext()) + .map(n -> fieldNamesIterator.next()) + .collect(Collectors.toList()); + + assertThat(columnNames.containsAll(List.of("id", "email id", "userName", "productName", "orderAmount", "orderStatus"))); + + + } catch (IOException e) { + e.printStackTrace(); + } + } + + @Test + public void testEmptyValuesInSomeColumnsNew() { + String data = "[\n" + + " {\n" + + " \"id\": 2381224,\n" + + " \"email id\": \"michael.lawson@reqres.in\",\n" + + " \"userName\": \"Michael Lawson\",\n" + + " \"productName\": \"Chicken Sandwich\",\n" + + " \"orderAmount\": 4.99,\n" + + " \"orderStatus\": \"READY\"\n" + + " },\n" + + " {\n" + + " \"id\": \"\",\n" + + " \"email id\": \"\",\n" + + " \"userName\": \"Lindsay Ferguson\",\n" + + " \"productName\": \"Tuna Salad\",\n" + + " \"orderAmount\": 9.99,\n" + + " \"orderStatus\": \"READY\"\n" + + " },\n" + + " {\n" + + " \"id\": \"\",\n" + + " \"email id\": \"\",\n" + + " \"userName\": \"Tobias Funke\",\n" + + " \"productName\": \"Beef steak\",\n" + + " \"orderAmount\": 19.99,\n" + + " \"orderStatus\": \"READY\"\n" + + " }\n" + + "]"; + + String whereJson = "{\n" + + " \"where\": {\n" + + " \"children\": [\n" + + " {\n" + + " \"key\": \"orderAmount\",\n" + + " \"condition\": \"LT\",\n" + + " \"value\": \"15\"\n" + + " }\n" + + " ],\n" + + " \"condition\": \"AND\"\n" + + " }\n" + + "}"; + + try { + ArrayNode items = (ArrayNode) objectMapper.readTree(data); + + Map whereClause = objectMapper.readValue(whereJson, HashMap.class); + Map unparsedWhereClause = (Map) whereClause.get("where"); + Condition condition = parseWhereClause(unparsedWhereClause); + + ArrayNode filteredData = filterDataService.filterDataNew(items, condition); + + assertEquals(filteredData.size(), 2); + + + } catch (IOException e) { + e.printStackTrace(); + } + } + + @Test + public void testValuesOfUnsupportedDataTypeNew() { + String data = "[\n" + + " {\n" + + " \"id\": 2381224,\n" + + " \"email id\": \"michael.lawson@reqres.in\",\n" + + " \"userName\": \"Michael Lawson\",\n" + + " \"productName\": \"Chicken Sandwich\",\n" + + " \"orderAmount\": 4.99,\n" + + " \"date\": \"2021-09-01\"\n" + + " },\n" + + " {\n" + + " \"id\": \"\",\n" + + " \"email id\": \"\",\n" + + " \"userName\": \"Lindsay Ferguson\",\n" + + " \"productName\": \"Tuna Salad\",\n" + + " \"orderAmount\": 9.99,\n" + + " \"date\": \"2021-09-01\"\n" + + " },\n" + + " {\n" + + " \"id\": \"\",\n" + + " \"email id\": \"\",\n" + + " \"userName\": \"Tobias Funke\",\n" + + " \"productName\": \"Beef steak\",\n" + + " \"orderAmount\": 19.99,\n" + + " \"date\": \"2021-09-01\"\n" + + " }\n" + + "]"; + + String whereJson = "{\n" + + " \"where\": {\n" + + " \"children\": [\n" + + " {\n" + + " \"key\": \"orderAmount\",\n" + + " \"condition\": \"LT\",\n" + + " \"value\": \"15\"\n" + + " }\n" + + " ],\n" + + " \"condition\": \"AND\"\n" + + " }\n" + + "}"; + + try { + ArrayNode items = (ArrayNode) objectMapper.readTree(data); + + Map whereClause = objectMapper.readValue(whereJson, HashMap.class); + Map unparsedWhereClause = (Map) whereClause.get("where"); + Condition condition = parseWhereClause(unparsedWhereClause); + + ArrayNode filteredData = filterDataService.filterDataNew(items, condition); + + assertEquals(filteredData.size(), 2); + + + } catch (IOException e) { + e.printStackTrace(); + } + } + + @Test + public void testConditionTypeMismatch() { + String data = "[\n" + + " {\n" + + " \"id\": 2381224,\n" + + " \"email id\": \"michael.lawson@reqres.in\",\n" + + " \"userName\": \"Michael Lawson\",\n" + + " \"productName\": \"Chicken Sandwich\",\n" + + " \"orderAmount\": 4.99,\n" + + " \"date\": \"2021-09-01\"\n" + + " },\n" + + " {\n" + + " \"id\": \"\",\n" + + " \"email id\": \"\",\n" + + " \"userName\": \"Lindsay Ferguson\",\n" + + " \"productName\": \"Tuna Salad\",\n" + + " \"orderAmount\": 9.99,\n" + + " \"date\": \"2021-09-01\"\n" + + " },\n" + + " {\n" + + " \"id\": \"\",\n" + + " \"email id\": \"\",\n" + + " \"userName\": \"Tobias Funke\",\n" + + " \"productName\": \"Beef steak\",\n" + + " \"orderAmount\": 19.99,\n" + + " \"date\": \"2021-09-01\"\n" + + " }\n" + + "]"; + + String whereJson = "{\n" + + " \"where\": {\n" + + " \"children\": [\n" + + " {\n" + + " \"key\": \"orderAmount\",\n" + + " \"condition\": \"LT\",\n" + + " \"value\": \"String here where number is expected\"\n" + + " }\n" + + " ],\n" + + " \"condition\": \"AND\"\n" + + " }\n" + + "}"; + + try { + ArrayNode items = (ArrayNode) objectMapper.readTree(data); + + Map whereClause = objectMapper.readValue(whereJson, HashMap.class); + Map unparsedWhereClause = (Map) whereClause.get("where"); + Condition condition = parseWhereClause(unparsedWhereClause); + + // Since the data type expected for orderAmount is float, but the value given is String, assert exception + assertThrows(AppsmithPluginException.class, + () -> filterDataService.filterDataNew(items, condition)); + + } catch (IOException e) { + e.printStackTrace(); + } + } + + @Test + public void testEmptyConditions() { + String data = "[\n" + + " {\n" + + " \"id\": 2381224,\n" + + " \"email id\": \"michael.lawson@reqres.in\",\n" + + " \"userName\": \"Michael Lawson\",\n" + + " \"productName\": \"Chicken Sandwich\",\n" + + " \"orderAmount\": 4.99,\n" + + " \"date\": \"2021-09-01\"\n" + + " },\n" + + " {\n" + + " \"id\": \"\",\n" + + " \"email id\": \"\",\n" + + " \"userName\": \"Lindsay Ferguson\",\n" + + " \"productName\": \"Tuna Salad\",\n" + + " \"orderAmount\": 9.99,\n" + + " \"date\": \"2021-09-01\"\n" + + " },\n" + + " {\n" + + " \"id\": \"\",\n" + + " \"email id\": \"\",\n" + + " \"userName\": \"Tobias Funke\",\n" + + " \"productName\": \"Beef steak\",\n" + + " \"orderAmount\": 19.99,\n" + + " \"date\": \"2021-09-01\"\n" + + " }\n" + + "]"; + + String whereJson = "{\n" + + " \"where\": {\n" + + " \"children\": [],\n" + + " \"condition\": \"AND\"\n" + + " }\n" + + "}"; + + try { + ArrayNode items = (ArrayNode) objectMapper.readTree(data); + + Map whereClause = objectMapper.readValue(whereJson, HashMap.class); + Map unparsedWhereClause = (Map) whereClause.get("where"); + Condition condition = parseWhereClause(unparsedWhereClause); + + ArrayNode filteredData = filterDataService.filterDataNew(items, condition); + + assertEquals(filteredData.size(), 3); + + } catch (IOException e) { + e.printStackTrace(); + } + } + + @Test + public void testConditionNullValueMatch() { + String data = "[\n" + + " {\n" + + " \"id\": 2381224,\n" + + " \"email id\": \"michael.lawson@reqres.in\",\n" + + " \"userName\": \"Michael Lawson\",\n" + + " \"productName\": \"Chicken Sandwich\",\n" + + " \"orderAmount\": 4.99,\n" + + " \"date\": \"2021-09-01\"\n" + + " },\n" + + " {\n" + + " \"id\": \"\",\n" + + " \"email id\": \"\",\n" + + " \"userName\": \"Lindsay Ferguson\",\n" + + " \"productName\": \"Tuna Salad\",\n" + + " \"orderAmount\": 9.99,\n" + + " \"date\": \"2021-09-01\"\n" + + " },\n" + + " {\n" + + " \"id\": \"\",\n" + + " \"email id\": \"\",\n" + + " \"userName\": \"Tobias Funke\",\n" + + " \"productName\": \"Beef steak\",\n" + + " \"orderAmount\": 19.99,\n" + + " \"date\": \"2021-09-01\"\n" + + " }\n" + + "]"; + + String whereJson = "{\n" + + " \"where\": {\n" + + " \"children\": [\n" + + " {\n" + + " \"key\": \"orderAmount\",\n" + + " \"condition\": \"EQ\",\n" + + " \"value\": \"null\"\n" + + " }\n" + + " ],\n" + + " \"condition\": \"AND\"\n" + + " }\n" + + "}"; + + try { + ArrayNode items = (ArrayNode) objectMapper.readTree(data); + + Map whereClause = objectMapper.readValue(whereJson, HashMap.class); + Map unparsedWhereClause = (Map) whereClause.get("where"); + Condition condition = parseWhereClause(unparsedWhereClause); + + ArrayNode filteredData = filterDataService.filterDataNew(items, condition); + + // Since there are no null orderAmounts, the filtered data would be empty. + assertEquals(filteredData.size(), 0); + + } catch (IOException e) { + e.printStackTrace(); + } + } + + @Test + public void testDateCondition() { + String data = "[\n" + + " {\n" + + " \"id\": 2381224,\n" + + " \"email id\": \"michael.lawson@reqres.in\",\n" + + " \"userName\": \"Michael Lawson\",\n" + + " \"productName\": \"Chicken Sandwich\",\n" + + " \"orderAmount\": 4.99,\n" + + " \"date\": \"2021-09-01\"\n" + + " },\n" + + " {\n" + + " \"id\": \"\",\n" + + " \"email id\": \"\",\n" + + " \"userName\": \"Lindsay Ferguson\",\n" + + " \"productName\": \"Tuna Salad\",\n" + + " \"orderAmount\": 9.99,\n" + + " \"date\": \"2021-09-02\"\n" + + " },\n" + + " {\n" + + " \"id\": \"\",\n" + + " \"email id\": \"\",\n" + + " \"userName\": \"Tobias Funke\",\n" + + " \"productName\": \"Beef steak\",\n" + + " \"orderAmount\": 19.99,\n" + + " \"date\": \"2021-09-03\"\n" + + " }\n" + + "]"; + + String whereJson = "{\n" + + " \"where\": {\n" + + " \"children\": [\n" + + " {\n" + + " \"key\": \"date\",\n" + + " \"condition\": \"GTE\",\n" + + " \"value\": \"2021-09-02\"\n" + + " }\n" + + " ],\n" + + " \"condition\": \"AND\"\n" + + " }\n" + + "}"; + + try { + ArrayNode items = (ArrayNode) objectMapper.readTree(data); + + Map whereClause = objectMapper.readValue(whereJson, HashMap.class); + Map unparsedWhereClause = (Map) whereClause.get("where"); + Condition condition = parseWhereClause(unparsedWhereClause); + + ArrayNode filteredData = filterDataService.filterDataNew(items, condition); + + assertEquals(filteredData.size(), 2); + + } catch (IOException e) { + e.printStackTrace(); + } + } } diff --git a/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/plugins/AmazonS3Plugin.java b/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/plugins/AmazonS3Plugin.java index 3194592412..803d0581a4 100644 --- a/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/plugins/AmazonS3Plugin.java +++ b/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/plugins/AmazonS3Plugin.java @@ -23,6 +23,7 @@ import com.appsmith.external.helpers.MustacheHelper; import com.appsmith.external.models.ActionConfiguration; import com.appsmith.external.models.ActionExecutionRequest; import com.appsmith.external.models.ActionExecutionResult; +import com.appsmith.external.models.Condition; import com.appsmith.external.models.DBAuth; import com.appsmith.external.models.DatasourceConfiguration; import com.appsmith.external.models.DatasourceStructure; @@ -32,7 +33,9 @@ import com.appsmith.external.models.RequestParamDTO; import com.appsmith.external.plugins.BasePlugin; import com.appsmith.external.plugins.PluginExecutor; import com.appsmith.external.plugins.SmartSubstitutionInterface; +import com.appsmith.external.services.FilterDataService; import com.external.plugins.constants.AmazonS3Action; +import com.fasterxml.jackson.databind.node.ArrayNode; import lombok.extern.slf4j.Slf4j; import org.pf4j.Extension; import org.pf4j.PluginWrapper; @@ -64,6 +67,7 @@ import static com.appsmith.external.constants.ActionConstants.ACTION_CONFIGURATI import static com.appsmith.external.constants.ActionConstants.ACTION_CONFIGURATION_PATH; import static com.appsmith.external.helpers.PluginUtils.getValueSafelyFromFormData; import static com.appsmith.external.helpers.PluginUtils.getValueSafelyFromFormDataOrDefault; +import static com.appsmith.external.helpers.PluginUtils.parseWhereClause; import static com.appsmith.external.helpers.PluginUtils.setValueSafelyInFormData; import static com.external.plugins.constants.FieldName.BUCKET; import static com.external.plugins.constants.FieldName.COMMAND; @@ -73,6 +77,7 @@ import static com.external.plugins.constants.FieldName.LIST_EXPIRY; import static com.external.plugins.constants.FieldName.LIST_PREFIX; import static com.external.plugins.constants.FieldName.LIST_SIGNED_URL; import static com.external.plugins.constants.FieldName.LIST_UNSIGNED_URL; +import static com.external.plugins.constants.FieldName.LIST_WHERE; import static com.external.plugins.constants.FieldName.PATH; import static com.external.plugins.constants.FieldName.READ_USING_BASE64_ENCODING; import static com.external.utils.DatasourceUtils.getS3ClientBuilder; @@ -100,6 +105,11 @@ public class AmazonS3Plugin extends BasePlugin { @Extension public static class S3PluginExecutor implements PluginExecutor, SmartSubstitutionInterface { private final Scheduler scheduler = Schedulers.elastic(); + private final FilterDataService filterDataService; + + public S3PluginExecutor() { + this.filterDataService = FilterDataService.getInstance(); + } /* * - Exception thrown by this method is expected to be handled by the caller. @@ -535,6 +545,17 @@ public class AmazonS3Plugin extends BasePlugin { null, null)); } + // Check if where condition is configured + Object whereFormObject = getValueSafelyFromFormData(formData, LIST_WHERE); + + if (whereFormObject != null) { + Map whereForm = (Map) whereFormObject; + Condition condition = parseWhereClause(whereForm); + ArrayNode preFilteringResponse = objectMapper.valueToTree(actionResult); + actionResult = filterDataService.filterDataNew(preFilteringResponse, condition); + + } + break; case UPLOAD_FILE_FROM_BODY: diff --git a/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/plugins/constants/FieldName.java b/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/plugins/constants/FieldName.java index d6f62b716f..635fcb8c44 100644 --- a/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/plugins/constants/FieldName.java +++ b/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/plugins/constants/FieldName.java @@ -17,6 +17,7 @@ public class FieldName { public static final String PREFIX = "prefix"; public static final String SIGNED_URL = "signedUrl"; public static final String UNSIGNED_URL = "unSignedUrl"; + public static final String WHERE = "where"; public static final String CREATE_EXPIRY = CREATE + "." + EXPIRY; public static final String CREATE_DATATYPE = CREATE + "." + DATATYPE; @@ -28,6 +29,7 @@ public class FieldName { public static final String LIST_UNSIGNED_URL = LIST + "." + UNSIGNED_URL; public static final String READ_DATATYPE = READ + "." + DATATYPE; public static final String READ_USING_BASE64_ENCODING = READ + "." + USING_BASE64_ENCODING; + public static final String LIST_WHERE = LIST + "." + WHERE; } diff --git a/app/server/appsmith-plugins/amazons3Plugin/src/main/resources/editor/list.json b/app/server/appsmith-plugins/amazons3Plugin/src/main/resources/editor/list.json index f3a9fb4a33..8a05c8e4e2 100644 --- a/app/server/appsmith-plugins/amazons3Plugin/src/main/resources/editor/list.json +++ b/app/server/appsmith-plugins/amazons3Plugin/src/main/resources/editor/list.json @@ -29,6 +29,40 @@ "configProperty": "actionConfiguration.formData.list.prefix", "controlType": "QUERY_DYNAMIC_INPUT_TEXT", "initialValue": "" + }, + { + "label": "Where", + "configProperty": "actionConfiguration.formData.list.where", + "nestedLevels": 3, + "controlType": "WHERE_CLAUSE", + "logicalTypes": [ + { + "label": "AND", + "value": "AND" + }, + { + "label": "OR", + "value": "OR" + } + ], + "comparisonTypes": [ + { + "label": "==", + "value": "EQ" + }, + { + "label": "!=", + "value": "NOT_EQ" + }, + { + "label": "in", + "value": "IN" + }, + { + "label": "not in", + "value": "NOT_IN" + } + ] } ] },