feat: Add server-side base data types (#15784) (#16240)

* feat: Add server-side base data types (#15784)

This commit takes care of the following two things
- It adds the server-side base data types which are considered common data types across plugins
- It adds a few MySQL specific types

* feat: Add MySQLBooleanType and refactor MySQLDateTimeType with camelCase (#15784)

* feat: Move MySQL specific types to mysqlPlugin folder (#15784)

app/server/appsmith-interfaces/src/main/java/com/appsmith/external/constants/DataType.java

@@ -17,5 +17,6 @@ public enum DataType {
     JSON_OBJECT,
     TIMESTAMP,
     BSON,
-    BSON_SPECIAL_DATA_TYPES
-}
+    BSON_SPECIAL_DATA_TYPES,
+    BIGDECIMAL
+}

app/server/appsmith-interfaces/src/main/java/com/appsmith/external/datatypes/AppsmithType.java

@@ -0,0 +1,10 @@
+package com.appsmith.external.datatypes;
+
+import com.appsmith.external.constants.DataType;
+
+import java.util.function.Predicate;
+
+public interface AppsmithType extends Predicate<String> {
+    String performSmartSubstitution(String value);
+    DataType type();
+}

app/server/appsmith-interfaces/src/main/java/com/appsmith/external/datatypes/ArrayType.java

@@ -0,0 +1,62 @@
+package com.appsmith.external.datatypes;
+
+import com.appsmith.external.constants.DataType;
+import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginError;
+import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginException;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import net.minidev.json.JSONArray;
+import net.minidev.json.parser.JSONParser;
+import reactor.core.Exceptions;
+
+import java.util.List;
+
+public class ArrayType implements AppsmithType {
+
+    private static final ObjectMapper objectMapper = new ObjectMapper();
+    private static final JSONParser parser = new JSONParser(JSONParser.MODE_PERMISSIVE);
+
+    @Override
+    public boolean test(String s) {
+        final String trimmedValue = s.trim();
+
+        if (trimmedValue.startsWith("[") && trimmedValue.endsWith("]")) {
+            String betweenBraces = trimmedValue.substring(1, trimmedValue.length() - 1);
+            String trimmedInputBetweenBraces = betweenBraces.trim();
+            // In case of no values in the array, set this as null otherwise plugins like postgres and ms-sql
+            // would break while creating a SQL array.
+            if (!trimmedInputBetweenBraces.isEmpty()) {
+                try {
+                    new ObjectMapper().readValue(trimmedValue, List.class);
+                    return true;
+                } catch (JsonProcessingException e) {
+                    return false;
+                }
+
+            }
+        }
+
+        return false;
+    }
+
+    @Override
+    public String performSmartSubstitution(String s) {
+        try {
+            JSONArray jsonArray = (JSONArray) parser.parse(s);
+            return objectMapper.writeValueAsString(jsonArray);
+        } catch (net.minidev.json.parser.ParseException | JsonProcessingException e) {
+            throw Exceptions.propagate(
+                    new AppsmithPluginException(
+                            AppsmithPluginError.PLUGIN_JSON_PARSE_ERROR,
+                            s,
+                            e.getMessage()
+                    )
+            );
+        }
+    }
+
+    @Override
+    public DataType type() {
+        return DataType.ARRAY;
+    }
+}

app/server/appsmith-interfaces/src/main/java/com/appsmith/external/datatypes/BigDecimalType.java

@@ -0,0 +1,28 @@
+package com.appsmith.external.datatypes;
+
+import com.appsmith.external.constants.DataType;
+
+import java.math.BigDecimal;
+
+public class BigDecimalType implements AppsmithType{
+    @Override
+    public String performSmartSubstitution(String s) {
+        return s;
+    }
+
+    @Override
+    public boolean test(String s) {
+        try {
+            new BigDecimal(s);
+            return true;
+        } catch (NumberFormatException e) {
+            // Not a BigDecimal
+        }
+        return false;
+    }
+
+    @Override
+    public DataType type() {
+        return DataType.BIGDECIMAL;
+    }
+}

app/server/appsmith-interfaces/src/main/java/com/appsmith/external/datatypes/BooleanType.java

@@ -0,0 +1,21 @@
+package com.appsmith.external.datatypes;
+
+import com.appsmith.external.constants.DataType;
+
+public class BooleanType implements AppsmithType {
+
+    @Override
+    public boolean test(String s) {
+        return "true".equalsIgnoreCase(s) || "false".equalsIgnoreCase(s);
+    }
+
+    @Override
+    public String performSmartSubstitution(String s) {
+        return String.valueOf(s);
+    }
+
+    @Override
+    public DataType type() {
+        return DataType.BOOLEAN;
+    }
+}

app/server/appsmith-interfaces/src/main/java/com/appsmith/external/datatypes/ClientDataType.java

@@ -0,0 +1,10 @@
+package com.appsmith.external.datatypes;
+
+public enum ClientDataType {
+    NULL,
+    ARRAY,
+    BOOLEAN,
+    NUMBER,
+    OBJECT,
+    STRING
+}

app/server/appsmith-interfaces/src/main/java/com/appsmith/external/datatypes/DateType.java

@@ -0,0 +1,55 @@
+package com.appsmith.external.datatypes;
+
+import com.appsmith.external.constants.DataType;
+import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginError;
+import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginException;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import reactor.core.Exceptions;
+
+import java.time.LocalDate;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeFormatterBuilder;
+import java.time.format.DateTimeParseException;
+import java.util.regex.Matcher;
+
+public class DateType implements AppsmithType {
+
+    private static final ObjectMapper objectMapper = new ObjectMapper();
+
+    @Override
+    public boolean test(String s) {
+        try {
+            final DateTimeFormatter dateFormatter = new DateTimeFormatterBuilder()
+                    .appendOptional(DateTimeFormatter.ISO_LOCAL_DATE)
+                    .toFormatter();
+            LocalDate.parse(s, dateFormatter);
+            return true;
+        } catch (DateTimeParseException ex) {
+            // Not date
+        }
+
+        return false;
+    }
+
+    @Override
+    public String performSmartSubstitution(String s) {
+        try {
+            String valueAsString = objectMapper.writeValueAsString(s);
+            return Matcher.quoteReplacement(valueAsString);
+        } catch (JsonProcessingException e) {
+            throw Exceptions.propagate(
+                    new AppsmithPluginException(
+                            AppsmithPluginError.PLUGIN_EXECUTE_ARGUMENT_ERROR,
+                            s,
+                            e.getMessage()
+                    )
+            );
+        }
+    }
+
+    @Override
+    public DataType type() {
+        return DataType.DATE;
+    }
+}

app/server/appsmith-interfaces/src/main/java/com/appsmith/external/datatypes/DoubleType.java

@@ -0,0 +1,26 @@
+package com.appsmith.external.datatypes;
+
+import com.appsmith.external.constants.DataType;
+
+public class DoubleType implements AppsmithType{
+    @Override
+    public String performSmartSubstitution(String value) {
+        return String.valueOf(value);
+    }
+
+    @Override
+    public boolean test(String s) {
+        try {
+            Double.parseDouble(s);
+            return true;
+        } catch (NumberFormatException e) {
+            // Not a double
+        }
+        return false;
+    }
+
+    @Override
+    public DataType type() {
+        return DataType.DOUBLE;
+    }
+}

app/server/appsmith-interfaces/src/main/java/com/appsmith/external/datatypes/FloatType.java

@@ -0,0 +1,27 @@
+package com.appsmith.external.datatypes;
+
+import com.appsmith.external.constants.DataType;
+
+public class FloatType implements AppsmithType {
+
+    @Override
+    public boolean test(String s) {
+        try {
+            Float.parseFloat(s);
+            return true;
+        } catch (NumberFormatException e) {
+            // Not a float
+        }
+        return false;
+    }
+
+    @Override
+    public String performSmartSubstitution(String s) {
+        return String.valueOf(s);
+    }
+
+    @Override
+    public DataType type() {
+        return DataType.FLOAT;
+    }
+}

app/server/appsmith-interfaces/src/main/java/com/appsmith/external/datatypes/IntegerType.java

@@ -0,0 +1,26 @@
+package com.appsmith.external.datatypes;
+
+import com.appsmith.external.constants.DataType;
+
+public class IntegerType implements AppsmithType{
+    @Override
+    public String performSmartSubstitution(String s) {
+        return s;
+    }
+
+    @Override
+    public boolean test(String s) {
+        try {
+            Integer.parseInt(s);
+            return true;
+        } catch (NumberFormatException e) {
+            // Not an integer
+        }
+        return false;
+    }
+
+    @Override
+    public DataType type() {
+        return DataType.INTEGER;
+    }
+}

app/server/appsmith-interfaces/src/main/java/com/appsmith/external/datatypes/JsonObjectType.java

@@ -0,0 +1,63 @@
+package com.appsmith.external.datatypes;
+
+import com.appsmith.external.constants.DataType;
+import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginError;
+import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginException;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.gson.Gson;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonSyntaxException;
+import com.google.gson.TypeAdapter;
+import com.google.gson.stream.JsonReader;
+import net.minidev.json.JSONObject;
+import net.minidev.json.parser.JSONParser;
+import reactor.core.Exceptions;
+
+import java.io.IOException;
+import java.io.StringReader;
+import java.util.regex.Matcher;
+
+public class JsonObjectType implements AppsmithType {
+
+    private static final TypeAdapter<JsonObject> strictGsonObjectAdapter =
+            new Gson().getAdapter(JsonObject.class);
+    private static final ObjectMapper objectMapper = new ObjectMapper();
+    private static final JSONParser parser = new JSONParser(JSONParser.MODE_PERMISSIVE);
+
+    @Override
+    public boolean test(String s) {
+        try (JsonReader reader = new JsonReader(new StringReader(s))) {
+            strictGsonObjectAdapter.read(reader);
+            reader.hasNext(); // throws on multiple top level values
+            return true;
+        } catch (IOException | JsonSyntaxException e) {
+            // Not a strict JSON object
+        }
+
+        return false;
+    }
+
+    @Override
+    public String performSmartSubstitution(String s) {
+        try {
+            JSONObject jsonObject = (JSONObject) parser.parse(s);
+            String jsonString = objectMapper.writeValueAsString(jsonObject);
+            // Adding Matcher.quoteReplacement so that "/" and "$" in the string are escaped during replacement
+            return Matcher.quoteReplacement(jsonString);
+        } catch (net.minidev.json.parser.ParseException | JsonProcessingException e) {
+            throw Exceptions.propagate(
+                    new AppsmithPluginException(
+                            AppsmithPluginError.PLUGIN_JSON_PARSE_ERROR,
+                            s,
+                            e.getMessage()
+                    )
+            );
+        }
+    }
+
+    @Override
+    public DataType type() {
+        return DataType.JSON_OBJECT;
+    }
+}

app/server/appsmith-interfaces/src/main/java/com/appsmith/external/datatypes/LongType.java

@@ -0,0 +1,27 @@
+package com.appsmith.external.datatypes;
+
+import com.appsmith.external.constants.DataType;
+
+public class LongType implements AppsmithType {
+
+    @Override
+    public boolean test(String s) {
+        try {
+            Long.parseLong(s);
+            return true;
+        } catch (NumberFormatException e) {
+            // Not a long
+        }
+        return false;
+    }
+
+    @Override
+    public String performSmartSubstitution(String s) {
+        return s;
+    }
+
+    @Override
+    public DataType type() {
+        return DataType.LONG;
+    }
+}

app/server/appsmith-interfaces/src/main/java/com/appsmith/external/datatypes/NullType.java

@@ -0,0 +1,27 @@
+package com.appsmith.external.datatypes;
+
+import com.appsmith.external.constants.DataType;
+
+import javax.naming.OperationNotSupportedException;
+
+public class NullType implements AppsmithType {
+
+    @Override
+    public boolean test(String s) {
+        if (s == null) {
+            return true;
+        }
+        final String trimmedValue = s.trim();
+        return "null".equalsIgnoreCase(trimmedValue);
+    }
+
+    @Override
+    public String performSmartSubstitution(String s) {
+        return null;
+    }
+
+    @Override
+    public DataType type() {
+        return DataType.NULL;
+    }
+}

app/server/appsmith-interfaces/src/main/java/com/appsmith/external/datatypes/StringType.java

@@ -0,0 +1,41 @@
+package com.appsmith.external.datatypes;
+
+import com.appsmith.external.constants.DataType;
+import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginError;
+import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginException;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import reactor.core.Exceptions;
+
+import java.util.regex.Matcher;
+
+public class StringType implements AppsmithType {
+
+    private static final ObjectMapper objectMapper = new ObjectMapper();
+
+    @Override
+    public boolean test(String s) {
+        return true;
+    }
+
+    @Override
+    public String performSmartSubstitution(String s) {
+        try {
+            String valueAsString = objectMapper.writeValueAsString(s);
+            return Matcher.quoteReplacement(valueAsString);
+        } catch (JsonProcessingException e) {
+            throw Exceptions.propagate(
+                    new AppsmithPluginException(
+                            AppsmithPluginError.PLUGIN_EXECUTE_ARGUMENT_ERROR,
+                            s,
+                            e.getMessage()
+                    )
+            );
+        }
+    }
+
+    @Override
+    public DataType type() {
+        return DataType.STRING;
+    }
+}

app/server/appsmith-interfaces/src/main/java/com/appsmith/external/datatypes/TimeType.java

@@ -0,0 +1,56 @@
+package com.appsmith.external.datatypes;
+
+import com.appsmith.external.constants.DataType;
+import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginError;
+import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginException;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import reactor.core.Exceptions;
+
+import java.time.LocalTime;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeFormatterBuilder;
+import java.time.format.DateTimeParseException;
+import java.util.regex.Matcher;
+
+public class TimeType implements AppsmithType {
+
+    private static final ObjectMapper objectMapper = new ObjectMapper();
+
+    @Override
+    public boolean test(String s) {
+        try {
+            final DateTimeFormatter timeFormatter = new DateTimeFormatterBuilder()
+                    .appendOptional(DateTimeFormatter.ISO_LOCAL_TIME)
+                    .appendOptional(DateTimeFormatter.ISO_TIME)
+                    .toFormatter();
+            LocalTime.parse(s, timeFormatter);
+            return true;
+        } catch (DateTimeParseException ex) {
+            // Not time
+        }
+
+        return false;
+    }
+
+    @Override
+    public String performSmartSubstitution(String s) {
+        try {
+            String valueAsString = objectMapper.writeValueAsString(s);
+            return Matcher.quoteReplacement(valueAsString);
+        } catch (JsonProcessingException e) {
+            throw Exceptions.propagate(
+                    new AppsmithPluginException(
+                            AppsmithPluginError.PLUGIN_EXECUTE_ARGUMENT_ERROR,
+                            s,
+                            e.getMessage()
+                    )
+            );
+        }
+    }
+
+    @Override
+    public DataType type() {
+        return DataType.TIME;
+    }
+}

app/server/appsmith-interfaces/src/main/java/com/appsmith/external/datatypes/TimestampType.java

@@ -0,0 +1,54 @@
+package com.appsmith.external.datatypes;
+
+import com.appsmith.external.constants.DataType;
+import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginError;
+import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginException;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import reactor.core.Exceptions;
+
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeFormatterBuilder;
+import java.time.format.DateTimeParseException;
+import java.util.regex.Matcher;
+
+public class TimestampType implements AppsmithType {
+
+    private static final ObjectMapper objectMapper = new ObjectMapper();
+
+    @Override
+    public boolean test(String s) {
+        try {
+            final DateTimeFormatter dateTimeFormatter = new DateTimeFormatterBuilder()
+                    .appendOptional(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"))
+                    .toFormatter();
+            LocalDateTime.parse(s, dateTimeFormatter);
+            return true;
+        } catch (DateTimeParseException ex) {
+            // Not timestamp
+        }
+        return false;
+    }
+
+    @Override
+    public String performSmartSubstitution(String s) {
+        try {
+            String valueAsString = objectMapper.writeValueAsString(s);
+            return Matcher.quoteReplacement(valueAsString);
+        } catch (JsonProcessingException e) {
+            throw Exceptions.propagate(
+                    new AppsmithPluginException(
+                            AppsmithPluginError.PLUGIN_EXECUTE_ARGUMENT_ERROR,
+                            s,
+                            e.getMessage()
+                    )
+            );
+        }
+    }
+
+    @Override
+    public DataType type() {
+        return DataType.TIMESTAMP;
+    }
+}

app/server/appsmith-plugins/mysqlPlugin/src/main/java/com/external/plugins/datatypes/MySQLBooleanType.java

@@ -0,0 +1,25 @@
+package com.external.plugins.datatypes;
+
+import com.appsmith.external.constants.DataType;
+import com.appsmith.external.datatypes.AppsmithType;
+
+public class MySQLBooleanType implements AppsmithType {
+
+    @Override
+    public boolean test(String s) {
+        return "true".equalsIgnoreCase(s) || "false".equalsIgnoreCase(s);
+    }
+
+    @Override
+    public String performSmartSubstitution(String s) {
+        if ("true".equalsIgnoreCase(s)) {
+            return String.valueOf(1);
+        }
+        return String.valueOf(0);
+    }
+
+    @Override
+    public DataType type() {
+        return DataType.BOOLEAN;
+    }
+}

app/server/appsmith-plugins/mysqlPlugin/src/main/java/com/external/plugins/datatypes/MySQLDateTimeType.java

@@ -0,0 +1,58 @@
+package com.external.plugins.datatypes;
+
+import com.appsmith.external.constants.DataType;
+import com.appsmith.external.datatypes.AppsmithType;
+import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginError;
+import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginException;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import reactor.core.Exceptions;
+
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeFormatterBuilder;
+import java.time.format.DateTimeParseException;
+import java.util.regex.Matcher;
+
+public class MySQLDateTimeType implements AppsmithType {
+    private static final ObjectMapper objectMapper = new ObjectMapper();
+
+    @Override
+    public boolean test(String s) {
+        try {
+            final DateTimeFormatter dateTimeFormatter = new DateTimeFormatterBuilder()
+                    .appendOptional(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"))
+                    .appendOptional(DateTimeFormatter.ISO_DATE_TIME)
+                    .appendOptional(DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss"))
+                    .appendOptional(DateTimeFormatter.ofPattern("yyyyMMddHHmmss"))
+                    .appendOptional(DateTimeFormatter.ofPattern("yyMMddHHmmss"))
+                    .toFormatter();
+            LocalDateTime.parse(s, dateTimeFormatter);
+            return true;
+        } catch (DateTimeParseException ex) {
+            // Not timestamp
+        }
+        return false;
+    }
+
+    @Override
+    public String performSmartSubstitution(String s) {
+        try {
+            String valueAsString = objectMapper.writeValueAsString(s);
+            return Matcher.quoteReplacement(valueAsString);
+        } catch (JsonProcessingException e) {
+            throw Exceptions.propagate(
+                    new AppsmithPluginException(
+                            AppsmithPluginError.PLUGIN_EXECUTE_ARGUMENT_ERROR,
+                            s,
+                            e.getMessage()
+                    )
+            );
+        }
+    }
+
+    @Override
+    public DataType type() {
+        return DataType.TIMESTAMP;
+    }
+}

app/server/appsmith-plugins/mysqlPlugin/src/main/java/com/external/plugins/datatypes/MySQLDateType.java

@@ -0,0 +1,29 @@
+package com.external.plugins.datatypes;
+
+import com.appsmith.external.datatypes.DateType;
+
+import java.time.LocalDate;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeFormatterBuilder;
+import java.time.format.DateTimeParseException;
+
+public class MySQLDateType extends DateType {
+
+    @Override
+    public boolean test(String s) {
+        try {
+            final DateTimeFormatter dateFormatter = new DateTimeFormatterBuilder()
+                    .appendOptional(DateTimeFormatter.ISO_LOCAL_DATE)
+                    .appendOptional(DateTimeFormatter.BASIC_ISO_DATE)
+                    .appendOptional(DateTimeFormatter.ofPattern("yyyy/MM/dd"))
+                    .appendOptional(DateTimeFormatter.ofPattern("yyMMdd"))
+                    .toFormatter();
+            LocalDate.parse(s, dateFormatter);
+            return true;
+        } catch (DateTimeParseException ex) {
+            // Not date
+        }
+
+        return false;
+    }
+}