feat: Implement data type handling in MySQL (#16621) (#17017)

* feat: Implement data type handling in MySQL (#16621)

With this implementation we can now achieve the following under prepared statement:
- Ability to distinguish between null object and "null" string
- Ability to distinguish values like {{"098765"}} and {{098765}}. The former is identified as a string and the latter is identified as an integer

* feat: Add unit test cases on data type handling in MySQL (#16621)

* chore: Move MySQL specific types to a separate class (#16621)

* chore: Remove import shortening (#16621)

* Fix testStructure test case to have loose coupling with the order of tables (#16621)

* Fix: Add missing client-side data types in params in a few test cases (#16621)

* Fix query in test case (#16621)

* Fix test cases and add small refactoring (#16621)

* Refactor assertion with a check on Optional (#16621)

* additional cypress test cases for mysql

* updated test case object

* update for failing test cases

* mysql failure point fix

* mysql false spec failure point fix

* fix flacy test cases

* flacky JSON test cases

Co-authored-by: ChandanBalajiBP <chandan@appsmith.com>
Co-authored-by: Aishwarya UR <aishwarya@appsmith.com>

app/client/cypress/integration/Smoke_TestSuite/ServerSideTests/Datatypes/MySQL_Spec.ts

@@ -36,7 +36,9 @@ describe("MySQL Datatype tests", function() {
     dataSources.RunQuery();
 
     ee.ActionContextMenuByEntityName(dsName, "Refresh");
-    agHelper.AssertElementVisible(ee._entityNameInExplorer(inputData.tableName));
+    agHelper.AssertElementVisible(
+      ee._entityNameInExplorer(inputData.tableName),
+    );
   });
 
   it("3. Creating SELECT query", () => {
@@ -54,32 +56,14 @@ describe("MySQL Datatype tests", function() {
     agHelper.RenameWithInPane("insertRecord");
     dataSources.EnterQuery(query);
 
-    query = inputData.query.deleteAllRecords;
-    ee.ActionTemplateMenuByEntityName(inputData.tableName, "DELETE");
-    agHelper.RenameWithInPane("deleteAllRecords");
-    dataSources.EnterQuery(query);
-
     query = inputData.query.dropTable;
     ee.ActionTemplateMenuByEntityName(inputData.tableName, "DELETE");
     agHelper.RenameWithInPane("dropTable");
     dataSources.EnterQuery(query);
   });
 
-  //Insert false values to each column and check for the error status of the request.
-  it("5. False Cases", () => {
-    ee.ActionTemplateMenuByEntityName(inputData.tableName, "INSERT");
-    agHelper.RenameWithInPane("falseCases");
-    inputData.falseResult.forEach((res_array, i) => {
-      res_array.forEach((value) => {
-        query = `INSERT INTO ${inputData.tableName} (${inputData.inputFieldName[i]}) VALUES (${value})`;
-        dataSources.EnterQuery(query);
-        dataSources.RunQuery(false);
-      });
-    });
-  });
-
   //Insert valid/true values into datasource
-  it("6. Inserting record", () => {
+  it("5. Inserting record", () => {
     ee.SelectEntityByName("Page1");
     deployMode.DeployApp();
     table.WaitForTableEmpty(); //asserting table is empty before inserting!
@@ -87,7 +71,8 @@ describe("MySQL Datatype tests", function() {
     inputData.input.forEach((valueArr, i) => {
       agHelper.ClickButton("Run InsertQuery");
       valueArr.forEach((value, index) => {
-        agHelper.EnterInputText(inputData.inputFieldName[index], value);
+        if (value !== "")
+          agHelper.EnterInputText(inputData.inputFieldName[index], value);
       });
       i % 2 && agHelper.ToggleSwitch("Bool_column");
       agHelper.ClickButton("insertRecord");
@@ -98,26 +83,41 @@ describe("MySQL Datatype tests", function() {
 
   //Verify weather expected value is present in each cell
   //i.e. weather right data is pushed and fetched from datasource.
-  it("7. Validating values in each cell", () => {
+  it("6. Validating values in each cell", () => {
     cy.wait(2000);
     inputData.result.forEach((res_array, i) => {
       res_array.forEach((value, j) => {
         table.ReadTableRowColumnData(j, i, 0).then(($cellData) => {
-          expect($cellData).to.eq(value);
+          if(i === inputData.result.length-1){
+            let obj = JSON.parse($cellData)
+            expect(JSON.stringify(obj)).to.eq(JSON.stringify(value));
+          }else{
+            expect($cellData).to.eq(value);
+          }
         });
       });
     });
   });
 
-  it("8. Deleting all records from table ", () => {
+  //null will be displayed as empty string in tables
-    agHelper.GetNClick(locator._deleteIcon);
+  //So test null we have to intercept execute request.
-    agHelper.Sleep(2000);
+  //And check response payload.
-    table.WaitForTableEmpty();
+  it("7. Testing null value", () => {
-  });
-
-  it("9. Validate Drop of the Newly Created - mysqlDTs - Table from MySQL datasource", () => {
     deployMode.NavigateBacktoEditor();
     ee.ExpandCollapseEntity("Queries/JS");
+    ee.SelectEntityByName("selectRecords");
+    dataSources.RunQuery(true, false);
+    cy.wait("@postExecute").then((intercept) => {
+      expect(
+        typeof intercept.response?.body.data.body[5].varchar_column,
+      ).to.be.equal("object");
+      expect(intercept.response?.body.data.body[5].varchar_column).to.be.equal(
+        null,
+      );
+    });
+  });
+
+  it("8. Validate drop of mysqlDTs - Table from MySQL datasource", () => {
     ee.SelectEntityByName("dropTable");
     dataSources.RunQuery();
     dataSources.ReadQueryTableResponse(0).then(($cellData) => {
@@ -134,12 +134,17 @@ describe("MySQL Datatype tests", function() {
     ee.ExpandCollapseEntity("Datasources", false);
   });
 
-  it("10. Verify Deletion of the datasource after all created queries are Deleted", () => {
+  it("9. Verify Deletion of the datasource after all created queries are Deleted", () => {
     dataSources.DeleteDatasouceFromWinthinDS(dsName, 409); //Since all queries exists
     ee.ExpandCollapseEntity("Queries/JS");
-    ["falseCases", "createTable", "deleteAllRecords", "dropTable", "insertRecord", "selectRecords"].forEach(type => {
+    [
+      "createTable",
+      "dropTable",
+      "insertRecord",
+      "selectRecords",
+    ].forEach((type) => {
       ee.ActionContextMenuByEntityName(type, "Delete", "Are you sure?");
-    })
+    });
     deployMode.DeployApp();
     deployMode.NavigateBacktoEditor();
     ee.ExpandCollapseEntity("Queries/JS");

app/client/cypress/integration/Smoke_TestSuite/ServerSideTests/Datatypes/MySQL_false_Spec.ts

@@ -0,0 +1,79 @@
+import { ObjectsRegistry } from "../../../../support/Objects/Registry";
+import inputData from "../../../../support/Objects/mySqlData";
+
+let dsName: any, query: string;
+const agHelper = ObjectsRegistry.AggregateHelper,
+  ee = ObjectsRegistry.EntityExplorer,
+  dataSources = ObjectsRegistry.DataSources,
+  propPane = ObjectsRegistry.PropertyPane,
+  table = ObjectsRegistry.Table,
+  locator = ObjectsRegistry.CommonLocators,
+  deployMode = ObjectsRegistry.DeployMode;
+
+describe("MySQL Datatype tests", function() {
+  it("1. Create Mysql DS", function() {
+    dataSources.CreateDataSource("MySql");
+    cy.get("@dsName").then(($dsName) => {
+      dsName = $dsName;
+    });
+  });
+
+  it("2. Creating mysqlDTs table", () => {
+    //IF NOT EXISTS can be used - which creates tabel if it does not exist and donot throw any error if table exists.
+    //But if we add this option then next case could fail inn that case.
+    query = inputData.query.createTable;
+    ee.CreateNewDsQuery(dsName);
+    agHelper.RenameWithInPane("createTable");
+    agHelper.GetNClick(dataSources._templateMenu);
+    dataSources.EnterQuery(query);
+    dataSources.RunQuery();
+
+    ee.ActionContextMenuByEntityName(dsName, "Refresh");
+    agHelper.AssertElementVisible(
+      ee._entityNameInExplorer(inputData.tableName),
+    );
+  });
+
+  //Insert false values to each column and check for the error status of the request.
+  it("3. False Cases", () => {
+    ee.ActionTemplateMenuByEntityName(inputData.tableName, "INSERT");
+    agHelper.RenameWithInPane("falseCases");
+    inputData.falseResult.forEach((res_array, i) => {
+      res_array.forEach((value) => {
+        query =
+          typeof value === "string"
+            ? `INSERT INTO ${inputData.tableName} (${inputData.inputFieldName[i]}) VALUES ({{"${value}"}})`
+            : `INSERT INTO ${inputData.tableName} (${inputData.inputFieldName[i]}) VALUES ({{${value}}})`;
+        dataSources.EnterQuery(query);
+        dataSources.RunQuery(false);
+      });
+    });
+    agHelper.Sleep(2000);
+    agHelper.WaitUntilAllToastsDisappear();
+  });
+
+  //This is a special case.
+  //Added due to server side checks, which was handled in Datatype handling.
+  it("4. Long Integer as query param", () => {
+    query = `SELECT * FROM ${inputData.tableName} LIMIT {{2147483648}}`;
+    dataSources.EnterQuery(query);
+    dataSources.RunQuery();
+  });
+
+  it("5. Drop Table", () => {
+    query = inputData.query.dropTable;
+    dataSources.EnterQuery(query);
+    dataSources.RunQuery();
+  });
+
+  it("6. Verify Deletion of the datasource after all created queries are Deleted", () => {
+    ee.ExpandCollapseEntity("Queries/JS");
+    ["falseCases", "createTable"].forEach((type) => {
+      ee.ActionContextMenuByEntityName(type, "Delete", "Are you sure?");
+    });
+    deployMode.DeployApp();
+    deployMode.NavigateBacktoEditor();
+    ee.ExpandCollapseEntity("Queries/JS");
+    dataSources.DeleteDatasouceFromWinthinDS(dsName, 200);
+  });
+});

app/client/cypress/support/Objects/mySqlData.ts

@@ -1,6 +1,6 @@
 const mySqlData = {
   tableName: "mysqlDTs",
-  inputFieldName : [
+  inputFieldName: [
     "Stinyint_column",
     "Utinyint_column",
     "Ssmallint_column",
@@ -23,7 +23,7 @@ const mySqlData = {
     "Enum_column",
     "Json_column",
   ],
-  input : [
+  input: [
     [
       "-128",
       "0",
@@ -88,11 +88,34 @@ const mySqlData = {
       "2012/12/31",
       "-838:59:59",
       "1901'",
-      "null",
+      "12345678912345",
-      "null",
+      "012345",
       "c",
       "[1, 2, 3, 4]",
     ],
+    [
+      "0",
+      "0",
+      "0",
+      "0",
+      "0",
+      "0",
+      "0123",
+      "0",
+      "0",
+      "0",
+      "0",
+      "0",
+      "20121231113045",
+      "121231113045",
+      "121231",
+      "11:12",
+      "2155",
+      "true",
+      "false",
+      "c",
+      "[]",
+    ],
     [
       "0",
       "0",
@@ -109,33 +132,10 @@ const mySqlData = {
       "20121231113045",
       "121231113045",
       "121231",
-      "11:12",
-      "2155",
-      "null",
-      "null",
-      "c",
-      "[]",
-    ],
-    [
-      "0",
-      "0",
-      "0",
-      "0",
-      "0",
-      "0",
-      "0",
-      "0",
-      "0",
-      "0",
-      "0",
-      "null",
-      "121231113045",
-      "null",
-      "null",
       "1112",
+      "2022",
       "null",
-      "null",
+      "NulL",
-      "null",
       "c",
       '["a",true,0,12.34]',
     ],
@@ -151,19 +151,19 @@ const mySqlData = {
       "0",
       "0",
       "0",
-      "null",
+      "0",
-      "null",
+      "20121231113045",
-      "null",
+      "121231113045",
-      "null",
+      "121231",
       "12",
-      "null",
+      "2022",
-      "null",
+      "",
-      "null",
+      "abc",
       "c",
-      "null",
+      '{}',
     ],
   ],
-  falseResult : [
+  falseResult: [
     [-129, 128],
     [-1, 256],
     [-32769, 32768],
@@ -172,9 +172,9 @@ const mySqlData = {
     [-1, 16777216],
     [-2147483649, 2147483648],
     [-1, 4294967296],
-    [-9223372036854775808, 9223372036854775807],
+    [],
     [123456789.45],
-    ['a'],
+    ["a"],
     [123456789.45],
     ["2012/12/31 11:30:451"],
     ["2012/12/311 11:30:45"],
@@ -184,9 +184,9 @@ const mySqlData = {
     ["abcdefghijklmnopqrstu"],
     ["abcdefghijk"],
     ["d"],
-    ["abc", "{", "["],
+    [],
   ],
-  result : [
+  result: [
     ["1", "2", "3", "4", "5"],
     ["-128", "0", "127"],
     ["0", "255"],
@@ -194,7 +194,7 @@ const mySqlData = {
     ["0", "65535"],
     ["-8388608", "0", "8388607"],
     ["0", "16777215"],
-    ["-2147483648", "0", "2147483647"],
+    ["-2147483648", "0", "2147483647", "123"],
     ["0", "4294967295"],
     ["123456", "456789", "123456789"],
     ["123.45", "123.46", "123.45"],
@@ -216,30 +216,28 @@ const mySqlData = {
     ["2012-12-31", "2012-12-31", "2012-12-31", "2012-12-31"],
     ["22:59:59", "00:00:00", "01:00:01", "11:12:00", "00:11:12", "00:00:12"],
     ["1901", "2155", "1901", "2155"],
-    ["a", "abcdefghijklmnopqrst"],
+    ["a", "abcdefghijklmnopqrst", "12345678912345", "true", "null"],
-    ["a", "abcdefghij"],
+    ["a", "abcdefghij", "012345", "false", "NulL"],
     ["a", "b", "c"],
     ["0", "1"],
-    ['{"abc": "123"}', "{}", "[1, 2, 3, 4]", "", '["a",true,0,12.34]'],
+    [{"abc": "123"}, {}, [1, 2, 3, 4], [], ["a",true,0,12.34]],
   ],
-  query :{
+  query: {
-    createTable : `CREATE TABLE mysqlDTs (serialId SERIAL not null primary key, stinyint_column TINYINT, utinyint_column TINYINT UNSIGNED, 
+    createTable: `CREATE TABLE mysqlDTs (serialId SERIAL not null primary key, stinyint_column TINYINT, utinyint_column TINYINT UNSIGNED, 
       ssmallint_column SMALLINT, usmallint_column SMALLINT UNSIGNED, smediumint_column MEDIUMINT, umediumint_column MEDIUMINT UNSIGNED, 
       sint_column INT, uint_column INT UNSIGNED, bigint_column BIGINT, float_column FLOAT( 10, 2 ), double_column DOUBLE, decimal_column DECIMAL( 10, 2 ), 
       datetime_column DATETIME, timestamp_column TIMESTAMP, date_column DATE, time_column TIME, year_column YEAR, varchar_column VARCHAR( 20 ), 
       char_column CHAR( 10 ), enum_column ENUM( 'a', 'b', 'c' ), bool_column BOOL, json_column JSON);`,
-    insertRecord : `INSERT INTO mysqlDTs (stinyint_column, utinyint_column, ssmallint_column, usmallint_column, smediumint_column, umediumint_column, 
+    insertRecord: `INSERT INTO mysqlDTs (stinyint_column, utinyint_column, ssmallint_column, usmallint_column, smediumint_column, umediumint_column, 
       sint_column, uint_column, bigint_column, float_column, double_column, decimal_column, datetime_column, timestamp_column, 
       date_column, time_column, year_column, varchar_column, char_column, enum_column, bool_column, json_column ) 
       VALUES 
       ({{InsertStinyint.text}}, {{InsertUtinyint.text}}, {{InsertSsmallint.text}}, {{InsertUsmallint.text}}, {{InsertSmediumint.text}}, 
       {{InsertUmediumint.text}}, {{InsertSint.text}}, {{InsertUint.text}}, {{InsertBigint.text}}, {{InsertFloat.text}}, {{InsertDouble.text}}, 
       {{InsertDecimal.text}}, {{InsertDatetime.text}}, {{InsertTimestamp.text}}, {{InsertDate.text}}, {{InsertTime.text}}, 
-      {{InsertYear.text}}, {{InsertVarchar.text}}, {{InsertChar.text}}, {{InsertEnum.text}}, {{InsertBoolean.isSwitchedOn}}, {{InputJson.text}});`,
+      {{InsertYear.text}}, {{InsertVarchar.text ? InsertVarchar.text : null}}, {{InsertChar.text}}, {{InsertEnum.text}}, {{InsertBoolean.isSwitchedOn}}, {{InputJson.text}});`,
-    deleteRecord : `DELETE FROM mysqlDTs WHERE serialId ={{Table1.selectedRow.serialid}}`,
-    deleteAllRecords : `DELETE FROM mysqlDTs`,
     dropTable: `drop table mysqlDTs`,
-  } 
+  },
 };
 
 export default mySqlData;

app/server/appsmith-interfaces/src/main/java/com/appsmith/external/plugins/SmartSubstitutionInterface.java

@@ -5,6 +5,7 @@ import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginError;
 import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginException;
 import com.appsmith.external.models.Param;
 
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -39,7 +40,7 @@ public interface SmartSubstitutionInterface {
                     String value = matchingParam.get().getValue();
 
                     input = substituteValueInInput(i + 1, key,
-                            value, input, insertedParams, args);
+                            value, input, insertedParams, append(args, matchingParam.get().getClientDataType()));
                 } else {
                     throw new AppsmithPluginException(AppsmithPluginError.PLUGIN_ERROR, "Uh oh! This is unexpected. " +
                             "Did not receive any information for the binding "
@@ -71,4 +72,11 @@ public interface SmartSubstitutionInterface {
     default String sanitizeReplacement(String replacementValue, DataType dataType) {
         return replacementValue;
     }
+
+    static <T> T[] append(T[] arr, T lastElement) {
+        final int N = arr.length;
+        arr = Arrays.copyOf(arr, N+1);
+        arr[N] = lastElement;
+        return arr;
+    }
 }

app/server/appsmith-plugins/mysqlPlugin/src/main/java/com/external/plugins/MySqlPlugin.java

@@ -1,11 +1,12 @@
 package com.external.plugins;
 
-import com.appsmith.external.constants.DataType;
+import com.appsmith.external.datatypes.AppsmithType;
+import com.appsmith.external.datatypes.ClientDataType;
 import com.appsmith.external.dtos.ExecuteActionDTO;
 import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginError;
 import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginException;
 import com.appsmith.external.exceptions.pluginExceptions.StaleConnectionException;
-import com.appsmith.external.helpers.DataTypeStringUtils;
+import com.appsmith.external.helpers.DataTypeServiceUtils;
 import com.appsmith.external.helpers.MustacheHelper;
 import com.appsmith.external.models.ActionConfiguration;
 import com.appsmith.external.models.ActionExecutionRequest;
@@ -22,6 +23,7 @@ import com.appsmith.external.models.SSLDetails;
 import com.appsmith.external.plugins.BasePlugin;
 import com.appsmith.external.plugins.PluginExecutor;
 import com.appsmith.external.plugins.SmartSubstitutionInterface;
+import com.external.plugins.datatypes.MySQLSpecificDataTypes;
 import com.external.utils.QueryUtils;
 import io.r2dbc.spi.ColumnMetadata;
 import io.r2dbc.spi.Connection;
@@ -340,7 +342,7 @@ public class MySqlPlugin extends BasePlugin {
 
         }
 
-        private boolean isIsOperatorUsed(String query) {
+        boolean isIsOperatorUsed(String query) {
             String queryKeyWordsOnly = query.replaceAll(MATCH_QUOTED_WORDS_REGEX, "");
             return Arrays.stream(queryKeyWordsOnly.split("\\s"))
                     .anyMatch(word -> IS_KEY.equalsIgnoreCase(word.trim()));
@@ -394,29 +396,36 @@ public class MySqlPlugin extends BasePlugin {
                                              Object... args) {
 
             Statement connectionStatement = (Statement) input;
-            DataType valueType = DataTypeStringUtils.stringToKnownDataTypeConverter(value);
+            ClientDataType clientDataType = (ClientDataType) args[0];
+            AppsmithType appsmithType = DataTypeServiceUtils.getAppsmithType(clientDataType, value, MySQLSpecificDataTypes.pluginSpecificTypes);
 
-            Map.Entry<String, String> parameter = new SimpleEntry<>(value, valueType.toString());
+            Map.Entry<String, String> parameter = new SimpleEntry<>(value, appsmithType.type().toString());
             insertedParams.add(parameter);
 
-            if (DataType.NULL.equals(valueType)) {
+            switch (appsmithType.type()) {
-                try {
+                case NULL:
-                    connectionStatement.bindNull((index - 1), Object.class);
+                    try {
-                } catch (UnsupportedOperationException e) {
+                        connectionStatement.bindNull((index - 1), Object.class);
-                    // Do nothing. Move on
+                    } catch (UnsupportedOperationException e) {
-                }
+                        // Do nothing. Move on
-            } else if (DataType.INTEGER.equals(valueType)) {
+                    }
-                /**
+                    break;
-                 * - NumberFormatException is NOT expected here since stringToKnownDataTypeConverter uses parseInt
+                case BOOLEAN:
-                 * method to detect INTEGER type.
+                    connectionStatement.bind((index - 1), appsmithType.performSmartSubstitution(value));
-                 */
+                    break;
-                connectionStatement.bind((index - 1), Integer.parseInt(value));
+                case INTEGER:
-            } else if (DataType.BOOLEAN.equals(valueType)) {
+                    connectionStatement.bind((index - 1), Integer.parseInt(value));
-                connectionStatement.bind((index - 1), Boolean.parseBoolean(value) == TRUE ? 1 : 0);
+                    break;
-            } else {
+                case LONG:
-                connectionStatement.bind((index - 1), value);
+                    connectionStatement.bind((index - 1), Long.parseLong(value));
+                    break;
+                case DOUBLE:
+                    connectionStatement.bind((index - 1), Double.parseDouble(value));
+                    break;
+                default:
+                    connectionStatement.bind((index - 1), value);
+                    break;
             }
-
             return connectionStatement;
         }
 

app/server/appsmith-plugins/mysqlPlugin/src/main/java/com/external/plugins/datatypes/MySQLSpecificDataTypes.java

@@ -0,0 +1,44 @@
+package com.external.plugins.datatypes;
+
+import com.appsmith.external.datatypes.AppsmithType;
+import com.appsmith.external.datatypes.BigDecimalType;
+import com.appsmith.external.datatypes.ClientDataType;
+import com.appsmith.external.datatypes.DoubleType;
+import com.appsmith.external.datatypes.IntegerType;
+import com.appsmith.external.datatypes.JsonObjectType;
+import com.appsmith.external.datatypes.LongType;
+import com.appsmith.external.datatypes.NullType;
+import com.appsmith.external.datatypes.StringType;
+import com.appsmith.external.datatypes.TimeType;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class MySQLSpecificDataTypes {
+    public static final Map<ClientDataType, List<AppsmithType>> pluginSpecificTypes;
+
+    static {
+        pluginSpecificTypes = new HashMap<>();
+        pluginSpecificTypes.put(ClientDataType.NULL, List.of(new NullType()));
+
+        pluginSpecificTypes.put(ClientDataType.BOOLEAN, List.of(new MySQLBooleanType()));
+
+        pluginSpecificTypes.put(ClientDataType.NUMBER, List.of(
+                new IntegerType(),
+                new LongType(),
+                new DoubleType(),
+                new BigDecimalType()
+        ));
+
+        pluginSpecificTypes.put(ClientDataType.OBJECT, List.of(new JsonObjectType()));
+
+        pluginSpecificTypes.put(ClientDataType.STRING, List.of(
+                new TimeType(),
+                new MySQLDateType(),
+                new MySQLDateTimeType(),
+                new StringType()
+        ));
+    }
+
+}