Protei/PromucFlow_constructor
2
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

feat: Implement data type handling in MySQL (#16621) (#17017)

Browse Source 
* feat: Implement data type handling in MySQL (#16621)

With this implementation we can now achieve the following under prepared statement:
- Ability to distinguish between null object and "null" string
- Ability to distinguish values like {{"098765"}} and {{098765}}. The former is identified as a string and the latter is identified as an integer

* feat: Add unit test cases on data type handling in MySQL (#16621)

* chore: Move MySQL specific types to a separate class (#16621)

* chore: Remove import shortening (#16621)

* Fix testStructure test case to have loose coupling with the order of tables (#16621)

* Fix: Add missing client-side data types in params in a few test cases (#16621)

* Fix query in test case (#16621)

* Fix test cases and add small refactoring (#16621)

* Refactor assertion with a check on Optional (#16621)

* additional cypress test cases for mysql

* updated test case object

* update for failing test cases

* mysql failure point fix

* mysql false spec failure point fix

* fix flacy test cases

* flacky JSON test cases

Co-authored-by: ChandanBalajiBP <chandan@appsmith.com>
Co-authored-by: Aishwarya UR <aishwarya@appsmith.com>
This commit is contained in:
subratadeypappu 2022-10-04 16:45:10 +06:00 committed by GitHub
parent e6b89d03aa
commit d2de8f7cec
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 1675 additions and 1226 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

73
app/client/cypress/integration/Smoke_TestSuite/ServerSideTests/Datatypes/MySQL_Spec.ts
View File

@ -36,7 +36,9 @@ describe("MySQL Datatype tests", function() {
dataSources.RunQuery();
ee.ActionContextMenuByEntityName(dsName, "Refresh");
agHelper.AssertElementVisible(ee._entityNameInExplorer(inputData.tableName));
agHelper.AssertElementVisible(
ee._entityNameInExplorer(inputData.tableName),
);
});
it("3. Creating SELECT query", () => {
@ -54,32 +56,14 @@ describe("MySQL Datatype tests", function() {
agHelper.RenameWithInPane("insertRecord");
dataSources.EnterQuery(query);
query = inputData.query.deleteAllRecords;
ee.ActionTemplateMenuByEntityName(inputData.tableName, "DELETE");
agHelper.RenameWithInPane("deleteAllRecords");
dataSources.EnterQuery(query);
query = inputData.query.dropTable;
ee.ActionTemplateMenuByEntityName(inputData.tableName, "DELETE");
agHelper.RenameWithInPane("dropTable");
dataSources.EnterQuery(query);
});
//Insert false values to each column and check for the error status of the request.
it("5. False Cases", () => {
ee.ActionTemplateMenuByEntityName(inputData.tableName, "INSERT");
agHelper.RenameWithInPane("falseCases");
inputData.falseResult.forEach((res_array, i) => {
res_array.forEach((value) => {
query = `INSERT INTO ${inputData.tableName} (${inputData.inputFieldName[i]}) VALUES (${value})`;
dataSources.EnterQuery(query);
dataSources.RunQuery(false);
});
});
});
//Insert valid/true values into datasource
it("6. Inserting record", () => {
it("5. Inserting record", () => {
ee.SelectEntityByName("Page1");
deployMode.DeployApp();
table.WaitForTableEmpty(); //asserting table is empty before inserting!
@ -87,7 +71,8 @@ describe("MySQL Datatype tests", function() {
inputData.input.forEach((valueArr, i) => {
agHelper.ClickButton("Run InsertQuery");
valueArr.forEach((value, index) => {
agHelper.EnterInputText(inputData.inputFieldName[index], value);
if (value !== "")
agHelper.EnterInputText(inputData.inputFieldName[index], value);
});
i % 2 && agHelper.ToggleSwitch("Bool_column");
agHelper.ClickButton("insertRecord");
@ -98,26 +83,41 @@ describe("MySQL Datatype tests", function() {
//Verify weather expected value is present in each cell
//i.e. weather right data is pushed and fetched from datasource.
it("7. Validating values in each cell", () => {
it("6. Validating values in each cell", () => {
cy.wait(2000);
inputData.result.forEach((res_array, i) => {
res_array.forEach((value, j) => {
table.ReadTableRowColumnData(j, i, 0).then(($cellData) => {
expect($cellData).to.eq(value);
if(i === inputData.result.length-1){
let obj = JSON.parse($cellData)
expect(JSON.stringify(obj)).to.eq(JSON.stringify(value));
}else{
expect($cellData).to.eq(value);
}
});
});
});
});
it("8. Deleting all records from table ", () => {
agHelper.GetNClick(locator._deleteIcon);
agHelper.Sleep(2000);
table.WaitForTableEmpty();
});
it("9. Validate Drop of the Newly Created - mysqlDTs - Table from MySQL datasource", () => {
//null will be displayed as empty string in tables
//So test null we have to intercept execute request.
//And check response payload.
it("7. Testing null value", () => {
deployMode.NavigateBacktoEditor();
ee.ExpandCollapseEntity("Queries/JS");
ee.SelectEntityByName("selectRecords");
dataSources.RunQuery(true, false);
cy.wait("@postExecute").then((intercept) => {
expect(
typeof intercept.response?.body.data.body[5].varchar_column,
).to.be.equal("object");
expect(intercept.response?.body.data.body[5].varchar_column).to.be.equal(
null,
);
});
});
it("8. Validate drop of mysqlDTs - Table from MySQL datasource", () => {
ee.SelectEntityByName("dropTable");
dataSources.RunQuery();
dataSources.ReadQueryTableResponse(0).then(($cellData) => {
@ -134,15 +134,20 @@ describe("MySQL Datatype tests", function() {
ee.ExpandCollapseEntity("Datasources", false);
});
it("10. Verify Deletion of the datasource after all created queries are Deleted", () => {
it("9. Verify Deletion of the datasource after all created queries are Deleted", () => {
dataSources.DeleteDatasouceFromWinthinDS(dsName, 409); //Since all queries exists
ee.ExpandCollapseEntity("Queries/JS");
["falseCases", "createTable", "deleteAllRecords", "dropTable", "insertRecord", "selectRecords"].forEach(type => {
[
"createTable",
"dropTable",
"insertRecord",
"selectRecords",
].forEach((type) => {
ee.ActionContextMenuByEntityName(type, "Delete", "Are you sure?");
})
});
deployMode.DeployApp();
deployMode.NavigateBacktoEditor();
ee.ExpandCollapseEntity("Queries/JS");
dataSources.DeleteDatasouceFromWinthinDS(dsName, 200);
dataSources.DeleteDatasouceFromWinthinDS(dsName, 200);
});
});

79
app/client/cypress/integration/Smoke_TestSuite/ServerSideTests/Datatypes/MySQL_false_Spec.ts Normal file
View File

@ -0,0 +1,79 @@
import { ObjectsRegistry } from "../../../../support/Objects/Registry";
import inputData from "../../../../support/Objects/mySqlData";
let dsName: any, query: string;
const agHelper = ObjectsRegistry.AggregateHelper,
ee = ObjectsRegistry.EntityExplorer,
dataSources = ObjectsRegistry.DataSources,
propPane = ObjectsRegistry.PropertyPane,
table = ObjectsRegistry.Table,
locator = ObjectsRegistry.CommonLocators,
deployMode = ObjectsRegistry.DeployMode;
describe("MySQL Datatype tests", function() {
it("1. Create Mysql DS", function() {
dataSources.CreateDataSource("MySql");
cy.get("@dsName").then(($dsName) => {
dsName = $dsName;
});
});
it("2. Creating mysqlDTs table", () => {
//IF NOT EXISTS can be used - which creates tabel if it does not exist and donot throw any error if table exists.
//But if we add this option then next case could fail inn that case.
query = inputData.query.createTable;
ee.CreateNewDsQuery(dsName);
agHelper.RenameWithInPane("createTable");
agHelper.GetNClick(dataSources._templateMenu);
dataSources.EnterQuery(query);
dataSources.RunQuery();
ee.ActionContextMenuByEntityName(dsName, "Refresh");
agHelper.AssertElementVisible(
ee._entityNameInExplorer(inputData.tableName),
);
});
//Insert false values to each column and check for the error status of the request.
it("3. False Cases", () => {
ee.ActionTemplateMenuByEntityName(inputData.tableName, "INSERT");
agHelper.RenameWithInPane("falseCases");
inputData.falseResult.forEach((res_array, i) => {
res_array.forEach((value) => {
query =
typeof value === "string"
? `INSERT INTO ${inputData.tableName} (${inputData.inputFieldName[i]}) VALUES ({{"${value}"}})`
: `INSERT INTO ${inputData.tableName} (${inputData.inputFieldName[i]}) VALUES ({{${value}}})`;
dataSources.EnterQuery(query);
dataSources.RunQuery(false);
});
});
agHelper.Sleep(2000);
agHelper.WaitUntilAllToastsDisappear();
});
//This is a special case.
//Added due to server side checks, which was handled in Datatype handling.
it("4. Long Integer as query param", () => {
query = `SELECT * FROM ${inputData.tableName} LIMIT {{2147483648}}`;
dataSources.EnterQuery(query);
dataSources.RunQuery();
});
it("5. Drop Table", () => {
query = inputData.query.dropTable;
dataSources.EnterQuery(query);
dataSources.RunQuery();
});
it("6. Verify Deletion of the datasource after all created queries are Deleted", () => {
ee.ExpandCollapseEntity("Queries/JS");
["falseCases", "createTable"].forEach((type) => {
ee.ActionContextMenuByEntityName(type, "Delete", "Are you sure?");
});
deployMode.DeployApp();
deployMode.NavigateBacktoEditor();
ee.ExpandCollapseEntity("Queries/JS");
dataSources.DeleteDatasouceFromWinthinDS(dsName, 200);
});
});

106
app/client/cypress/support/Objects/mySqlData.ts
View File

@ -1,6 +1,6 @@
const mySqlData = {
tableName: "mysqlDTs",
inputFieldName : [
inputFieldName: [
"Stinyint_column",
"Utinyint_column",
"Ssmallint_column",
@ -23,7 +23,7 @@ const mySqlData = {
"Enum_column",
"Json_column",
],
input : [
input: [
[
"-128",
"0",
@ -88,11 +88,34 @@ const mySqlData = {
"2012/12/31",
"-838:59:59",
"1901'",
"null",
"null",
"12345678912345",
"012345",
"c",
"[1, 2, 3, 4]",
],
[
"0",
"0",
"0",
"0",
"0",
"0",
"0123",
"0",
"0",
"0",
"0",
"0",
"20121231113045",
"121231113045",
"121231",
"11:12",
"2155",
"true",
"false",
"c",
"[]",
],
[
"0",
"0",
@ -109,33 +132,10 @@ const mySqlData = {
"20121231113045",
"121231113045",
"121231",
"11:12",
"2155",
"null",
"null",
"c",
"[]",
],
[
"0",
"0",
"0",
"0",
"0",
"0",
"0",
"0",
"0",
"0",
"0",
"null",
"121231113045",
"null",
"null",
"1112",
"2022",
"null",
"null",
"null",
"NulL",
"c",
'["a",true,0,12.34]',
],
@ -151,19 +151,19 @@ const mySqlData = {
"0",
"0",
"0",
"null",
"null",
"null",
"null",
"0",
"20121231113045",
"121231113045",
"121231",
"12",
"null",
"null",
"null",
"2022",
"",
"abc",
"c",
"null",
'{}',
],
],
falseResult : [
falseResult: [
[-129, 128],
[-1, 256],
[-32769, 32768],
@ -172,9 +172,9 @@ const mySqlData = {
[-1, 16777216],
[-2147483649, 2147483648],
[-1, 4294967296],
[-9223372036854775808, 9223372036854775807],
[],
[123456789.45],
['a'],
["a"],
[123456789.45],
["2012/12/31 11:30:451"],
["2012/12/311 11:30:45"],
@ -184,9 +184,9 @@ const mySqlData = {
["abcdefghijklmnopqrstu"],
["abcdefghijk"],
["d"],
["abc", "{", "["],
[],
],
result : [
result: [
["1", "2", "3", "4", "5"],
["-128", "0", "127"],
["0", "255"],
@ -194,7 +194,7 @@ const mySqlData = {
["0", "65535"],
["-8388608", "0", "8388607"],
["0", "16777215"],
["-2147483648", "0", "2147483647"],
["-2147483648", "0", "2147483647", "123"],
["0", "4294967295"],
["123456", "456789", "123456789"],
["123.45", "123.46", "123.45"],
@ -216,30 +216,28 @@ const mySqlData = {
["2012-12-31", "2012-12-31", "2012-12-31", "2012-12-31"],
["22:59:59", "00:00:00", "01:00:01", "11:12:00", "00:11:12", "00:00:12"],
["1901", "2155", "1901", "2155"],
["a", "abcdefghijklmnopqrst"],
["a", "abcdefghij"],
["a", "abcdefghijklmnopqrst", "12345678912345", "true", "null"],
["a", "abcdefghij", "012345", "false", "NulL"],
["a", "b", "c"],
["0", "1"],
['{"abc": "123"}', "{}", "[1, 2, 3, 4]", "", '["a",true,0,12.34]'],
[{"abc": "123"}, {}, [1, 2, 3, 4], [], ["a",true,0,12.34]],
],
query :{
createTable : `CREATE TABLE mysqlDTs (serialId SERIAL not null primary key, stinyint_column TINYINT, utinyint_column TINYINT UNSIGNED,
query: {
createTable: `CREATE TABLE mysqlDTs (serialId SERIAL not null primary key, stinyint_column TINYINT, utinyint_column TINYINT UNSIGNED,
ssmallint_column SMALLINT, usmallint_column SMALLINT UNSIGNED, smediumint_column MEDIUMINT, umediumint_column MEDIUMINT UNSIGNED,
sint_column INT, uint_column INT UNSIGNED, bigint_column BIGINT, float_column FLOAT( 10, 2 ), double_column DOUBLE, decimal_column DECIMAL( 10, 2 ),
datetime_column DATETIME, timestamp_column TIMESTAMP, date_column DATE, time_column TIME, year_column YEAR, varchar_column VARCHAR( 20 ),
char_column CHAR( 10 ), enum_column ENUM( 'a', 'b', 'c' ), bool_column BOOL, json_column JSON);`,
insertRecord : `INSERT INTO mysqlDTs (stinyint_column, utinyint_column, ssmallint_column, usmallint_column, smediumint_column, umediumint_column,
insertRecord: `INSERT INTO mysqlDTs (stinyint_column, utinyint_column, ssmallint_column, usmallint_column, smediumint_column, umediumint_column,
sint_column, uint_column, bigint_column, float_column, double_column, decimal_column, datetime_column, timestamp_column,
date_column, time_column, year_column, varchar_column, char_column, enum_column, bool_column, json_column )
VALUES
({{InsertStinyint.text}}, {{InsertUtinyint.text}}, {{InsertSsmallint.text}}, {{InsertUsmallint.text}}, {{InsertSmediumint.text}},
{{InsertUmediumint.text}}, {{InsertSint.text}}, {{InsertUint.text}}, {{InsertBigint.text}}, {{InsertFloat.text}}, {{InsertDouble.text}},
{{InsertDecimal.text}}, {{InsertDatetime.text}}, {{InsertTimestamp.text}}, {{InsertDate.text}}, {{InsertTime.text}},
{{InsertYear.text}}, {{InsertVarchar.text}}, {{InsertChar.text}}, {{InsertEnum.text}}, {{InsertBoolean.isSwitchedOn}}, {{InputJson.text}});`,
deleteRecord : `DELETE FROM mysqlDTs WHERE serialId ={{Table1.selectedRow.serialid}}`,
deleteAllRecords : `DELETE FROM mysqlDTs`,
{{InsertYear.text}}, {{InsertVarchar.text ? InsertVarchar.text : null}}, {{InsertChar.text}}, {{InsertEnum.text}}, {{InsertBoolean.isSwitchedOn}}, {{InputJson.text}});`,
dropTable: `drop table mysqlDTs`,
}
},
};
export default mySqlData;
export default mySqlData;

10
app/server/appsmith-interfaces/src/main/java/com/appsmith/external/plugins/SmartSubstitutionInterface.java vendored
View File

@ -5,6 +5,7 @@ import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginError;
import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginException;
import com.appsmith.external.models.Param;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Optional;
@ -39,7 +40,7 @@ public interface SmartSubstitutionInterface {
String value = matchingParam.get().getValue();
input = substituteValueInInput(i + 1, key,
value, input, insertedParams, args);
value, input, insertedParams, append(args, matchingParam.get().getClientDataType()));
} else {
throw new AppsmithPluginException(AppsmithPluginError.PLUGIN_ERROR, "Uh oh! This is unexpected. " +
"Did not receive any information for the binding "
@ -71,4 +72,11 @@ public interface SmartSubstitutionInterface {
default String sanitizeReplacement(String replacementValue, DataType dataType) {
return replacementValue;
}
static <T> T[] append(T[] arr, T lastElement) {
final int N = arr.length;
arr = Arrays.copyOf(arr, N+1);
arr[N] = lastElement;
return arr;
}
}

53
app/server/appsmith-plugins/mysqlPlugin/src/main/java/com/external/plugins/MySqlPlugin.java vendored
View File

@ -1,11 +1,12 @@
package com.external.plugins;
import com.appsmith.external.constants.DataType;
import com.appsmith.external.datatypes.AppsmithType;
import com.appsmith.external.datatypes.ClientDataType;
import com.appsmith.external.dtos.ExecuteActionDTO;
import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginError;
import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginException;
import com.appsmith.external.exceptions.pluginExceptions.StaleConnectionException;
import com.appsmith.external.helpers.DataTypeStringUtils;
import com.appsmith.external.helpers.DataTypeServiceUtils;
import com.appsmith.external.helpers.MustacheHelper;
import com.appsmith.external.models.ActionConfiguration;
import com.appsmith.external.models.ActionExecutionRequest;
@ -22,6 +23,7 @@ import com.appsmith.external.models.SSLDetails;
import com.appsmith.external.plugins.BasePlugin;
import com.appsmith.external.plugins.PluginExecutor;
import com.appsmith.external.plugins.SmartSubstitutionInterface;
import com.external.plugins.datatypes.MySQLSpecificDataTypes;
import com.external.utils.QueryUtils;
import io.r2dbc.spi.ColumnMetadata;
import io.r2dbc.spi.Connection;
@ -340,7 +342,7 @@ public class MySqlPlugin extends BasePlugin {
}
private boolean isIsOperatorUsed(String query) {
boolean isIsOperatorUsed(String query) {
String queryKeyWordsOnly = query.replaceAll(MATCH_QUOTED_WORDS_REGEX, "");
return Arrays.stream(queryKeyWordsOnly.split("\\s"))
.anyMatch(word -> IS_KEY.equalsIgnoreCase(word.trim()));
@ -394,29 +396,36 @@ public class MySqlPlugin extends BasePlugin {
Object... args) {
Statement connectionStatement = (Statement) input;
DataType valueType = DataTypeStringUtils.stringToKnownDataTypeConverter(value);
ClientDataType clientDataType = (ClientDataType) args[0];
AppsmithType appsmithType = DataTypeServiceUtils.getAppsmithType(clientDataType, value, MySQLSpecificDataTypes.pluginSpecificTypes);
Map.Entry<String, String> parameter = new SimpleEntry<>(value, valueType.toString());
Map.Entry<String, String> parameter = new SimpleEntry<>(value, appsmithType.type().toString());
insertedParams.add(parameter);
if (DataType.NULL.equals(valueType)) {
try {
connectionStatement.bindNull((index - 1), Object.class);
} catch (UnsupportedOperationException e) {
// Do nothing. Move on
}
} else if (DataType.INTEGER.equals(valueType)) {
/**
* - NumberFormatException is NOT expected here since stringToKnownDataTypeConverter uses parseInt
* method to detect INTEGER type.
*/
connectionStatement.bind((index - 1), Integer.parseInt(value));
} else if (DataType.BOOLEAN.equals(valueType)) {
connectionStatement.bind((index - 1), Boolean.parseBoolean(value) == TRUE ? 1 : 0);
} else {
connectionStatement.bind((index - 1), value);
switch (appsmithType.type()) {
case NULL:
try {
connectionStatement.bindNull((index - 1), Object.class);
} catch (UnsupportedOperationException e) {
// Do nothing. Move on
}
break;
case BOOLEAN:
connectionStatement.bind((index - 1), appsmithType.performSmartSubstitution(value));
break;
case INTEGER:
connectionStatement.bind((index - 1), Integer.parseInt(value));
break;
case LONG:
connectionStatement.bind((index - 1), Long.parseLong(value));
break;
case DOUBLE:
connectionStatement.bind((index - 1), Double.parseDouble(value));
break;
default:
connectionStatement.bind((index - 1), value);
break;
}
return connectionStatement;
}

44
app/server/appsmith-plugins/mysqlPlugin/src/main/java/com/external/plugins/datatypes/MySQLSpecificDataTypes.java vendored Normal file
View File

@ -0,0 +1,44 @@
package com.external.plugins.datatypes;
import com.appsmith.external.datatypes.AppsmithType;
import com.appsmith.external.datatypes.BigDecimalType;
import com.appsmith.external.datatypes.ClientDataType;
import com.appsmith.external.datatypes.DoubleType;
import com.appsmith.external.datatypes.IntegerType;
import com.appsmith.external.datatypes.JsonObjectType;
import com.appsmith.external.datatypes.LongType;
import com.appsmith.external.datatypes.NullType;
import com.appsmith.external.datatypes.StringType;
import com.appsmith.external.datatypes.TimeType;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class MySQLSpecificDataTypes {
public static final Map<ClientDataType, List<AppsmithType>> pluginSpecificTypes;
static {
pluginSpecificTypes = new HashMap<>();
pluginSpecificTypes.put(ClientDataType.NULL, List.of(new NullType()));
pluginSpecificTypes.put(ClientDataType.BOOLEAN, List.of(new MySQLBooleanType()));
pluginSpecificTypes.put(ClientDataType.NUMBER, List.of(
new IntegerType(),
new LongType(),
new DoubleType(),
new BigDecimalType()
));
pluginSpecificTypes.put(ClientDataType.OBJECT, List.of(new JsonObjectType()));
pluginSpecificTypes.put(ClientDataType.STRING, List.of(
new TimeType(),
new MySQLDateType(),
new MySQLDateTimeType(),
new StringType()
));
}
}

2536
app/server/appsmith-plugins/mysqlPlugin/src/test/java/com/external/plugins/MySqlPluginTest.java vendored
View File

File diff suppressed because it is too large Load Diff