Protei/PromucFlow_constructor
2
0
Fork 0
Code Issues Pull Requests Actions 4 Packages Projects Releases Wiki Activity

Remove autogenerate column from insert query for generate CRUD page from DB table (#6183)

Browse Source 
* Removed autogenerated column reference from InsertQuery

* Disable page generation flow for layout with widgets

* Updated Template

* TCs modify for updated template application

* Allow column with only String datatype as a search column in SelectQuery

* Remove AtomicRef as not needed

Co-authored-by: Nikhil Nandagopal <nikhil.nandagopal@gmail.com>
This commit is contained in:
Abhijeet 2021-07-28 13:14:04 +05:30 committed by GitHub
parent 2b60519b7c
commit c8928d34ed
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 127 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
app/server/appsmith-server/src/main/java/com/appsmith/server/constants/FieldName.java
View File

@ -78,4 +78,8 @@ public class FieldName {
public static String MONGO_UNESCAPED_ID = "_id";
public static String MONGO_UNESCAPED_CLASS = "_class";
public static String DATASOURCE_STRUCTURE = "datasource structure";
public static final String OBJECT_ID = "ObjectId";
public static final String PLACEHOLDER_TEXT = "placeholderText";
public static final String IS_DISABLED = "isDisabled";
public static final String IS_REQUIRED = "isRequired";
}

1
app/server/appsmith-server/src/main/java/com/appsmith/server/exceptions/AppsmithError.java
View File

@ -63,6 +63,7 @@ public enum AppsmithError {
VALIDATION_FAILURE(400, 4028, "Validation Failure(s): {0}", AppsmithErrorAction.DEFAULT, null),
INVALID_CURL_COMMAND(400, 4029, "Invalid cURL command, couldn't import.", AppsmithErrorAction.DEFAULT, null),
REMOVE_LAST_ORG_ADMIN_ERROR(400, 4037, "The last admin can not be removed from an organization", AppsmithErrorAction.DEFAULT, null),
INVALID_CRUD_PAGE_REQUEST(400, 4038, "Unable to process page generation request, {0}", AppsmithErrorAction.DEFAULT, null),
INTERNAL_SERVER_ERROR(500, 5000, "Internal server error while processing request", AppsmithErrorAction.LOG_EXTERNALLY, null),
REPOSITORY_SAVE_FAILED(500, 5001, "Failed to save the repository. Try again.", AppsmithErrorAction.DEFAULT, null),
PLUGIN_INSTALLATION_FAILED_DOWNLOAD_ERROR(500, 5002, "Plugin installation failed due to an error while " +

152
app/server/appsmith-server/src/main/java/com/appsmith/server/solutions/CreateDBTablePageSolution.java
View File

@ -30,8 +30,6 @@ import com.appsmith.server.services.LayoutActionService;
import com.appsmith.server.services.NewPageService;
import com.appsmith.server.services.PluginService;
import com.appsmith.server.services.SessionUserService;
import com.google.common.collect.Maps;
import com.google.common.collect.Streams;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import lombok.RequiredArgsConstructor;
@ -53,6 +51,7 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.atomic.AtomicReference;
import java.util.regex.Matcher;
@ -87,6 +86,10 @@ public class CreateDBTablePageSolution {
private static final String LIST_QUERY = "ListFiles";
private static final String INSERT_QUERY = "InsertQuery";
private static final String TEMPLATE_AUTOGENERATED_INSERT_COLUMN = "insert_col_input1";
// Default SelectWidget dropdown value for SQL and Postgres template pages which will be used in select query for sort operator
private static final String SQL_DEFAULT_DROPDOWN_VALUE = "asc";
@ -95,7 +98,7 @@ public class CreateDBTablePageSolution {
// This column will be used to map filter in Find and Select query. This particular field is added to have
// uniformity across different datasources
private static final String DEFAULT_SEARCH_COLUMN = "col3";
private static final String DEFAULT_SEARCH_COLUMN = "col2";
private static final long MIN_TABLE_COLUMNS = 2;
@ -105,6 +108,10 @@ public class CreateDBTablePageSolution {
"defaultText", "placeholderText", "text", "options", "defaultOptionValue", "primaryColumns", "isVisible"
);
// Currently we only support string matching (like/ilike etc) for WHERE operator in SelectQuery so the allowed
// types will refer to the equivalent datatype in different databases
private static final Set<String> ALLOWED_TYPE_FOR_WHERE_CLAUSE = Set.of("string", "text", "varchar", "char", "character");
// Pattern to match all words in the text
private static final Pattern WORD_PATTERN = Pattern.compile("\\w+");
@ -134,7 +141,7 @@ public class CreateDBTablePageSolution {
"redshift-plugin",
"snowflake-plugin"
);
StringBuffer templateAutogeneratedKey = new StringBuffer();
AtomicReference<String> savedPageId = new AtomicReference<>(pageId);
if (pageResourceDTO.getTableName() == null || pageResourceDTO.getDatasourceId() == null) {
return Mono.error(new AppsmithException(AppsmithError.INVALID_PARAMETER, ", tableName and datasourceId must be present"));
@ -254,7 +261,7 @@ public class CreateDBTablePageSolution {
new AppsmithException(AppsmithError.NO_RESOURCE_FOUND, FieldName.DATASOURCE_STRUCTURE, datasource.getName())
);
}
mappedColumnsAndTableName.putAll(mapTableColumnNames(templateTable, table, searchColumn, columns));
mappedColumnsAndTableName.putAll(mapTableColumnNames(templateTable, table, searchColumn, columns, templateAutogeneratedKey));
} else {
int colCount = 1;
// If the structure and tables not present in the template datasource then map the columns from
@ -282,7 +289,7 @@ public class CreateDBTablePageSolution {
Set<String> deletedWidgets = new HashSet<>();
layout.setDsl(
extractAndUpdateAllWidgetFromDSL(layout.getDsl(), mappedColumnsAndTableName, deletedWidgets)
extractAndUpdateAllWidgetFromDSL(layout.getDsl(), mappedColumnsAndTableName, deletedWidgets, templateAutogeneratedKey.toString())
);
List<NewAction> templateActionList = applicationJson.getActionList()
@ -325,7 +332,8 @@ public class CreateDBTablePageSolution {
templateActionList,
mappedColumnsAndTableName,
deletedWidgets,
pluginSpecificParams)
pluginSpecificParams,
templateAutogeneratedKey.toString())
.flatMap(actionDTO -> StringUtils.equals(actionDTO.getName(), SELECT_QUERY)
|| StringUtils.equals(actionDTO.getName(), FIND_QUERY)
@ -357,7 +365,14 @@ public class CreateDBTablePageSolution {
return newPageService.findById(pageId, AclPermission.MANAGE_PAGES)
.switchIfEmpty(Mono.error(new AppsmithException(
AppsmithError.ACL_NO_RESOURCE_FOUND, FieldName.PAGE, pageId))
);
)
.map(newPage -> {
Layout layout = newPage.getUnpublishedPage().getLayouts().get(0);
if (layout.getWidgetNames().size() > 1) {
throw new AppsmithException(AppsmithError.INVALID_CRUD_PAGE_REQUEST, "please try on empty layout");
}
return newPage;
});
}
return newPageService.findByApplicationId(applicationId, AclPermission.MANAGE_PAGES, false)
@ -449,8 +464,8 @@ public class CreateDBTablePageSolution {
List<NewAction> templateActionList,
Map<String, String> mappedColumns,
Set<String> deletedWidgetNames,
Map<String, String> pluginSpecificTemplateParams
) {
Map<String, String> pluginSpecificTemplateParams,
final String templateAutogeneratedKey) {
/*
1. Clone actions from the template pages
2. Update actionConfiguration to replace the template table fields with users datasource fields
@ -474,11 +489,22 @@ public class CreateDBTablePageSolution {
List<Property> pluginSpecifiedTemplates = actionConfiguration.getPluginSpecifiedTemplates();
if (actionBody != null) {
// If the primary key is autogenerated remove primaryKey's reference from InsertQuery
final String temp = mappedColumns.get(templateAutogeneratedKey);
if (!templateAutogeneratedKey.isEmpty() && INSERT_QUERY.equals(actionDTO.getName())) {
mappedColumns.put(templateAutogeneratedKey, DELETE_FIELD);
}
String body = actionBody.replaceFirst(TEMPLATE_TABLE_NAME, tableName);
final Matcher matcher = WORD_PATTERN.matcher(body);
actionConfiguration.setBody(matcher.replaceAll(key ->
mappedColumns.get(key.group()) == null ? key.group() : mappedColumns.get(key.group()))
);
// Reassign the previous column mapping
if (!templateAutogeneratedKey.isEmpty() && INSERT_QUERY.equals(actionDTO.getName())) {
mappedColumns.put(templateAutogeneratedKey, temp);
}
}
log.debug("Cloning plugin specified templates for action {}", actionDTO.getId());
@ -518,7 +544,8 @@ public class CreateDBTablePageSolution {
private Map<String, String> mapTableColumnNames(Table sourceTable,
Table destTable,
final String searchColumn,
Set<String> tableColumns) {
Set<String> tableColumns,
StringBuffer templateAutogeneratedKey) {
/*
1. Fetch and map primary keys for source and destination columns if available
@ -527,24 +554,37 @@ public class CreateDBTablePageSolution {
log.debug("Mapping column names with template application for table {}", destTable.getName());
Map<String, String> mappedTableColumns = new HashMap<>();
// Assign string type column as default search column. We are only supporting String matching for filter clause
if (searchColumn != null && !searchColumn.isEmpty()) {
mappedTableColumns.put(DEFAULT_SEARCH_COLUMN, searchColumn);
sourceTable.getColumns().removeIf(column -> DEFAULT_SEARCH_COLUMN.equals(column.getName()));
destTable.getColumns().removeIf(column -> searchColumn.equals(column.getName()));
} else {
final String tempSearchColumn = destTable.getColumns()
.stream()
.filter(column -> ALLOWED_TYPE_FOR_WHERE_CLAUSE
.contains(column.getType().replaceAll("[^A-Za-z]+", "").toLowerCase())
)
.findAny()
.map(Column::getName)
.orElse(null);
mappedTableColumns.put(DEFAULT_SEARCH_COLUMN, Objects.requireNonNullElse(tempSearchColumn, DELETE_FIELD));
}
mappedTableColumns.putAll(mapKeys(sourceTable, destTable));
mappedTableColumns.putAll(mapKeys(sourceTable, destTable, templateAutogeneratedKey));
List<Column> sourceTableColumns = sourceTable.getColumns(), destTableColumns = destTable.getColumns();
if (!CollectionUtils.isEmpty(tableColumns) && tableColumns.size() > MIN_TABLE_COLUMNS) {
destTableColumns = destTableColumns.stream()
.filter(column -> tableColumns.contains(column.getName()))
.collect(Collectors.toList());
}
// Remove column names which are already mapped
sourceTableColumns = sourceTableColumns.stream()
.filter(key -> !mappedTableColumns.containsKey(key.toString()))
.filter(key -> !mappedTableColumns.containsKey(key.getName()))
.collect(Collectors.toList());
destTableColumns = destTableColumns.stream()
.filter(key -> !mappedTableColumns.containsValue(key.toString()))
.filter(key -> !mappedTableColumns.containsValue(key.getName()))
.collect(Collectors.toList());
int idx = 0;
@ -572,32 +612,64 @@ public class CreateDBTablePageSolution {
* @return Map of <sourceKeyColumnName, destinationKeyColumnName>
*/
private Map<String, String> mapKeys(Table sourceTable, Table destTable) {
private Map<String, String> mapKeys(Table sourceTable, Table destTable, StringBuffer templateAutogeneratedKey) {
/*
1. Get pKey for source table and destination table
2. Map column names from source pKey to destination pKey
*/
Map<String, String> primaryKeyNameMap = new HashMap<>();
List<String> sourceKeys = new ArrayList<>();
List<String> destKeys = new ArrayList<>();
if (CollectionUtils.isEmpty(sourceTable.getKeys()) || CollectionUtils.isEmpty(destTable.getKeys())) {
return primaryKeyNameMap;
}
// Map keys for SQL datasources
if (DatasourceStructure.TableType.TABLE.equals(sourceTable.getType())) {
sourceKeys.add("col1");
} else if (DatasourceStructure.TableType.COLLECTION.equals(sourceTable.getType())) {
sourceKeys.add("_id");
}
destTable.getKeys().forEach(key -> {
if (key instanceof PrimaryKey) {
PrimaryKey pKey = (PrimaryKey) key;
destKeys.addAll(pKey.getColumnNames());
final String sourceKey = "col1";
DatasourceStructure.Key primaryKey = destTable.getKeys()
.stream()
.filter(key -> key instanceof PrimaryKey)
.findAny()
.orElse(null);
if (primaryKey != null) {
PrimaryKey key = (PrimaryKey) primaryKey;
final String destKey = key.getColumnNames().get(0);
primaryKeyNameMap.put(sourceKey, destKey);
// Check if the destKey is autogenerated, and save source column name which will be used to remove reference
// from InsertQuery otherwise InserQuery will fail with error : Trying to insert autogenerated field
templateAutogeneratedKey.append(destTable.getColumns()
.stream()
.filter(column -> column.getIsAutogenerated() != null && column.getIsAutogenerated() && destKey.equals(column.getName()))
.findAny()
.map(column -> sourceKey)
.orElse(""));
}
});
return Streams.zip(sourceKeys.stream(), destKeys.stream(), Maps::immutableEntry)
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
}
// Map keys for NoSQL datasources like MongoDB
else if (DatasourceStructure.TableType.COLLECTION.equals(sourceTable.getType())) {
final String sourceKey = FieldName.MONGO_UNESCAPED_ID;
List<Column> autogeneratedColumns = destTable.getColumns()
.stream()
// This makes sure only keys with instance of objectId will be considered as we don't have definitive
// structure like primaryKey in SQL for mongoDB. We can safely assume that these field will act as a
// unique key which will then be used for update query
.filter(column -> FieldName.OBJECT_ID.equals(column.getType()))
.collect(Collectors.toList());
final String destKey = autogeneratedColumns
.stream()
.filter(column -> sourceKey.equals(column.getName()))
.findAny()
.map(Column::getName)
.orElse(null);
// If column name "_id" is present in user's collection then map template datasource key "_id" with column name ,
// else map template datasource key "_id" to any autogenerated field from user's collection
if (destKey != null) {
primaryKeyNameMap.put(sourceKey, destKey);
} else if (!CollectionUtils.isEmpty(autogeneratedColumns)) {
primaryKeyNameMap.put(sourceKey, autogeneratedColumns.get(0).getName());
}
}
return primaryKeyNameMap;
}
/**
@ -609,7 +681,8 @@ public class CreateDBTablePageSolution {
*/
private JSONObject extractAndUpdateAllWidgetFromDSL(JSONObject dsl,
Map<String, String> mappedColumnsAndTableNames,
Set<String> deletedWidgets) {
Set<String> deletedWidgets,
final String templateAutogeneratedKey) {
/*
1. Update dsl : Replace names of template columns with the user connected datasource columns
@ -640,14 +713,25 @@ public class CreateDBTablePageSolution {
if (!CollectionUtils.isEmpty(data)) {
object.putAll(data);
JSONObject child =
extractAndUpdateAllWidgetFromDSL(object, mappedColumnsAndTableNames, deletedWidgets);
extractAndUpdateAllWidgetFromDSL(object, mappedColumnsAndTableNames, deletedWidgets, templateAutogeneratedKey);
String widgetType = child.getAsString(FieldName.WIDGET_TYPE);
String widgetName = child.getAsString(FieldName.WIDGET_NAME);
if (FieldName.TABLE_WIDGET.equals(widgetType)
|| FieldName.CONTAINER_WIDGET.equals(widgetType)
|| FieldName.CANVAS_WIDGET.equals(widgetType)
|| FieldName.FORM_WIDGET.equals(widgetType)
|| !child.toString().contains(DELETE_FIELD)
) {
// Update widget for the field which is autogenerated as per DB structure from InsertQuery
if (!templateAutogeneratedKey.isEmpty() && TEMPLATE_AUTOGENERATED_INSERT_COLUMN.equals(widgetName)) {
// Give clear message to user that it's autogenerated column using disabled status and
// placeholder. Add this widget in deletedWidget list as we want to delete the widget
// reference from InsertQuery.
child.put(FieldName.PLACEHOLDER_TEXT, "Autogenerated Field");
child.put(FieldName.IS_DISABLED, true);
child.put(FieldName.IS_REQUIRED, false);
deletedWidgets.add(child.getAsString(FieldName.WIDGET_NAME));
}
newChildren.add(child);
} else {
deletedWidgets.add(child.getAsString(FieldName.WIDGET_NAME));

2
app/server/appsmith-server/src/main/resources/CRUD-DB-Table-Template-Application.json
View File

File diff suppressed because one or more lines are too long

8
app/server/appsmith-server/src/test/java/com/appsmith/server/solutions/CreateDBTablePageSolutionTests.java
View File

@ -98,14 +98,12 @@ public class CreateDBTablePageSolutionTests {
" WHERE \"primaryKey\" = {{Table1.selectedRow.primaryKey}};",
"InsertQuery", "INSERT INTO sampleTable (\n" +
"\t\"primaryKey\",\n" +
"\t\"field1\", \n" +
"\t\"field2\",\n" +
"\t\"field3\", \n" +
"\t\"field4\"\n" +
")\n" +
"VALUES (\n" +
"\t\t\t\t{{insert_col_input1.text}}, \n" +
"\t\t\t\t{{insert_col_input2.text}}, \n" +
"\t\t\t\t{{insert_col_input3.text}}, \n" +
"\t\t\t\t{{insert_col_input4.text}}, \n" +
@ -113,7 +111,7 @@ public class CreateDBTablePageSolutionTests {
");",
"SelectQuery", "SELECT * FROM sampleTable\n" +
"WHERE \"field2\" like '%{{Table1.searchText || \"\"}}%'\n" +
"WHERE \"field1\" like '%{{Table1.searchText || \"\"}}%'\n" +
"ORDER BY \"{{col_select.selectedOptionValue}}\" {{order_select.selectedOptionLabel}}\n" +
"LIMIT {{Table1.pageSize}}" +
"OFFSET {{(Table1.pageNo - 1) * Table1.pageSize}};",
@ -151,7 +149,7 @@ public class CreateDBTablePageSolutionTests {
List<Key> keys = List.of(new DatasourceStructure.PrimaryKey("pKey", List.of("primaryKey")));
List<Column> columns = List.of(
new Column("primaryKey", "type1", null, true),
new Column("field1", "type2", null, false),
new Column("field1", "VARCHAR(23)", null, false),
new Column("field2", "type3", null, false),
new Column("field3", "type4", null, false),
new Column("field4", "type5", null, false)
@ -656,7 +654,7 @@ public class CreateDBTablePageSolutionTests {
.isEqualTo("{ primaryKey: ObjectId('{{data_table.selectedRow.primaryKey}}') }");
assertThat(pluginSpecifiedTemplate.get(12).getValue().toString().replaceAll(specialCharactersRegex, ""))
.isEqualTo("{\"field1\" : {{update_col_1.text}},\"field2\" : {{update_col_2.text}},\"field3\" : {{update_col_3.text}},\"field4\" : {{update_col_4.text}}\"}"
.isEqualTo("{\"field2\" : {{update_col_1.text}},\"field1\" : {{update_col_2.text}},\"field3\" : {{update_col_3.text}},\"field4\" : {{update_col_4.text}}\"}"
.replaceAll(specialCharactersRegex, ""));
} else if (queryType.equals("DELETE")) {
assertThat(pluginSpecifiedTemplate.get(13).getValue().toString().replaceAll(specialCharactersRegex, ""))