From 4380d38a841e714443479bbcf463f9a7052b391a Mon Sep 17 00:00:00 2001 From: Trisha Anand Date: Tue, 9 Nov 2021 16:02:47 +0530 Subject: [PATCH] chore: Removes incorrect trigger paths from list widget via database migrations (#8989) * Migrations for updating dynamic trigger paths for list widget in the existing pages to remove incorrect trigger paths. * Removed an unnecessary line * Review comment * If widgetType is null, then this would ensure that NPE is not thrown. --- .../appsmith/server/constants/FieldName.java | 1 + .../server/migrations/DatabaseChangelog.java | 181 +++++++++++++++++- 2 files changed, 178 insertions(+), 4 deletions(-) diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/constants/FieldName.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/constants/FieldName.java index 2b49e10e8d..b58b8da15e 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/constants/FieldName.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/constants/FieldName.java @@ -91,4 +91,5 @@ public class FieldName { public static final String UNUSED_DATASOURCE = "UNUSED_DATASOURCE"; public static final String BRANCH_NAME = "branchName"; public static final String DEFAULT = "default"; + public static final String DYNAMIC_TRIGGER_PATH_LIST = "dynamicTriggerPathList"; } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/DatabaseChangelog.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/DatabaseChangelog.java index 870ed9790b..a8f864ef9a 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/DatabaseChangelog.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/DatabaseChangelog.java @@ -60,6 +60,10 @@ import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoCursor; import com.mongodb.client.model.Filters; import com.mysema.commons.lang.Pair; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; import lombok.extern.slf4j.Slf4j; import net.minidev.json.JSONObject; import org.apache.commons.lang.ArrayUtils; @@ -111,6 +115,7 @@ import static com.appsmith.server.acl.AclPermission.MAKE_PUBLIC_APPLICATIONS; import static com.appsmith.server.acl.AclPermission.ORGANIZATION_EXPORT_APPLICATIONS; import static com.appsmith.server.acl.AclPermission.ORGANIZATION_INVITE_USERS; import static com.appsmith.server.acl.AclPermission.READ_ACTIONS; +import static com.appsmith.server.constants.FieldName.DYNAMIC_TRIGGER_PATH_LIST; import static com.appsmith.server.helpers.CollectionUtils.isNullOrEmpty; import static com.appsmith.server.repositories.BaseAppsmithRepositoryImpl.fieldName; import static java.lang.Boolean.FALSE; @@ -123,6 +128,15 @@ import static org.springframework.data.mongodb.core.query.Update.update; @ChangeLog(order = "001") public class DatabaseChangelog { + @AllArgsConstructor + @NoArgsConstructor + @Setter + @Getter + class DslUpdateDto { + private JSONObject dsl; + private Boolean updated; + } + /** * A private, pure utility function to create instances of Index objects to pass to `IndexOps.ensureIndex` method. * Note: The order of the fields here is important. An index with the fields `"name", "organizationId"` is different @@ -3591,7 +3605,8 @@ public class DatabaseChangelog { .include(fieldName(QApplication.application.name)); List applications = mongockTemplate.find(applicationQuery, Application.class); - for(Application application : applications) { + + for (Application application : applications) { mongockTemplate.updateFirst( query(where(fieldName(QApplication.application.id)).is(application.getId())), new Update().set(fieldName(QApplication.application.slug), TextUtils.makeSlug(application.getName())), @@ -3610,15 +3625,16 @@ public class DatabaseChangelog { )); List pages = mongockTemplate.find(pageQuery, NewPage.class); - for(NewPage page : pages) { + + for (NewPage page : pages) { Update update = new Update(); - if(page.getUnpublishedPage() != null) { + if (page.getUnpublishedPage() != null) { String fieldName = String.format("%s.%s", fieldName(QNewPage.newPage.unpublishedPage), fieldName(QNewPage.newPage.unpublishedPage.slug) ); update = update.set(fieldName, TextUtils.makeSlug(page.getUnpublishedPage().getName())); } - if(page.getPublishedPage() != null) { + if (page.getPublishedPage() != null) { String fieldName = String.format("%s.%s", fieldName(QNewPage.newPage.publishedPage), fieldName(QNewPage.newPage.publishedPage.slug) ); @@ -3631,4 +3647,161 @@ public class DatabaseChangelog { ); } } + + private DslUpdateDto updateListWidgetTriggerPaths(DslUpdateDto dslUpdateDto) { + JSONObject dsl = dslUpdateDto.getDsl(); + Boolean updated = dslUpdateDto.getUpdated(); + + if (dsl == null) { + // This isn't a valid widget configuration. No need to traverse this. + return dslUpdateDto; + } + + String widgetType = dsl.getAsString(FieldName.WIDGET_TYPE); + if ("LIST_WIDGET".equals(widgetType)) { + // Only List Widget would go through the following processing + + // Start by picking all fields where we expect to find dynamic triggers for this particular widget + List dynamicTriggerPaths = (ArrayList) dsl.get(DYNAMIC_TRIGGER_PATH_LIST); + + Set newTriggerPaths = new HashSet<>(); + + if (dynamicTriggerPaths != null) { + // Each of these might have nested structures, so we iterate through them to find the leaf node for each + for (Object x : dynamicTriggerPaths) { + Boolean validPath = true; + final String fieldPath = String.valueOf(((Map) x).get(FieldName.KEY)); + String[] fields = fieldPath.split("[].\\[]"); + // For nested fields, the parent dsl to search in would shift by one level every iteration + Object parent = dsl; + Iterator fieldsIterator = Arrays.stream(fields).filter(fieldToken -> !fieldToken.isBlank()).iterator(); + boolean isLeafNode = false; + // This loop will end at either a leaf node, or the last identified JSON field (by throwing an exception) + // Valid forms of the fieldPath for this search could be: + // root.field.list[index].childField.anotherList.indexWithDotOperator.multidimensionalList[index1][index2] + while (fieldsIterator.hasNext()) { + String nextKey = fieldsIterator.next(); + if (parent instanceof JSONObject) { + parent = ((JSONObject) parent).get(nextKey); + } else if (parent instanceof Map) { + parent = ((Map) parent).get(nextKey); + } else if (parent instanceof List) { + if (Pattern.matches(Pattern.compile("[0-9]+").toString(), nextKey)) { + try { + parent = ((List) parent).get(Integer.parseInt(nextKey)); + } catch (IndexOutOfBoundsException e) { + // The index being referred does not exist. Hence the path would not exist. + validPath = false; + } + } else { + validPath = false; + } + } + // After updating the parent, check for the types + if (parent == null) { + validPath = false; + } else if (parent instanceof String) { + // If we get String value, then this is a leaf node + isLeafNode = true; + } + + // Only extract mustache keys from leaf nodes + if (isLeafNode && validPath) { + + // We found the path. + if (!MustacheHelper.laxIsBindingPresentInString((String) parent)) { + // No bindings found. + break; + } + + newTriggerPaths.add(fieldPath); + } + } + } + + // Check if the newly computed trigger paths are different from the existing ones and if true, set it in the dsl + if (dynamicTriggerPaths.size() != newTriggerPaths.size() || !newTriggerPaths.containsAll(dynamicTriggerPaths)) { + updated = Boolean.TRUE; + List finalTriggerPaths = new ArrayList<>(); + for (String triggerPath : newTriggerPaths) { + Map entry = new HashMap<>(); + entry.put("key", triggerPath); + finalTriggerPaths.add(entry); + } + dsl.put(DYNAMIC_TRIGGER_PATH_LIST, finalTriggerPaths); + } + } + } + + // Fetch the children of the current node in the DSL and recursively iterate over them to extract bindings + ArrayList children = (ArrayList) dsl.get(FieldName.CHILDREN); + ArrayList newChildren = new ArrayList<>(); + if (children != null) { + for (int i = 0; i < children.size(); i++) { + Map data = (Map) children.get(i); + JSONObject object = new JSONObject(); + // If the children tag exists and there are entries within it + if (!CollectionUtils.isEmpty(data)) { + object.putAll(data); + DslUpdateDto childUpdated = updateListWidgetTriggerPaths(new DslUpdateDto(object, updated)); + updated = childUpdated.getUpdated(); + newChildren.add(childUpdated.getDsl()); + } + } + dsl.put(FieldName.CHILDREN, newChildren); + } + + return new DslUpdateDto(dsl, updated); + } + + @ChangeSet(order = "095", id = "update-list-widget-trigger-paths", author = "") + public void removeUnusedTriggerPathsListWidget(MongockTemplate mongockTemplate) { + + + // Find all the pages which haven't been deleted + + final Criteria possibleCandidatePagesCriteria = new Criteria().andOperator( + where("deletedAt").is(null), + where("unpublishedPage.layouts.0.dsl").exists(true) + ); + + Query pageQuery = query(possibleCandidatePagesCriteria); + pageQuery.fields() + .include(fieldName(QNewPage.newPage.id)); + + final List pages = mongockTemplate.find( + pageQuery, + NewPage.class + ); + + for (NewPage onlyIdPage : pages) { + + // Fetch one page at a time to avoid OOM. + NewPage page = mongockTemplate.findOne( + query(where(fieldName(QNewPage.newPage.id)).is(onlyIdPage.getId())), + NewPage.class + ); + + List layouts = page.getUnpublishedPage().getLayouts(); + + Layout layout = layouts.get(0); + // update the dsl + DslUpdateDto dslUpdateDto = updateListWidgetTriggerPaths(new DslUpdateDto(layout.getDsl(), FALSE)); + layout.setDsl(dslUpdateDto.getDsl()); + + if (page.getPublishedPage() != null) { + layouts = page.getPublishedPage().getLayouts(); + if (!CollectionUtils.isEmpty(layouts)) { + layout = layouts.get(0); + // update the dsl + dslUpdateDto = updateListWidgetTriggerPaths(new DslUpdateDto(layout.getDsl(), dslUpdateDto.getUpdated())); + layout.setDsl(dslUpdateDto.getDsl()); + } + } + + if (dslUpdateDto.getUpdated().equals(TRUE)) { + mongockTemplate.save(page); + } + } + } }