diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/controllers/ApplicationController.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/controllers/ApplicationController.java index 8c75530013..3c83beaf0d 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/controllers/ApplicationController.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/controllers/ApplicationController.java @@ -4,6 +4,7 @@ import com.appsmith.server.constants.Url; import com.appsmith.server.controllers.ce.ApplicationControllerCE; import com.appsmith.server.services.ApplicationPageService; import com.appsmith.server.services.ApplicationService; +import com.appsmith.server.services.ApplicationSnapshotService; import com.appsmith.server.services.ThemeService; import com.appsmith.server.solutions.ApplicationFetcher; import com.appsmith.server.solutions.ApplicationForkingService; @@ -20,10 +21,11 @@ public class ApplicationController extends ApplicationControllerCE { ApplicationFetcher applicationFetcher, ApplicationForkingService applicationForkingService, ImportExportApplicationService importExportApplicationService, - ThemeService themeService) { + ThemeService themeService, + ApplicationSnapshotService applicationSnapshotService) { super(service, applicationPageService, applicationFetcher, applicationForkingService, - importExportApplicationService, themeService); + importExportApplicationService, themeService, applicationSnapshotService); } } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/controllers/ce/ApplicationControllerCE.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/controllers/ce/ApplicationControllerCE.java index 4261d78961..937cf94d65 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/controllers/ce/ApplicationControllerCE.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/controllers/ce/ApplicationControllerCE.java @@ -4,23 +4,26 @@ import com.appsmith.external.models.Datasource; import com.appsmith.server.constants.FieldName; import com.appsmith.server.constants.Url; import com.appsmith.server.domains.Application; +import com.appsmith.server.domains.ApplicationSnapshot; import com.appsmith.server.domains.GitAuth; import com.appsmith.server.domains.Theme; import com.appsmith.server.dtos.ApplicationAccessDTO; import com.appsmith.server.dtos.ApplicationImportDTO; import com.appsmith.server.dtos.ApplicationPagesDTO; import com.appsmith.server.dtos.GitAuthDTO; +import com.appsmith.server.dtos.ReleaseItemsDTO; import com.appsmith.server.dtos.ResponseDTO; import com.appsmith.server.dtos.UserHomepageDTO; -import com.appsmith.server.dtos.ReleaseItemsDTO; import com.appsmith.server.exceptions.AppsmithError; import com.appsmith.server.exceptions.AppsmithException; import com.appsmith.server.services.ApplicationPageService; import com.appsmith.server.services.ApplicationService; +import com.appsmith.server.services.ApplicationSnapshotService; import com.appsmith.server.services.ThemeService; import com.appsmith.server.solutions.ApplicationFetcher; import com.appsmith.server.solutions.ApplicationForkingService; import com.appsmith.server.solutions.ImportExportApplicationService; +import jakarta.validation.Valid; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpHeaders; @@ -44,7 +47,6 @@ import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.server.ServerWebExchange; import reactor.core.publisher.Mono; -import jakarta.validation.Valid; import java.util.List; @Slf4j @@ -56,6 +58,7 @@ public class ApplicationControllerCE extends BaseController> createSnapshot(@PathVariable String id, @RequestHeader(name = FieldName.BRANCH_NAME, required = false) String branchName) { + log.debug("Going to create snapshot with application id: {}, branch: {}", id, branchName); + + return applicationSnapshotService.createApplicationSnapshot(id, branchName) + .map(result -> new ResponseDTO<>(HttpStatus.CREATED.value(), result, null)); + } + + @GetMapping("/snapshot/{id}") + public Mono> getSnapshotWithoutApplicationJson(@PathVariable String id, @RequestHeader(name = FieldName.BRANCH_NAME, required = false) String branchName) { + log.debug("Going to get snapshot with application id: {}, branch: {}", id, branchName); + + return applicationSnapshotService.getWithoutDataByApplicationId(id, branchName) + .map(applicationSnapshot -> new ResponseDTO<>(HttpStatus.OK.value(), applicationSnapshot, null)); + } + + @PostMapping("/snapshot/{id}/restore") + public Mono> restoreSnapshot(@PathVariable String id, @RequestHeader(name = FieldName.BRANCH_NAME, required = false) String branchName) { + log.debug("Going to restore snapshot with application id: {}, branch: {}", id, branchName); + + return applicationSnapshotService.restoreSnapshot(id, branchName) + .map(application -> new ResponseDTO<>(HttpStatus.OK.value(), application, null)); + } + @PostMapping(value = "/import/{workspaceId}", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) public Mono> importApplicationFromFile(@RequestPart("file") Mono fileMono, @PathVariable String workspaceId) { diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/domains/ApplicationSnapshot.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/domains/ApplicationSnapshot.java new file mode 100644 index 0000000000..a5ba2475ff --- /dev/null +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/domains/ApplicationSnapshot.java @@ -0,0 +1,40 @@ +package com.appsmith.server.domains; + +import com.appsmith.external.models.BaseDomain; +import com.appsmith.server.helpers.DateUtils; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import org.springframework.data.mongodb.core.mapping.Document; + +/** + * This stores a snapshot of an application. If a snapshot is more than 15 MB, we'll break it into smaller chunks. + * Both the root chunk and the child chunks will be stored in this collection. + * We'll use some attributes to create and maintain the sequence of the chunks. + */ +@Getter +@Setter +@NoArgsConstructor +@Document +public class ApplicationSnapshot extends BaseDomain { + private String applicationId; + + /** + * binary data, will be present always + */ + private byte [] data; + + /** + * chunkOrder: present only in child chunks. Used to maintain the order of the chunks. + * if a parent has 3 child chunks, the first one will have chunkOrder=1, second one 2 and third one 3 + */ + private int chunkOrder; + + /** + * Adding this method as updatedAt field in BaseDomain is annotated with @JsonIgnore + * @return Updated at timestamp in ISO format + */ + public String getUpdatedTime() { + return DateUtils.ISO_FORMATTER.format(this.getUpdatedAt()); + } +} diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/db/ce/Migration004CreateIndexForApplicationSnapshotMigration.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/db/ce/Migration004CreateIndexForApplicationSnapshotMigration.java new file mode 100644 index 0000000000..aad8497324 --- /dev/null +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/db/ce/Migration004CreateIndexForApplicationSnapshotMigration.java @@ -0,0 +1,36 @@ +package com.appsmith.server.migrations.db.ce; + +import com.appsmith.server.domains.ApplicationSnapshot; +import com.appsmith.server.domains.QApplicationSnapshot; +import io.mongock.api.annotations.ChangeUnit; +import io.mongock.api.annotations.Execution; +import io.mongock.api.annotations.RollbackExecution; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.data.mongodb.core.index.Index; + +import static com.appsmith.server.migrations.DatabaseChangelog1.ensureIndexes; +import static com.appsmith.server.migrations.DatabaseChangelog1.makeIndex; +import static com.appsmith.server.repositories.ce.BaseAppsmithRepositoryCEImpl.fieldName; + +@ChangeUnit(order = "004", id="create-index-for-application-snapshot-collection") +public class Migration004CreateIndexForApplicationSnapshotMigration { + private final MongoTemplate mongoTemplate; + + public Migration004CreateIndexForApplicationSnapshotMigration(MongoTemplate mongoTemplate) { + this.mongoTemplate = mongoTemplate; + } + + @RollbackExecution + public void demoRollbackExecution() { + } + + @Execution + public void addIndexOnApplicationIdAndChunkOrder() { + Index applicationIdChunkOrderUniqueIndex = makeIndex( + fieldName(QApplicationSnapshot.applicationSnapshot.applicationId), + fieldName(QApplicationSnapshot.applicationSnapshot.chunkOrder) + ).named("applicationId_chunkOrder_unique_index").unique(); + + ensureIndexes(mongoTemplate, ApplicationSnapshot.class, applicationIdChunkOrderUniqueIndex); + } +} diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/ApplicationSnapshotRepository.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/ApplicationSnapshotRepository.java new file mode 100644 index 0000000000..6f8b55a1fd --- /dev/null +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/ApplicationSnapshotRepository.java @@ -0,0 +1,9 @@ +package com.appsmith.server.repositories; + +import com.appsmith.server.repositories.ce.ApplicationSnapshotRepositoryCE; +import org.springframework.stereotype.Repository; + +@Repository +public interface ApplicationSnapshotRepository extends ApplicationSnapshotRepositoryCE, CustomApplicationSnapshotRepository { + +} diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/CustomApplicationSnapshotRepository.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/CustomApplicationSnapshotRepository.java new file mode 100644 index 0000000000..7b965a4692 --- /dev/null +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/CustomApplicationSnapshotRepository.java @@ -0,0 +1,7 @@ +package com.appsmith.server.repositories; + +import com.appsmith.server.repositories.ce.CustomApplicationSnapshotRepositoryCE; + +public interface CustomApplicationSnapshotRepository extends CustomApplicationSnapshotRepositoryCE { + +} diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/CustomApplicationSnapshotRepositoryImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/CustomApplicationSnapshotRepositoryImpl.java new file mode 100644 index 0000000000..d0fd60ccae --- /dev/null +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/CustomApplicationSnapshotRepositoryImpl.java @@ -0,0 +1,13 @@ +package com.appsmith.server.repositories; + +import com.appsmith.server.repositories.ce.CustomApplicationSnapshotRepositoryCEImpl; +import org.springframework.data.mongodb.core.ReactiveMongoOperations; +import org.springframework.data.mongodb.core.convert.MongoConverter; +import org.springframework.stereotype.Component; + +@Component +public class CustomApplicationSnapshotRepositoryImpl extends CustomApplicationSnapshotRepositoryCEImpl implements CustomApplicationSnapshotRepository { + public CustomApplicationSnapshotRepositoryImpl(ReactiveMongoOperations mongoOperations, MongoConverter mongoConverter, CacheableRepositoryHelper cacheableRepositoryHelper) { + super(mongoOperations, mongoConverter, cacheableRepositoryHelper); + } +} diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/ce/ApplicationSnapshotRepositoryCE.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/ce/ApplicationSnapshotRepositoryCE.java new file mode 100644 index 0000000000..4ff1af0e0e --- /dev/null +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/ce/ApplicationSnapshotRepositoryCE.java @@ -0,0 +1,11 @@ +package com.appsmith.server.repositories.ce; + +import com.appsmith.server.domains.ApplicationSnapshot; +import com.appsmith.server.repositories.BaseRepository; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +public interface ApplicationSnapshotRepositoryCE extends CustomApplicationSnapshotRepositoryCE, BaseRepository { + Flux findByApplicationId(String applicationId); + Mono deleteAllByApplicationId(String applicationId); +} diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/ce/CustomApplicationSnapshotRepositoryCE.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/ce/CustomApplicationSnapshotRepositoryCE.java new file mode 100644 index 0000000000..52887f17a9 --- /dev/null +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/ce/CustomApplicationSnapshotRepositoryCE.java @@ -0,0 +1,9 @@ +package com.appsmith.server.repositories.ce; + +import com.appsmith.server.domains.ApplicationSnapshot; +import com.appsmith.server.repositories.AppsmithRepository; +import reactor.core.publisher.Mono; + +public interface CustomApplicationSnapshotRepositoryCE extends AppsmithRepository { + Mono findWithoutData(String applicationId); +} diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/ce/CustomApplicationSnapshotRepositoryCEImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/ce/CustomApplicationSnapshotRepositoryCEImpl.java new file mode 100644 index 0000000000..a5d01593de --- /dev/null +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/ce/CustomApplicationSnapshotRepositoryCEImpl.java @@ -0,0 +1,40 @@ +package com.appsmith.server.repositories.ce; + +import com.appsmith.server.domains.ApplicationSnapshot; +import com.appsmith.server.domains.QApplicationSnapshot; +import com.appsmith.server.repositories.BaseAppsmithRepositoryImpl; +import com.appsmith.server.repositories.CacheableRepositoryHelper; +import org.springframework.data.mongodb.core.ReactiveMongoOperations; +import org.springframework.data.mongodb.core.convert.MongoConverter; +import org.springframework.data.mongodb.core.query.Criteria; +import reactor.core.publisher.Mono; + +import java.util.ArrayList; +import java.util.List; + +public class CustomApplicationSnapshotRepositoryCEImpl extends BaseAppsmithRepositoryImpl + implements CustomApplicationSnapshotRepositoryCE { + + public CustomApplicationSnapshotRepositoryCEImpl(ReactiveMongoOperations mongoOperations, MongoConverter mongoConverter, CacheableRepositoryHelper cacheableRepositoryHelper) { + super(mongoOperations, mongoConverter, cacheableRepositoryHelper); + } + + @Override + public Mono findWithoutData(String applicationId) { + List criteriaList = new ArrayList<>(); + criteriaList.add( + Criteria.where(fieldName(QApplicationSnapshot.applicationSnapshot.applicationId)).is(applicationId) + ); + criteriaList.add( + Criteria.where(fieldName(QApplicationSnapshot.applicationSnapshot.chunkOrder)).is(1) + ); + + List fieldNames = List.of( + fieldName(QApplicationSnapshot.applicationSnapshot.applicationId), + fieldName(QApplicationSnapshot.applicationSnapshot.chunkOrder), + fieldName(QApplicationSnapshot.applicationSnapshot.createdAt), + fieldName(QApplicationSnapshot.applicationSnapshot.updatedAt) + ); + return queryOne(criteriaList, fieldNames); + } +} diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ApplicationSnapshotService.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ApplicationSnapshotService.java new file mode 100644 index 0000000000..bd6ce294c4 --- /dev/null +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ApplicationSnapshotService.java @@ -0,0 +1,6 @@ +package com.appsmith.server.services; + +import com.appsmith.server.services.ce.ApplicationSnapshotServiceCE; + +public interface ApplicationSnapshotService extends ApplicationSnapshotServiceCE { +} diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ApplicationSnapshotServiceImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ApplicationSnapshotServiceImpl.java new file mode 100644 index 0000000000..8de8a3e740 --- /dev/null +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ApplicationSnapshotServiceImpl.java @@ -0,0 +1,18 @@ +package com.appsmith.server.services; + +import com.appsmith.server.repositories.ApplicationSnapshotRepository; +import com.appsmith.server.services.ce.ApplicationSnapshotServiceCEImpl; +import com.appsmith.server.solutions.ApplicationPermission; +import com.appsmith.server.solutions.ImportExportApplicationService; +import com.google.gson.Gson; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; + +@Slf4j +@Service +public class ApplicationSnapshotServiceImpl extends ApplicationSnapshotServiceCEImpl implements ApplicationSnapshotService { + + public ApplicationSnapshotServiceImpl(ApplicationSnapshotRepository applicationSnapshotRepository, ApplicationService applicationService, ImportExportApplicationService importExportApplicationService, ApplicationPermission applicationPermission, Gson gson) { + super(applicationSnapshotRepository, applicationService, importExportApplicationService, applicationPermission, gson); + } +} diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/ApplicationSnapshotServiceCE.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/ApplicationSnapshotServiceCE.java new file mode 100644 index 0000000000..2548271337 --- /dev/null +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/ApplicationSnapshotServiceCE.java @@ -0,0 +1,20 @@ +package com.appsmith.server.services.ce; + +import com.appsmith.server.domains.Application; +import com.appsmith.server.domains.ApplicationSnapshot; +import reactor.core.publisher.Mono; + +public interface ApplicationSnapshotServiceCE { + /** + * This method will create a new snapshot of the provided applicationId and branch name and store in the + * ApplicationSnapshot collection. + * @param applicationId ID of the application, default application ID if application is connected to Git + * @param branchName name of the Git branch, null or empty if not connected to Git + * @return Created snapshot ID + */ + Mono createApplicationSnapshot(String applicationId, String branchName); + + Mono getWithoutDataByApplicationId(String applicationId, String branchName); + + Mono restoreSnapshot(String applicationId, String branchName); +} diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/ApplicationSnapshotServiceCEImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/ApplicationSnapshotServiceCEImpl.java new file mode 100644 index 0000000000..eb864625ad --- /dev/null +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/ApplicationSnapshotServiceCEImpl.java @@ -0,0 +1,137 @@ +package com.appsmith.server.services.ce; + +import com.appsmith.server.constants.FieldName; +import com.appsmith.server.constants.SerialiseApplicationObjective; +import com.appsmith.server.domains.Application; +import com.appsmith.server.domains.ApplicationSnapshot; +import com.appsmith.server.dtos.ApplicationJson; +import com.appsmith.server.exceptions.AppsmithError; +import com.appsmith.server.exceptions.AppsmithException; +import com.appsmith.server.repositories.ApplicationSnapshotRepository; +import com.appsmith.server.services.ApplicationService; +import com.appsmith.server.solutions.ApplicationPermission; +import com.appsmith.server.solutions.ImportExportApplicationService; +import com.google.gson.Gson; +import lombok.RequiredArgsConstructor; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; + +@RequiredArgsConstructor +public class ApplicationSnapshotServiceCEImpl implements ApplicationSnapshotServiceCE { + private final ApplicationSnapshotRepository applicationSnapshotRepository; + private final ApplicationService applicationService; + private final ImportExportApplicationService importExportApplicationService; + private final ApplicationPermission applicationPermission; + private final Gson gson; + + private static final int MAX_SNAPSHOT_SIZE = 15*1024*1024; // 15 MB + + @Override + public Mono createApplicationSnapshot(String applicationId, String branchName) { + return applicationService.findBranchedApplicationId(branchName, applicationId, applicationPermission.getEditPermission()) + /* SerialiseApplicationObjective=VERSION_CONTROL because this API can be invoked from developers. + exportApplicationById method check for MANAGE_PERMISSION if SerialiseApplicationObjective=SHARE. + */ + .flatMap(branchedAppId -> + Mono.zip( + importExportApplicationService.exportApplicationById(branchedAppId, SerialiseApplicationObjective.VERSION_CONTROL), + Mono.just(branchedAppId) + ) + ) + .flatMapMany(objects -> { + String branchedAppId = objects.getT2(); + ApplicationJson applicationJson = objects.getT1(); + return applicationSnapshotRepository.deleteAllByApplicationId(branchedAppId) + .thenMany(createSnapshots(branchedAppId, applicationJson)); + }) + .then(Mono.just(Boolean.TRUE)); + } + + private Flux createSnapshots(String applicationId, ApplicationJson applicationJson) { + String json = gson.toJson(applicationJson); + // check the size of the exported json before storing to avoid mongodb document size limit + byte[] utf8JsonString = json.getBytes(StandardCharsets.UTF_8); + List applicationSnapshots = createSnapshotsObjects(utf8JsonString, applicationId); + return applicationSnapshotRepository.saveAll(applicationSnapshots); + } + + @Override + public Mono getWithoutDataByApplicationId(String applicationId, String branchName) { + // get application first to check the permission and get child aka branched application ID + return applicationService.findBranchedApplicationId(branchName, applicationId, applicationPermission.getEditPermission()) + .switchIfEmpty(Mono.error( + new AppsmithException(AppsmithError.NO_RESOURCE_FOUND, FieldName.APPLICATION, applicationId)) + ) + .flatMap(applicationSnapshotRepository::findWithoutData) + .switchIfEmpty(Mono.error( + new AppsmithException(AppsmithError.NO_RESOURCE_FOUND, FieldName.APPLICATION, applicationId)) + ); + } + + @Override + public Mono restoreSnapshot(String applicationId, String branchName) { + return applicationService.findByBranchNameAndDefaultApplicationId(branchName, applicationId, applicationPermission.getEditPermission()) + .switchIfEmpty(Mono.error( + new AppsmithException(AppsmithError.NO_RESOURCE_FOUND, FieldName.APPLICATION, applicationId)) + ) + .flatMap( + application -> getApplicationJsonStringFromSnapShot(application.getId()) + .zipWith(Mono.just(application)) + ) + .flatMap(objects -> { + String applicationJsonString = objects.getT1(); + Application application = objects.getT2(); + ApplicationJson applicationJson = gson.fromJson(applicationJsonString, ApplicationJson.class); + return importExportApplicationService.importApplicationInWorkspace( + application.getWorkspaceId(), applicationJson, application.getId(), branchName + ); + }); + } + + private Mono getApplicationJsonStringFromSnapShot(String applicationId) { + return applicationSnapshotRepository.findByApplicationId(applicationId) + .sort(Comparator.comparingInt(ApplicationSnapshot::getChunkOrder)) + .map(ApplicationSnapshot::getData) + .collectList() + .map(bytes -> { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + for(byte [] b: bytes) { + outputStream.writeBytes(b); + } + return outputStream.toString(StandardCharsets.UTF_8); + }); + } + + private List createSnapshotsObjects(byte [] bytes, String applicationId) { + List applicationSnapshots = new ArrayList<>(); + int total = bytes.length; + int copiedCount = 0; + int chunkOrder = 1; + + while (copiedCount < total) { + int currentChunkSize = MAX_SNAPSHOT_SIZE; + if(copiedCount + currentChunkSize > total) { + currentChunkSize = total - copiedCount; + } + byte [] sub = new byte[currentChunkSize]; + System.arraycopy(bytes, copiedCount, sub, 0, currentChunkSize); + copiedCount += currentChunkSize; + + // create snapshot that'll contain the chunk of data + ApplicationSnapshot applicationSnapshot = new ApplicationSnapshot(); + applicationSnapshot.setData(sub); + applicationSnapshot.setApplicationId(applicationId); + applicationSnapshot.setChunkOrder(chunkOrder); + applicationSnapshots.add(applicationSnapshot); + + chunkOrder++; + } + return applicationSnapshots; + } +} diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/solutions/ImportExportApplicationServiceImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/solutions/ImportExportApplicationServiceImpl.java index d2a85c74aa..3b15113d3d 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/solutions/ImportExportApplicationServiceImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/solutions/ImportExportApplicationServiceImpl.java @@ -9,6 +9,7 @@ import com.appsmith.server.services.ActionCollectionService; import com.appsmith.server.services.AnalyticsService; import com.appsmith.server.services.ApplicationPageService; import com.appsmith.server.services.ApplicationService; +import com.appsmith.server.services.ApplicationSnapshotService; import com.appsmith.server.services.CustomJSLibService; import com.appsmith.server.services.DatasourceService; import com.appsmith.server.services.NewActionService; diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/solutions/ImportExportApplicationServiceImplV2.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/solutions/ImportExportApplicationServiceImplV2.java index 0ea598ca09..7791ec4565 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/solutions/ImportExportApplicationServiceImplV2.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/solutions/ImportExportApplicationServiceImplV2.java @@ -9,6 +9,8 @@ import com.appsmith.server.services.ActionCollectionService; import com.appsmith.server.services.AnalyticsService; import com.appsmith.server.services.ApplicationPageService; import com.appsmith.server.services.ApplicationService; +import com.appsmith.server.services.ApplicationSnapshotService; +import com.appsmith.server.services.CustomJSLibService; import com.appsmith.server.services.DatasourceService; import com.appsmith.server.services.NewActionService; import com.appsmith.server.services.NewPageService; @@ -16,13 +18,11 @@ import com.appsmith.server.services.SequenceService; import com.appsmith.server.services.SessionUserService; import com.appsmith.server.services.ThemeService; import com.appsmith.server.services.WorkspaceService; -import com.appsmith.server.services.CustomJSLibService; import com.appsmith.server.solutions.ce.ImportExportApplicationServiceCEImplV2; import com.google.gson.Gson; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Component; -import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.reactive.TransactionalOperator; @Slf4j @@ -54,12 +54,14 @@ public class ImportExportApplicationServiceImplV2 extends ImportExportApplicatio PagePermission pagePermission, ActionPermission actionPermission, Gson gson, - TransactionalOperator transactionalOperator) { + TransactionalOperator transactionalOperator, + ApplicationSnapshotService applicationSnapshotService) { super(datasourceService, sessionUserService, newActionRepository, datasourceRepository, pluginRepository, workspaceService, applicationService, newPageService, applicationPageService, newPageRepository, newActionService, sequenceService, examplesWorkspaceCloner, actionCollectionRepository, actionCollectionService, themeService, analyticsService, customJSLibService, datasourcePermission, - workspacePermission, applicationPermission, pagePermission, actionPermission, gson, transactionalOperator); + workspacePermission, applicationPermission, pagePermission, actionPermission, gson, transactionalOperator, + applicationSnapshotService); } } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/solutions/ce/ImportExportApplicationServiceCEImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/solutions/ce/ImportExportApplicationServiceCEImpl.java index 069db8ad61..5e5ec1c2a3 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/solutions/ce/ImportExportApplicationServiceCEImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/solutions/ce/ImportExportApplicationServiceCEImpl.java @@ -23,7 +23,6 @@ import com.appsmith.server.domains.Application; import com.appsmith.server.domains.ApplicationPage; import com.appsmith.server.domains.CustomJSLib; import com.appsmith.server.domains.GitApplicationMetadata; -import com.appsmith.server.domains.GitApplicationMetadata; import com.appsmith.server.domains.Layout; import com.appsmith.server.domains.NewAction; import com.appsmith.server.domains.NewPage; @@ -844,6 +843,7 @@ public class ImportExportApplicationServiceCEImpl implements ImportExportApplica // The isPublic flag has a default value as false and this would be confusing to user // when it is reset to false during importing where the application already is present in DB importedApplication.setIsPublic(null); + importedApplication.setPolicies(null); copyNestedNonNullProperties(importedApplication, existingApplication); // We are expecting the changes present in DB are committed to git directory // so that these won't be lost when we are pulling changes from remote and @@ -851,7 +851,16 @@ public class ImportExportApplicationServiceCEImpl implements ImportExportApplica // the changes from remote // We are using the save instead of update as we are using @Encrypted // for GitAuth - return applicationService.findById(existingApplication.getGitApplicationMetadata().getDefaultApplicationId()) + Mono parentApplicationMono; + if (existingApplication.getGitApplicationMetadata() != null) { + parentApplicationMono = applicationService.findById( + existingApplication.getGitApplicationMetadata().getDefaultApplicationId() + ); + } else { + parentApplicationMono = Mono.just(existingApplication); + } + + return parentApplicationMono .flatMap(application1 -> { // Set the policies from the defaultApplication existingApplication.setPolicies(application1.getPolicies()); diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/solutions/ce/ImportExportApplicationServiceCEImplV2.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/solutions/ce/ImportExportApplicationServiceCEImplV2.java index 2c08595e54..728f9c7556 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/solutions/ce/ImportExportApplicationServiceCEImplV2.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/solutions/ce/ImportExportApplicationServiceCEImplV2.java @@ -51,6 +51,7 @@ import com.appsmith.server.services.ActionCollectionService; import com.appsmith.server.services.AnalyticsService; import com.appsmith.server.services.ApplicationPageService; import com.appsmith.server.services.ApplicationService; +import com.appsmith.server.services.ApplicationSnapshotService; import com.appsmith.server.services.CustomJSLibService; import com.appsmith.server.services.DatasourceService; import com.appsmith.server.services.NewActionService; @@ -78,7 +79,6 @@ import org.springframework.http.ContentDisposition; import org.springframework.http.HttpHeaders; import org.springframework.http.MediaType; import org.springframework.http.codec.multipart.Part; -import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.reactive.TransactionalOperator; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @@ -135,6 +135,7 @@ public class ImportExportApplicationServiceCEImplV2 implements ImportExportAppli private final ActionPermission actionPermission; private final Gson gson; private final TransactionalOperator transactionalOperator; + private final ApplicationSnapshotService applicationSnapshotService; private static final Set ALLOWED_CONTENT_TYPES = Set.of(MediaType.APPLICATION_JSON); private static final String INVALID_JSON_FILE = "invalid json file"; @@ -887,6 +888,7 @@ public class ImportExportApplicationServiceCEImplV2 implements ImportExportAppli // The isPublic flag has a default value as false and this would be confusing to user // when it is reset to false during importing where the application already is present in DB importedApplication.setIsPublic(null); + importedApplication.setPolicies(null); copyNestedNonNullProperties(importedApplication, existingApplication); // We are expecting the changes present in DB are committed to git directory // so that these won't be lost when we are pulling changes from remote and @@ -894,7 +896,16 @@ public class ImportExportApplicationServiceCEImplV2 implements ImportExportAppli // the changes from remote // We are using the save instead of update as we are using @Encrypted // for GitAuth - return applicationService.findById(existingApplication.getGitApplicationMetadata().getDefaultApplicationId()) + Mono parentApplicationMono; + if (existingApplication.getGitApplicationMetadata() != null) { + parentApplicationMono = applicationService.findById( + existingApplication.getGitApplicationMetadata().getDefaultApplicationId() + ); + } else { + parentApplicationMono = Mono.just(existingApplication); + } + + return parentApplicationMono .flatMap(application1 -> { // Set the policies from the defaultApplication existingApplication.setPolicies(application1.getPolicies()); diff --git a/app/server/appsmith-server/src/test/java/com/appsmith/server/controllers/ApplicationControllerTest.java b/app/server/appsmith-server/src/test/java/com/appsmith/server/controllers/ApplicationControllerTest.java index 376d1b30be..0fd2ab04d3 100644 --- a/app/server/appsmith-server/src/test/java/com/appsmith/server/controllers/ApplicationControllerTest.java +++ b/app/server/appsmith-server/src/test/java/com/appsmith/server/controllers/ApplicationControllerTest.java @@ -8,6 +8,7 @@ import com.appsmith.server.helpers.RedisUtils; import com.appsmith.server.exceptions.AppsmithErrorCode; import com.appsmith.server.services.ApplicationPageService; import com.appsmith.server.services.ApplicationService; +import com.appsmith.server.services.ApplicationSnapshotService; import com.appsmith.server.services.ThemeService; import com.appsmith.server.services.UserDataService; import com.appsmith.server.solutions.ApplicationFetcher; @@ -53,6 +54,9 @@ public class ApplicationControllerTest { @MockBean ImportExportApplicationService importExportApplicationService; + @MockBean + ApplicationSnapshotService applicationSnapshotService; + @MockBean ThemeService themeService; diff --git a/app/server/appsmith-server/src/test/java/com/appsmith/server/repositories/ApplicationSnapshotRepositoryTest.java b/app/server/appsmith-server/src/test/java/com/appsmith/server/repositories/ApplicationSnapshotRepositoryTest.java new file mode 100644 index 0000000000..6950e84bb1 --- /dev/null +++ b/app/server/appsmith-server/src/test/java/com/appsmith/server/repositories/ApplicationSnapshotRepositoryTest.java @@ -0,0 +1,140 @@ +package com.appsmith.server.repositories; + +import com.appsmith.server.domains.ApplicationSnapshot; +import com.google.gson.Gson; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.security.test.context.support.WithUserDetails; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; +import reactor.test.StepVerifier; + +import java.util.List; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; + + +@SpringBootTest +public class ApplicationSnapshotRepositoryTest { + @Autowired + private ApplicationSnapshotRepository applicationSnapshotRepository; + + @Autowired + private Gson gson; + + @Test + @WithUserDetails("api_user") + public void findWithoutData_WhenMatched_ReturnsMatchedDocumentWithoutData() { + String testAppId1 = UUID.randomUUID().toString(), + testAppId2 = UUID.randomUUID().toString(); + + ApplicationSnapshot snapshot1 = new ApplicationSnapshot(); + snapshot1.setApplicationId(testAppId1); + snapshot1.setChunkOrder(1); + + ApplicationSnapshot snapshot2 = new ApplicationSnapshot(); + snapshot2.setData(new byte[10]); + snapshot2.setApplicationId(testAppId2); + snapshot2.setChunkOrder(1); + + Mono snapshotMono = applicationSnapshotRepository.saveAll(List.of(snapshot1, snapshot2)) + .then(applicationSnapshotRepository.findWithoutData(testAppId2)); + + StepVerifier.create(snapshotMono).assertNext(applicationSnapshot -> { + assertThat(applicationSnapshot.getApplicationId()).isEqualTo(testAppId2); + assertThat(applicationSnapshot.getData()).isNull(); + assertThat(applicationSnapshot.getChunkOrder()).isEqualTo(1); + }).verifyComplete(); + } + + @Test + @WithUserDetails("api_user") + public void findWithoutData_WhenMultipleChunksArePresent_ReturnsSingleOne() { + String testAppId1 = UUID.randomUUID().toString(); + + // create two snapshots with same application id and another one with different application id + ApplicationSnapshot snapshot1 = new ApplicationSnapshot(); + snapshot1.setApplicationId(testAppId1); + snapshot1.setChunkOrder(1); + + ApplicationSnapshot snapshot2 = new ApplicationSnapshot(); + snapshot2.setApplicationId(testAppId1); + snapshot2.setChunkOrder(2); + + Mono snapshotMono = applicationSnapshotRepository.saveAll(List.of(snapshot1, snapshot2)) + .then(applicationSnapshotRepository.findWithoutData(testAppId1)); + + StepVerifier.create(snapshotMono).assertNext(applicationSnapshot -> { + assertThat(applicationSnapshot.getApplicationId()).isEqualTo(testAppId1); + assertThat(applicationSnapshot.getChunkOrder()).isEqualTo(1); + }).verifyComplete(); + } + + @Test + @WithUserDetails("api_user") + public void deleteAllByApplicationId_WhenMatched_ReturnsMatchedDocumentWithoutData() { + String testAppId1 = UUID.randomUUID().toString(), + testAppId2 = UUID.randomUUID().toString(); + + // create two snapshots with same application id and another one with different application id + ApplicationSnapshot snapshot1 = new ApplicationSnapshot(); + snapshot1.setApplicationId(testAppId1); + snapshot1.setChunkOrder(1); + + ApplicationSnapshot snapshot2 = new ApplicationSnapshot(); + snapshot2.setApplicationId(testAppId1); + snapshot2.setChunkOrder(2); + + ApplicationSnapshot snapshot3 = new ApplicationSnapshot(); + snapshot3.setApplicationId(testAppId2); + snapshot3.setChunkOrder(1); + + Flux applicationSnapshots = applicationSnapshotRepository.saveAll(List.of(snapshot1, snapshot2, snapshot3)) + .then(applicationSnapshotRepository.deleteAllByApplicationId(testAppId1)) + .thenMany(applicationSnapshotRepository.findByApplicationId(testAppId1)); + + StepVerifier.create(applicationSnapshots) + .verifyComplete(); + + StepVerifier.create(applicationSnapshotRepository.findByApplicationId(testAppId2)) + .assertNext(applicationSnapshot -> { + assertThat(applicationSnapshot.getApplicationId()).isEqualTo(testAppId2); + assertThat(applicationSnapshot.getChunkOrder()).isEqualTo(1); + }) + .verifyComplete(); + } + + @Test + @WithUserDetails("api_user") + public void findByApplicationId_WhenMatched_ReturnsMatchedDocumentWithoutData() { + String testAppId1 = UUID.randomUUID().toString(), + testAppId2 = UUID.randomUUID().toString(); + + // create two snapshots with same application id and another one with different application id + ApplicationSnapshot snapshot1 = new ApplicationSnapshot(); + snapshot1.setApplicationId(testAppId1); + snapshot1.setChunkOrder(1); + + ApplicationSnapshot snapshot2 = new ApplicationSnapshot(); + snapshot2.setApplicationId(testAppId1); + snapshot2.setChunkOrder(2); + + ApplicationSnapshot snapshot3 = new ApplicationSnapshot(); + snapshot3.setApplicationId(testAppId2); + snapshot3.setChunkOrder(1); + + Flux applicationSnapshots = applicationSnapshotRepository.saveAll(List.of(snapshot1, snapshot2, snapshot3)) + .thenMany(applicationSnapshotRepository.findByApplicationId(testAppId1)); + + StepVerifier.create(applicationSnapshots) + .assertNext(applicationSnapshot -> { + assertThat(applicationSnapshot.getApplicationId()).isEqualTo(testAppId1); + }) + .assertNext(applicationSnapshot -> { + assertThat(applicationSnapshot.getApplicationId()).isEqualTo(testAppId1); + }) + .verifyComplete(); + } +} \ No newline at end of file diff --git a/app/server/appsmith-server/src/test/java/com/appsmith/server/services/ApplicationSnapshotServiceTest.java b/app/server/appsmith-server/src/test/java/com/appsmith/server/services/ApplicationSnapshotServiceTest.java new file mode 100644 index 0000000000..5d74b7ce38 --- /dev/null +++ b/app/server/appsmith-server/src/test/java/com/appsmith/server/services/ApplicationSnapshotServiceTest.java @@ -0,0 +1,228 @@ +package com.appsmith.server.services; + +import com.appsmith.server.domains.Application; +import com.appsmith.server.domains.ApplicationMode; +import com.appsmith.server.domains.ApplicationSnapshot; +import com.appsmith.server.domains.GitApplicationMetadata; +import com.appsmith.server.domains.Workspace; +import com.appsmith.server.dtos.ApplicationPagesDTO; +import com.appsmith.server.dtos.PageDTO; +import com.appsmith.server.repositories.ApplicationSnapshotRepository; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.security.test.context.support.WithUserDetails; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; +import reactor.test.StepVerifier; +import reactor.util.function.Tuple2; + +import java.nio.charset.StandardCharsets; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; + +@SpringBootTest +public class ApplicationSnapshotServiceTest { + @Autowired + private ApplicationPageService applicationPageService; + + @Autowired + private NewPageService newPageService; + + @Autowired + private ApplicationSnapshotService applicationSnapshotService; + + @Autowired + private WorkspaceService workspaceService; + + @Autowired + private ApplicationSnapshotRepository applicationSnapshotRepository; + + @Test + @WithUserDetails("api_user") + public void createApplicationSnapshot_WhenNoPreviousSnapshotExists_NewCreated() { + // create a new workspace + Workspace workspace = new Workspace(); + workspace.setName("Test workspace " + UUID.randomUUID()); + + Mono snapshotMono = workspaceService.create(workspace) + .flatMap(createdWorkspace -> { + Application testApplication = new Application(); + testApplication.setName("Test app for snapshot"); + testApplication.setWorkspaceId(createdWorkspace.getId()); + return applicationPageService.createApplication(testApplication); + }) + .flatMap(application -> { + assert application.getId() != null; + return applicationSnapshotService.createApplicationSnapshot(application.getId(), "") + .thenReturn(application.getId()); + }) + .flatMap(applicationId -> applicationSnapshotService.getWithoutDataByApplicationId(applicationId, null)); + + StepVerifier.create(snapshotMono) + .assertNext(snapshot -> { + assertThat(snapshot.getApplicationId()).isNotNull(); + assertThat(snapshot.getData()).isNull(); + }) + .verifyComplete(); + } + + @Test + @WithUserDetails("api_user") + public void createApplicationSnapshot_WhenSnapshotExists_ExistingSnapshotUpdated() { + // create a new workspace + Workspace workspace = new Workspace(); + workspace.setName("Test workspace " + UUID.randomUUID()); + + Mono snapshotMono = workspaceService.create(workspace) + .flatMap(createdWorkspace -> { + Application testApplication = new Application(); + testApplication.setName("Test app for snapshot"); + testApplication.setWorkspaceId(createdWorkspace.getId()); + return applicationPageService.createApplication(testApplication); + }) + .flatMap(application -> { + assert application.getId() != null; + // create snapshot twice + return applicationSnapshotService.createApplicationSnapshot(application.getId(), "") + .then(applicationSnapshotService.createApplicationSnapshot(application.getId(), "")) + .thenReturn(application.getId()); + }) + .flatMap(applicationId -> applicationSnapshotService.getWithoutDataByApplicationId(applicationId, null)); + + StepVerifier.create(snapshotMono) + .assertNext(snapshot -> { + assertThat(snapshot.getApplicationId()).isNotNull(); + assertThat(snapshot.getData()).isNull(); + }) + .verifyComplete(); + } + + @Test + @WithUserDetails("api_user") + public void createApplicationSnapshot_WhenGitBranchExists_SnapshotCreatedWithBranchedAppId() { + String uniqueString = UUID.randomUUID().toString(); + String testDefaultAppId = "default-app-" + uniqueString; + String testBranchName = "hello/world"; + // create a new workspace + Workspace workspace = new Workspace(); + workspace.setName("Test workspace " + uniqueString); + + Mono> tuple2Mono = workspaceService.create(workspace) + .flatMap(createdWorkspace -> { + Application testApplication = new Application(); + testApplication.setName("Test app for snapshot"); + testApplication.setWorkspaceId(createdWorkspace.getId()); + + // this app will have default app id=testDefaultAppId and branch name=test branch name + GitApplicationMetadata gitApplicationMetadata = new GitApplicationMetadata(); + gitApplicationMetadata.setDefaultApplicationId(testDefaultAppId); + gitApplicationMetadata.setBranchName(testBranchName); + testApplication.setGitApplicationMetadata(gitApplicationMetadata); + + return applicationPageService.createApplication(testApplication); + }) + .flatMap(application -> applicationSnapshotService.createApplicationSnapshot(testDefaultAppId, testBranchName) + .then(applicationSnapshotService.getWithoutDataByApplicationId(testDefaultAppId, testBranchName)) + .zipWith(Mono.just(application))); + + StepVerifier.create(tuple2Mono) + .assertNext(objects -> { + ApplicationSnapshot applicationSnapshot = objects.getT1(); + Application application = objects.getT2(); + assertThat(applicationSnapshot.getData()).isNull(); + assertThat(applicationSnapshot.getApplicationId()).isEqualTo(application.getId()); + }) + .verifyComplete(); + } + + @Test + @WithUserDetails("api_user") + public void createApplicationSnapshot_OlderSnapshotExists_OlderSnapshotsRemoved() { + String uniqueString = UUID.randomUUID().toString(); + String testDefaultAppId = "default-app-" + uniqueString; + String testBranchName = null; + // create a new workspace + Workspace workspace = new Workspace(); + workspace.setName("Test workspace " + uniqueString); + + Flux applicationSnapshotFlux = workspaceService.create(workspace) + .flatMap(createdWorkspace -> { + Application testApplication = new Application(); + testApplication.setName("Test app for snapshot"); + testApplication.setWorkspaceId(createdWorkspace.getId()); + return applicationPageService.createApplication(testApplication); + }) + .flatMap(application -> { + ApplicationSnapshot applicationSnapshot = new ApplicationSnapshot(); + applicationSnapshot.setApplicationId(application.getId()); + applicationSnapshot.setChunkOrder(5); + applicationSnapshot.setData("Hello".getBytes(StandardCharsets.UTF_8)); + return applicationSnapshotRepository.save(applicationSnapshot).thenReturn(application); + }) + .flatMapMany(application -> + applicationSnapshotService.createApplicationSnapshot(application.getId(), null) + .thenMany(applicationSnapshotRepository.findByApplicationId(application.getId())) + ); + + StepVerifier.create(applicationSnapshotFlux) + .assertNext(applicationSnapshot -> { + assertThat(applicationSnapshot.getChunkOrder()).isEqualTo(1); + }) + .verifyComplete(); + } + + @Test + @WithUserDetails("api_user") + public void restoreSnapshot_WhenNewPagesAddedAfterSnapshotTaken_NewPagesRemovedAfterSnapshotIsRestored() { + String uniqueString = UUID.randomUUID().toString(); + String testDefaultAppId = "default-app-" + uniqueString; + String testBranchName = "hello/world"; + + // create a new workspace + Workspace workspace = new Workspace(); + workspace.setName("Test workspace " + uniqueString); + + Mono applicationMono = workspaceService.create(workspace) + .flatMap(createdWorkspace -> { + Application testApplication = new Application(); + testApplication.setName("App before snapshot"); + return applicationPageService.createApplication(testApplication, workspace.getId()); + }).cache(); + + Mono pagesBeforeSnapshot = applicationMono + .flatMap(createdApp -> { + // add a page to the application + PageDTO pageDTO = new PageDTO(); + pageDTO.setName("Home"); + pageDTO.setApplicationId(createdApp.getId()); + return applicationPageService.createPage(pageDTO) + .then(newPageService.findApplicationPages(createdApp.getId(), null, null, ApplicationMode.EDIT)); + }); + + Mono pagesAfterSnapshot = applicationMono.flatMap(application -> { // create a snapshot + return applicationSnapshotService.createApplicationSnapshot(application.getId(), null) + .thenReturn(application); + }).flatMap(application -> { // add a new page to the application + PageDTO pageDTO = new PageDTO(); + pageDTO.setName("About"); + pageDTO.setApplicationId(application.getId()); + return applicationPageService.createPage(pageDTO) + .then(applicationSnapshotService.restoreSnapshot(application.getId(), null)) + .then(newPageService.findApplicationPages(application.getId(), null, null, ApplicationMode.EDIT)); + }); + + // not using Mono.zip because we want pagesBeforeSnapshot to finish first + Mono> tuple2Mono = pagesBeforeSnapshot + .flatMap(applicationPagesDTO -> pagesAfterSnapshot.zipWith(Mono.just(applicationPagesDTO))); + + StepVerifier.create(tuple2Mono) + .assertNext(objects -> { + ApplicationPagesDTO beforePages = objects.getT2(); + ApplicationPagesDTO afterPages = objects.getT1(); + assertThat(beforePages.getPages().size()).isEqualTo(afterPages.getPages().size()); + }) + .verifyComplete(); + } +} \ No newline at end of file diff --git a/app/server/appsmith-server/src/test/java/com/appsmith/server/services/ce/ApplicationSnapshotServiceUnitTest.java b/app/server/appsmith-server/src/test/java/com/appsmith/server/services/ce/ApplicationSnapshotServiceUnitTest.java new file mode 100644 index 0000000000..c3f9f10151 --- /dev/null +++ b/app/server/appsmith-server/src/test/java/com/appsmith/server/services/ce/ApplicationSnapshotServiceUnitTest.java @@ -0,0 +1,166 @@ +package com.appsmith.server.services.ce; + +import com.appsmith.server.acl.AclPermission; +import com.appsmith.server.constants.SerialiseApplicationObjective; +import com.appsmith.server.domains.Application; +import com.appsmith.server.domains.ApplicationSnapshot; +import com.appsmith.server.domains.Layout; +import com.appsmith.server.domains.NewPage; +import com.appsmith.server.dtos.ApplicationJson; +import com.appsmith.server.dtos.PageDTO; +import com.appsmith.server.repositories.ApplicationSnapshotRepository; +import com.appsmith.server.services.ApplicationService; +import com.appsmith.server.services.ApplicationSnapshotService; +import com.appsmith.server.solutions.ImportExportApplicationService; +import com.google.gson.Gson; +import net.minidev.json.JSONObject; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentMatcher; +import org.mockito.Mockito; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; +import reactor.test.StepVerifier; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; + +import static java.util.Arrays.copyOfRange; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.ArgumentMatchers.eq; + +@SpringBootTest +public class ApplicationSnapshotServiceUnitTest { + + @MockBean + ApplicationService applicationService; + + @MockBean + ImportExportApplicationService importExportApplicationService; + + @MockBean + ApplicationSnapshotRepository applicationSnapshotRepository; + + @Autowired + ApplicationSnapshotService applicationSnapshotService; + + @Autowired + Gson gson; + + @Test + public void createApplicationSnapshot_WhenApplicationTooLarge_SnapshotCreatedSuccessfully() { + String defaultAppId = "default-app-id", + branchName = "develop", + branchedAppId = "branched-app-id"; + + // Create a large ApplicationJson object that exceeds the 15 MB size + JSONObject jsonObject = new JSONObject(); + jsonObject.put("key", generateRandomString(16)); + + Layout layout = new Layout(); + layout.setDsl(jsonObject); + + PageDTO pageDTO = new PageDTO(); + pageDTO.setLayouts(new ArrayList<>()); + pageDTO.getLayouts().add(layout); + NewPage newPage = new NewPage(); + newPage.setUnpublishedPage(pageDTO); + + ApplicationJson applicationJson = new ApplicationJson(); + applicationJson.setPageList(List.of(newPage)); + + Mockito.when(applicationService.findBranchedApplicationId(branchName, defaultAppId, AclPermission.MANAGE_APPLICATIONS)) + .thenReturn(Mono.just(branchedAppId)); + + Mockito.when(importExportApplicationService.exportApplicationById(branchedAppId, SerialiseApplicationObjective.VERSION_CONTROL)) + .thenReturn(Mono.just(applicationJson)); + + Mockito.when(applicationSnapshotRepository.deleteAllByApplicationId(branchedAppId)) + .thenReturn(Mono.just("").then()); + + // we're expecting to receive two application snapshots, create a matcher to check the size + ArgumentMatcher> snapshotListHasTwoSnapshot = snapshotList -> snapshotList.size() == 2; + + Mockito.when(applicationSnapshotRepository.saveAll(argThat(snapshotListHasTwoSnapshot))) + .thenReturn(Flux.just(new ApplicationSnapshot(), new ApplicationSnapshot())); + + StepVerifier.create(applicationSnapshotService.createApplicationSnapshot(defaultAppId, branchName)) + .assertNext(aBoolean -> { + assertThat(aBoolean).isTrue(); + }) + .verifyComplete(); + } + + @Test + public void restoreSnapshot_WhenSnapshotHasMultipleChunks_RestoredSuccessfully() { + String defaultAppId = "default-app-id", + branchedAppId = "branched-app-id", + workspaceId = "workspace-id", + branch = "development"; + + Application application = new Application(); + application.setName("Snapshot test"); + application.setWorkspaceId(workspaceId); + application.setId(branchedAppId); + + Mockito.when(applicationService.findByBranchNameAndDefaultApplicationId(branch, defaultAppId, AclPermission.MANAGE_APPLICATIONS)) + .thenReturn(Mono.just(application)); + + ApplicationJson applicationJson = new ApplicationJson(); + applicationJson.setExportedApplication(application); + + String jsonString = gson.toJson(applicationJson); + byte[] jsonStringBytes = jsonString.getBytes(StandardCharsets.UTF_8); + + int chunkSize = jsonStringBytes.length / 3; + + List snapshots = List.of( + createSnapshot(branchedAppId, copyOfRange(jsonStringBytes, chunkSize*2, jsonStringBytes.length), 3), + createSnapshot(branchedAppId, copyOfRange(jsonStringBytes, 0, chunkSize), 1), + createSnapshot(branchedAppId, copyOfRange(jsonStringBytes, chunkSize, chunkSize*2), 2) + ); + + Mockito.when(applicationSnapshotRepository.findByApplicationId(branchedAppId)) + .thenReturn(Flux.fromIterable(snapshots)); + + // matcher to check that ApplicationJson created from chunks matches the original one + ArgumentMatcher matchApplicationJson; + matchApplicationJson = applicationJson1 -> applicationJson1.getExportedApplication().getName().equals(application.getName()); + Mockito.when(importExportApplicationService.importApplicationInWorkspace(eq(application.getWorkspaceId()), argThat(matchApplicationJson), eq(branchedAppId), eq(branch))) + .thenReturn(Mono.just(application)); + + StepVerifier.create(applicationSnapshotService.restoreSnapshot(defaultAppId, branch)) + .assertNext(application1 -> { + assertThat(application1.getName()).isEqualTo(application.getName()); + }) + .verifyComplete(); + } + + private ApplicationSnapshot createSnapshot(String applicationId, byte [] data, int chunkOrder) { + ApplicationSnapshot applicationSnapshot = new ApplicationSnapshot(); + applicationSnapshot.setApplicationId(applicationId); + applicationSnapshot.setData(data); + applicationSnapshot.setChunkOrder(chunkOrder); + return applicationSnapshot; + } + + private String generateRandomString(int targetStringSizeInMB) { + int targetSizeInBytes = targetStringSizeInMB * 1024 * 1024; + int leftLimit = 48; // numeral '0' + int rightLimit = 122; // letter 'z' + Random random = new Random(); + + String generatedString = random.ints(leftLimit, rightLimit + 1) + .filter(i -> (i <= 57 || i >= 65) && (i <= 90 || i >= 97)) + .limit(targetSizeInBytes) + .collect(StringBuilder::new, StringBuilder::appendCodePoint, StringBuilder::append) + .toString(); + + return generatedString; + } +} diff --git a/app/server/appsmith-server/src/test/java/com/appsmith/server/solutions/ImportExportApplicationServiceTests.java b/app/server/appsmith-server/src/test/java/com/appsmith/server/solutions/ImportExportApplicationServiceTests.java index fc4f7c5026..54d4239664 100644 --- a/app/server/appsmith-server/src/test/java/com/appsmith/server/solutions/ImportExportApplicationServiceTests.java +++ b/app/server/appsmith-server/src/test/java/com/appsmith/server/solutions/ImportExportApplicationServiceTests.java @@ -3642,4 +3642,72 @@ public class ImportExportApplicationServiceTests { .verifyComplete(); } + @Test + @WithUserDetails(value = "api_user") + public void importApplication_existingApplication_ApplicationReplacedWithImportedOne() { + String randomUUID = UUID.randomUUID().toString(); + Mono applicationJson = createAppJson("test_assets/ImportExportServiceTest/valid-application.json"); + + // Create the initial application + Application application = new Application(); + application.setName("Application_" + randomUUID); + application.setWorkspaceId(workspaceId); + + Mono, List, List>> importedApplication = applicationPageService.createApplication(application).flatMap(createdApp -> { + PageDTO pageDTO = new PageDTO(); + pageDTO.setApplicationId(application.getId()); + pageDTO.setName("Home Page"); + return applicationPageService.createPage(pageDTO).thenReturn(createdApp); + }) + .zipWith(applicationJson) + .flatMap(objects -> { + return importExportApplicationService.importApplicationInWorkspace(workspaceId, objects.getT2(), objects.getT1().getId(), null) + .zipWith(Mono.just(objects.getT1())); + }).flatMap(objects -> { + Application newApp = objects.getT1(); + Application oldApp = objects.getT2(); + // after import, application id should not change + assert Objects.equals(newApp.getId(), oldApp.getId()); + + Mono> pageList = newPageService.findNewPagesByApplicationId(newApp.getId(), MANAGE_PAGES).collectList(); + Mono> actionList = newActionService.findAllByApplicationIdAndViewMode(newApp.getId(), false, MANAGE_ACTIONS, null).collectList(); + Mono> actionCollectionList = actionCollectionService.findAllByApplicationIdAndViewMode(newApp.getId(), false, MANAGE_ACTIONS, null).collectList(); + return Mono.zip(Mono.just(newApp), pageList, actionList, actionCollectionList); + }); + + StepVerifier + .create(importedApplication) + .assertNext(tuple -> { + List pageList = tuple.getT2(); + List actionList = tuple.getT3(); + List actionCollectionList = tuple.getT4(); + + assertThat(pageList.size()).isEqualTo(2); + assertThat(actionList.size()).isEqualTo(3); + + List pageNames = pageList.stream() + .map(p -> p.getUnpublishedPage().getName()) + .collect(Collectors.toList()); + + List actionNames = actionList.stream() + .map(p -> p.getUnpublishedAction().getName()) + .collect(Collectors.toList()); + + List actionCollectionNames = actionCollectionList.stream() + .map(p -> p.getUnpublishedCollection().getName()) + .collect(Collectors.toList()); + + // Verify the pages after importing the application + assertThat(pageNames).contains("Page1", "Page2"); + + // Verify the actions after importing the application + assertThat(actionNames).contains("api_wo_auth", "get_users", "run"); + + // Verify the actionCollections after importing the application + assertThat(actionCollectionNames).contains("JSObject1", "JSObject2"); + }) + .verifyComplete(); + + } + } diff --git a/app/server/appsmith-server/src/test/java/com/appsmith/server/solutions/ImportExportApplicationServiceV2Tests.java b/app/server/appsmith-server/src/test/java/com/appsmith/server/solutions/ImportExportApplicationServiceV2Tests.java index 235627d2f6..fe660c5caf 100644 --- a/app/server/appsmith-server/src/test/java/com/appsmith/server/solutions/ImportExportApplicationServiceV2Tests.java +++ b/app/server/appsmith-server/src/test/java/com/appsmith/server/solutions/ImportExportApplicationServiceV2Tests.java @@ -3732,4 +3732,72 @@ public class ImportExportApplicationServiceV2Tests { }) .verifyComplete(); } + + @Test + @WithUserDetails(value = "api_user") + public void importApplication_existingApplication_ApplicationReplacedWithImportedOne() { + String randomUUID = UUID.randomUUID().toString(); + Mono applicationJson = createAppJson("test_assets/ImportExportServiceTest/valid-application.json"); + + // Create the initial application + Application application = new Application(); + application.setName("Application_" + randomUUID); + application.setWorkspaceId(workspaceId); + + Mono, List, List>> importedApplication = applicationPageService.createApplication(application).flatMap(createdApp -> { + PageDTO pageDTO = new PageDTO(); + pageDTO.setApplicationId(application.getId()); + pageDTO.setName("Home Page"); + return applicationPageService.createPage(pageDTO).thenReturn(createdApp); + }) + .zipWith(applicationJson) + .flatMap(objects -> { + return importExportApplicationService.importApplicationInWorkspace(workspaceId, objects.getT2(), objects.getT1().getId(), null) + .zipWith(Mono.just(objects.getT1())); + }).flatMap(objects -> { + Application newApp = objects.getT1(); + Application oldApp = objects.getT2(); + // after import, application id should not change + assert Objects.equals(newApp.getId(), oldApp.getId()); + + Mono> pageList = newPageService.findNewPagesByApplicationId(newApp.getId(), MANAGE_PAGES).collectList(); + Mono> actionList = newActionService.findAllByApplicationIdAndViewMode(newApp.getId(), false, MANAGE_ACTIONS, null).collectList(); + Mono> actionCollectionList = actionCollectionService.findAllByApplicationIdAndViewMode(newApp.getId(), false, MANAGE_ACTIONS, null).collectList(); + return Mono.zip(Mono.just(newApp), pageList, actionList, actionCollectionList); + }); + + StepVerifier + .create(importedApplication) + .assertNext(tuple -> { + List pageList = tuple.getT2(); + List actionList = tuple.getT3(); + List actionCollectionList = tuple.getT4(); + + assertThat(pageList.size()).isEqualTo(2); + assertThat(actionList.size()).isEqualTo(3); + + List pageNames = pageList.stream() + .map(p -> p.getUnpublishedPage().getName()) + .collect(Collectors.toList()); + + List actionNames = actionList.stream() + .map(p -> p.getUnpublishedAction().getName()) + .collect(Collectors.toList()); + + List actionCollectionNames = actionCollectionList.stream() + .map(p -> p.getUnpublishedCollection().getName()) + .collect(Collectors.toList()); + + // Verify the pages after importing the application + assertThat(pageNames).contains("Page1", "Page2"); + + // Verify the actions after importing the application + assertThat(actionNames).contains("api_wo_auth", "get_users", "run"); + + // Verify the actionCollections after importing the application + assertThat(actionCollectionNames).contains("JSObject1", "JSObject2"); + }) + .verifyComplete(); + + } }