chore: refactor customJs lib import flow for pg transaction support (#34622)

## Description
Please refer this document for more details -
https://www.notion.so/appsmith/Transaction-Handling-in-PG-468cf8d4255749c3915699e59e91dc2f



## Automation

/ok-to-test tags="@tag.Git, @tag.ImportExport, @tag.Templates"

### 🔍 Cypress test results
<!-- This is an auto-generated comment: Cypress test results  -->
> [!TIP]
> 🟢 🟢 🟢 All cypress tests have passed! 🎉 🎉 🎉
> Workflow run:
<https://github.com/appsmithorg/appsmith/actions/runs/9855863943>
> Commit: b9dfe1d758410e831f751f1fe1047373c1137513
> <a
href="https://internal.appsmith.com/app/cypress-dashboard/rundetails-65890b3c81d7400d08fa9ee5?branch=master&workflowId=9855863943&attempt=1"
target="_blank">Cypress dashboard</a>.
> Tags: `@tag.Git, @tag.ImportExport, @tag.Templates`
> Spec:
> <hr>Tue, 09 Jul 2024 11:25:30 UTC
<!-- end of auto-generated comment: Cypress test results  -->


## Communication
Should the DevRel and Marketing teams inform users about this change?
- [ ] Yes
- [ ] No


<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

- **New Features**
- Introduced support for managing custom JavaScript libraries, including
dry run operations.
- **Enhancements**
- Enhanced dry operation handling with new methods and improved logic
for managing custom JavaScript libraries.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
Anagh Hegde 2024-07-09 17:15:38 +05:30 committed by GitHub
parent f17036bd52
commit 09410143d3
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 64 additions and 2 deletions

View File

@ -3,6 +3,7 @@ package com.appsmith.server.dtos.ce;
import com.appsmith.external.models.Datasource; import com.appsmith.external.models.Datasource;
import com.appsmith.external.models.DatasourceStorage; import com.appsmith.external.models.DatasourceStorage;
import com.appsmith.server.domains.Context; import com.appsmith.server.domains.Context;
import com.appsmith.server.domains.CustomJSLib;
import com.appsmith.server.dtos.CustomJSLibContextDTO; import com.appsmith.server.dtos.CustomJSLibContextDTO;
import com.appsmith.server.dtos.ImportActionCollectionResultDTO; import com.appsmith.server.dtos.ImportActionCollectionResultDTO;
import com.appsmith.server.dtos.ImportActionResultDTO; import com.appsmith.server.dtos.ImportActionResultDTO;
@ -50,4 +51,6 @@ public class MappedImportableResourcesCE_DTO {
Map<String, List<Datasource>> datasourceDryRunQueries = new HashMap<>(); Map<String, List<Datasource>> datasourceDryRunQueries = new HashMap<>();
Map<String, List<DatasourceStorage>> datasourceStorageDryRunQueries = new HashMap<>(); Map<String, List<DatasourceStorage>> datasourceStorageDryRunQueries = new HashMap<>();
Map<String, List<CustomJSLib>> customJSLibsDryOps = new HashMap<>();
} }

View File

@ -9,6 +9,7 @@ import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono; import reactor.core.publisher.Mono;
import java.util.List; import java.util.List;
import java.util.Map;
import java.util.Set; import java.util.Set;
public interface CustomJSLibServiceCE extends CrudService<CustomJSLib, String> { public interface CustomJSLibServiceCE extends CrudService<CustomJSLib, String> {
@ -32,6 +33,12 @@ public interface CustomJSLibServiceCE extends CrudService<CustomJSLib, String> {
Mono<CustomJSLibContextDTO> persistCustomJSLibMetaDataIfDoesNotExistAndGetDTO( Mono<CustomJSLibContextDTO> persistCustomJSLibMetaDataIfDoesNotExistAndGetDTO(
CustomJSLib jsLib, Boolean isForceInstall); CustomJSLib jsLib, Boolean isForceInstall);
Mono<CustomJSLibContextDTO> persistCustomJSLibMetaDataIfDoesNotExistAndGetDTO(
CustomJSLib jsLib,
Boolean isForceInstall,
Map<String, List<CustomJSLib>> customJSLibsDryOps,
boolean isDryOps);
Flux<CustomJSLib> getAllVisibleJSLibsInContext( Flux<CustomJSLib> getAllVisibleJSLibsInContext(
@NotNull String contextId, CreatorContextType contextType, String branchName, Boolean isViewMode); @NotNull String contextId, CreatorContextType contextType, String branchName, Boolean isViewMode);
} }

View File

@ -4,6 +4,7 @@ import com.appsmith.external.models.CreatorContextType;
import com.appsmith.server.domains.Application; import com.appsmith.server.domains.Application;
import com.appsmith.server.domains.CustomJSLib; import com.appsmith.server.domains.CustomJSLib;
import com.appsmith.server.dtos.CustomJSLibContextDTO; import com.appsmith.server.dtos.CustomJSLibContextDTO;
import com.appsmith.server.dtos.DBOpsType;
import com.appsmith.server.jslibs.context.ContextBasedJsLibService; import com.appsmith.server.jslibs.context.ContextBasedJsLibService;
import com.appsmith.server.repositories.CustomJSLibRepository; import com.appsmith.server.repositories.CustomJSLibRepository;
import com.appsmith.server.services.AnalyticsService; import com.appsmith.server.services.AnalyticsService;
@ -14,8 +15,10 @@ import lombok.extern.slf4j.Slf4j;
import reactor.core.publisher.Flux; import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono; import reactor.core.publisher.Mono;
import java.util.ArrayList;
import java.util.Comparator; import java.util.Comparator;
import java.util.List; import java.util.List;
import java.util.Map;
import java.util.Set; import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
@ -73,11 +76,27 @@ public class CustomJSLibServiceCEImpl extends BaseService<CustomJSLibRepository,
@Override @Override
public Mono<CustomJSLibContextDTO> persistCustomJSLibMetaDataIfDoesNotExistAndGetDTO( public Mono<CustomJSLibContextDTO> persistCustomJSLibMetaDataIfDoesNotExistAndGetDTO(
CustomJSLib jsLib, Boolean isForceInstall) { CustomJSLib jsLib, Boolean isForceInstall) {
return persistCustomJSLibMetaDataIfDoesNotExistAndGetDTO(jsLib, isForceInstall, null, false);
}
@Override
public Mono<CustomJSLibContextDTO> persistCustomJSLibMetaDataIfDoesNotExistAndGetDTO(
CustomJSLib jsLib,
Boolean isForceInstall,
Map<String, List<CustomJSLib>> customJSLibsDryOps,
boolean isDryOps) {
return repository return repository
.findUniqueCustomJsLib(jsLib) .findUniqueCustomJsLib(jsLib)
// Read more why Mono.defer is used here. // Read more why Mono.defer is used here.
// https://stackoverflow.com/questions/54373920/mono-switchifempty-is-always-called // https://stackoverflow.com/questions/54373920/mono-switchifempty-is-always-called
.switchIfEmpty(Mono.defer(() -> repository.save(jsLib))) .switchIfEmpty(Mono.defer(() -> {
if (isDryOps) {
jsLib.updateForBulkWriteOperation();
addDryOpsForEntity(DBOpsType.SAVE.name(), customJSLibsDryOps, jsLib);
return Mono.just(jsLib);
}
return repository.save(jsLib);
}))
.flatMap(foundJSLib -> { .flatMap(foundJSLib -> {
/* /*
The first check is to make sure that we are able to detect any previously truncated data and overwrite it the next time we receive valid data. The first check is to make sure that we are able to detect any previously truncated data and overwrite it the next time we receive valid data.
@ -86,6 +105,10 @@ public class CustomJSLibServiceCEImpl extends BaseService<CustomJSLibRepository,
*/ */
if ((jsLib.getDefs().length() > foundJSLib.getDefs().length()) || isForceInstall) { if ((jsLib.getDefs().length() > foundJSLib.getDefs().length()) || isForceInstall) {
jsLib.setId(foundJSLib.getId()); jsLib.setId(foundJSLib.getId());
if (isDryOps) {
addDryOpsForEntity(DBOpsType.SAVE.name(), customJSLibsDryOps, jsLib);
return Mono.just(jsLib);
}
return repository.save(jsLib); return repository.save(jsLib);
} }
@ -141,4 +164,15 @@ public class CustomJSLibServiceCEImpl extends BaseService<CustomJSLibRepository,
.getAllVisibleJSLibContextDTOFromContext(contextId, branchName, isViewMode) .getAllVisibleJSLibContextDTOFromContext(contextId, branchName, isViewMode)
.flatMapMany(repository::findCustomJsLibsInContext); .flatMapMany(repository::findCustomJsLibsInContext);
} }
private void addDryOpsForEntity(
String queryType, Map<String, List<CustomJSLib>> dryRunOpsMap, CustomJSLib createdCustomJsLib) {
if (dryRunOpsMap.containsKey(queryType)) {
dryRunOpsMap.get(queryType).add(createdCustomJsLib);
} else {
List<CustomJSLib> customJsLibList = new ArrayList<>();
customJsLibList.add(createdCustomJsLib);
dryRunOpsMap.put(queryType, customJsLibList);
}
}
} }

View File

@ -2,6 +2,7 @@ package com.appsmith.server.repositories;
import com.appsmith.external.models.Datasource; import com.appsmith.external.models.Datasource;
import com.appsmith.external.models.DatasourceStorage; import com.appsmith.external.models.DatasourceStorage;
import com.appsmith.server.domains.CustomJSLib;
import com.appsmith.server.dtos.MappedImportableResourcesDTO; import com.appsmith.server.dtos.MappedImportableResourcesDTO;
import jakarta.annotation.PostConstruct; import jakarta.annotation.PostConstruct;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
@ -22,6 +23,8 @@ public class DryOperationRepository {
private final DatasourceStorageRepository datasourceStorageRepository; private final DatasourceStorageRepository datasourceStorageRepository;
private final CustomJSLibRepository customJSLibRepository;
private Map<Class<?>, AppsmithRepository<?>> repoByEntityClass; private Map<Class<?>, AppsmithRepository<?>> repoByEntityClass;
@PostConstruct @PostConstruct
@ -29,6 +32,7 @@ public class DryOperationRepository {
final Map<Class<?>, AppsmithRepository<?>> map = new HashMap<>(); final Map<Class<?>, AppsmithRepository<?>> map = new HashMap<>();
map.put(Datasource.class, datasourceRepository); map.put(Datasource.class, datasourceRepository);
map.put(DatasourceStorage.class, datasourceStorageRepository); map.put(DatasourceStorage.class, datasourceStorageRepository);
map.put(CustomJSLib.class, customJSLibRepository);
repoByEntityClass = Collections.unmodifiableMap(map); repoByEntityClass = Collections.unmodifiableMap(map);
} }
@ -44,6 +48,10 @@ public class DryOperationRepository {
return datasourceStorageRepository.saveAll(datasourceStorage); return datasourceStorageRepository.saveAll(datasourceStorage);
} }
private Flux<CustomJSLib> saveCustomJSLibToDb(List<CustomJSLib> customJSLibs) {
return customJSLibRepository.saveAll(customJSLibs);
}
public Mono<Void> executeAllDbOps(MappedImportableResourcesDTO mappedImportableResourcesDTO) { public Mono<Void> executeAllDbOps(MappedImportableResourcesDTO mappedImportableResourcesDTO) {
Flux<List<Datasource>> datasourceFLux = Flux.fromIterable(mappedImportableResourcesDTO Flux<List<Datasource>> datasourceFLux = Flux.fromIterable(mappedImportableResourcesDTO
@ -65,6 +73,16 @@ public class DryOperationRepository {
.get(key); .get(key);
return saveDatasourceStorageToDb(datasourceStorageList).collectList(); return saveDatasourceStorageToDb(datasourceStorageList).collectList();
}); });
return Flux.merge(datasourceFLux, datasourceStorageFLux).then();
Flux<List<CustomJSLib>> customJSLibFLux = Flux.fromIterable(
mappedImportableResourcesDTO.getCustomJSLibsDryOps().keySet())
.flatMap(key -> {
List<CustomJSLib> customJSLibList =
mappedImportableResourcesDTO.getCustomJSLibsDryOps().get(key);
return saveCustomJSLibToDb(customJSLibList).collectList();
});
return Flux.merge(datasourceFLux, datasourceStorageFLux, customJSLibFLux)
.then();
} }
} }