diff --git a/app/server/appsmith-git/src/test/java/com/appsmith/git/helpers/FileUtilsImplTest.java b/app/server/appsmith-git/src/test/java/com/appsmith/git/helpers/FileUtilsImplTest.java index aba00eb4a2..faca42be0b 100644 --- a/app/server/appsmith-git/src/test/java/com/appsmith/git/helpers/FileUtilsImplTest.java +++ b/app/server/appsmith-git/src/test/java/com/appsmith/git/helpers/FileUtilsImplTest.java @@ -5,6 +5,7 @@ import com.appsmith.git.configurations.GitServiceConfig; import com.appsmith.git.service.GitExecutorImpl; import org.apache.commons.io.FileUtils; import org.eclipse.jgit.api.errors.GitAPIException; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -33,6 +34,8 @@ public class FileUtilsImplTest { @MockBean private GitExecutorImpl gitExecutor; private GitServiceConfig gitServiceConfig; + private static final String localTestDirectory = "localTestDirectory"; + private static final Path localTestDirectoryPath = Path.of(localTestDirectory); @BeforeEach public void setUp() { @@ -41,8 +44,10 @@ public class FileUtilsImplTest { fileUtils = new FileUtilsImpl(gitServiceConfig, gitExecutor); } - private static final String localTestDirectory = "localTestDirectory"; - private static final Path localTestDirectoryPath = Path.of(localTestDirectory); + @AfterEach + public void tearDown() { + this.deleteLocalTestDirectoryPath(); + } @Test public void saveApplicationRef_removeActionAndActionCollectionDirectoryCreatedInV1FileFormat_success() throws GitAPIException, IOException { @@ -110,7 +115,6 @@ public class FileUtilsImplTest { Assertions.fail("Error while scanning directory"); } - this.deleteLocalTestDirectoryPath(); } @Test @@ -164,7 +168,6 @@ public class FileUtilsImplTest { Assertions.fail("Error while scanning directory"); } - this.deleteLocalTestDirectoryPath(); } /** diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/configurations/CommonConfig.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/configurations/CommonConfig.java index 0709da4d2c..648310f5fa 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/configurations/CommonConfig.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/configurations/CommonConfig.java @@ -31,6 +31,7 @@ import java.util.Set; public class CommonConfig { private static final String ELASTIC_THREAD_POOL_NAME = "appsmith-elastic-pool"; + public static final Integer LATEST_INSTANCE_SCHEMA_VERSION = 2; @Value("${appsmith.instance.name:}") private String instanceName; @@ -67,7 +68,6 @@ public class CommonConfig { private List allowedDomains; - @Bean public Scheduler scheduler() { return Schedulers.newElastic(ELASTIC_THREAD_POOL_NAME); diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/configurations/InstanceConfig.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/configurations/InstanceConfig.java index 5b5f6bdfb8..91b6dfbb9f 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/configurations/InstanceConfig.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/configurations/InstanceConfig.java @@ -10,7 +10,10 @@ import com.appsmith.util.WebClientUtils; import io.sentry.Sentry; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.SpringApplication; import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.ApplicationContext; import org.springframework.context.ApplicationListener; import org.springframework.core.ParameterizedTypeReference; import org.springframework.stereotype.Component; @@ -18,6 +21,8 @@ import org.springframework.util.StringUtils; import org.springframework.web.reactive.function.BodyInserters; import reactor.core.publisher.Mono; +import java.util.LinkedList; +import java.util.List; import java.util.Map; import java.util.Objects; @@ -34,20 +39,64 @@ public class InstanceConfig implements ApplicationListener registrationAndRtsCheckMono = configService.getByName(Appsmith.APPSMITH_REGISTERED) .filter(config -> Boolean.TRUE.equals(config.getConfig().get("value"))) .switchIfEmpty(registerInstance()) .doOnError(errorSignal -> log.debug("Instance registration failed with error: \n{}", errorSignal.getMessage())) .then(performRtsHealthCheck()) - .doFinally(ignored -> this.printReady()) + .doFinally(ignored -> this.printReady()); + + Mono.when(checkInstanceSchemaVersion(), registrationAndRtsCheckMono) .subscribe(null, e -> { - log.debug(e.getMessage()); + log.debug("Application start up encountered an error: {}", e.getMessage()); Sentry.captureException(e); }); } + private Mono checkInstanceSchemaVersion() { + return configService.getByName(Appsmith.INSTANCE_SCHEMA_VERSION) + .onErrorMap(AppsmithException.class, e -> new AppsmithException(AppsmithError.SCHEMA_VERSION_NOT_FOUND_ERROR)) + .flatMap(config -> { + if (CommonConfig.LATEST_INSTANCE_SCHEMA_VERSION == config.getConfig().get("value")) { + return Mono.just(config); + } + return Mono.error(populateSchemaMismatchError((Integer) config.getConfig().get("value"))); + }) + .doOnError(errorSignal -> { + log.error("\n" + + "################################################\n" + + "Error while trying to start up Appsmith instance: \n" + + "{}\n" + + "################################################\n", + errorSignal.getMessage()); + + SpringApplication.exit(applicationContext, () -> 1); + System.exit(1); + }) + .then(); + } + + private AppsmithException populateSchemaMismatchError(Integer currentInstanceSchemaVersion) { + + List versions = new LinkedList<>(); + + // Keep adding version numbers that brought in breaking instance schema migrations here + switch (currentInstanceSchemaVersion) { + // Example, we expect that in v1.8.14, all instances will have been migrated to instanceSchemaVer 2 + case 1: + versions.add("v1.9"); + default: + } + + return new AppsmithException(AppsmithError.SCHEMA_MISMATCH_ERROR, versions); + } + private Mono registerInstance() { log.debug("Triggering registration of this instance..."); diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/configurations/SecurityConfig.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/configurations/SecurityConfig.java index 5707940356..0a28feda07 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/configurations/SecurityConfig.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/configurations/SecurityConfig.java @@ -21,6 +21,7 @@ import org.springframework.security.config.web.server.ServerHttpSecurity; import org.springframework.security.oauth2.client.registration.ReactiveClientRegistrationRepository; import org.springframework.security.web.server.SecurityWebFilterChain; import org.springframework.security.web.server.ServerAuthenticationEntryPoint; +import org.springframework.security.web.server.authentication.ServerAuthenticationEntryPointFailureHandler; import org.springframework.security.web.server.authentication.ServerAuthenticationFailureHandler; import org.springframework.security.web.server.authentication.ServerAuthenticationSuccessHandler; import org.springframework.security.web.server.util.matcher.ServerWebExchangeMatchers; @@ -45,7 +46,7 @@ import static com.appsmith.server.constants.Url.USER_URL; import static java.time.temporal.ChronoUnit.DAYS; @EnableWebFluxSecurity -@EnableReactiveMethodSecurity +@EnableReactiveMethodSecurity(useAuthorizationManager = true) public class SecurityConfig { @Autowired @@ -104,6 +105,8 @@ public class SecurityConfig { @Bean public SecurityWebFilterChain securityWebFilterChain(ServerHttpSecurity http) { + ServerAuthenticationEntryPointFailureHandler failureHandler = new ServerAuthenticationEntryPointFailureHandler(authenticationEntryPoint); + return http // This picks up the configurationSource from the bean corsConfigurationSource() .csrf().disable() @@ -139,25 +142,28 @@ public class SecurityConfig { .pathMatchers("/public/**", "/oauth2/**").permitAll() .anyExchange() .authenticated() - .and().httpBasic() - .and().formLogin() - .loginPage(Url.LOGIN_URL) - .authenticationEntryPoint(authenticationEntryPoint) - .requiresAuthenticationMatcher(ServerWebExchangeMatchers.pathMatchers(HttpMethod.POST, Url.LOGIN_URL)) - .authenticationSuccessHandler(authenticationSuccessHandler) - .authenticationFailureHandler(authenticationFailureHandler) + .and() + .httpBasic(httpBasicSpec -> httpBasicSpec.authenticationFailureHandler(failureHandler)) + .formLogin(formLoginSpec -> formLoginSpec.authenticationFailureHandler(failureHandler) + .loginPage(Url.LOGIN_URL) + .authenticationEntryPoint(authenticationEntryPoint) + .requiresAuthenticationMatcher(ServerWebExchangeMatchers.pathMatchers(HttpMethod.POST, Url.LOGIN_URL)) + .authenticationSuccessHandler(authenticationSuccessHandler) + .authenticationFailureHandler(authenticationFailureHandler)) // For Github SSO Login, check transformation class: CustomOAuth2UserServiceImpl // For Google SSO Login, check transformation class: CustomOAuth2UserServiceImpl - .and().oauth2Login() - .authorizationRequestResolver(new CustomServerOAuth2AuthorizationRequestResolver(reactiveClientRegistrationRepository, commonConfig, redirectHelper)) - .authenticationSuccessHandler(authenticationSuccessHandler) - .authenticationFailureHandler(authenticationFailureHandler) - .authorizedClientRepository(new ClientUserRepository(userService, commonConfig)) - .and().logout() + .oauth2Login(oAuth2LoginSpec -> oAuth2LoginSpec.authenticationFailureHandler(failureHandler) + .authorizationRequestResolver(new CustomServerOAuth2AuthorizationRequestResolver(reactiveClientRegistrationRepository, commonConfig, redirectHelper)) + .authenticationSuccessHandler(authenticationSuccessHandler) + .authenticationFailureHandler(authenticationFailureHandler) + .authorizedClientRepository(new ClientUserRepository(userService, commonConfig))) + + .logout() .logoutUrl(Url.LOGOUT_URL) .logoutSuccessHandler(new LogoutSuccessHandler(objectMapper, analyticsService)) - .and().build(); + .and() + .build(); } /** diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/constants/Appsmith.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/constants/Appsmith.java index 9d2ecc71b0..ef04c8c4df 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/constants/Appsmith.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/constants/Appsmith.java @@ -2,6 +2,7 @@ package com.appsmith.server.constants; public class Appsmith { public final static String APPSMITH_REGISTERED = "appsmith_registered"; + public final static String INSTANCE_SCHEMA_VERSION = "schemaVersion"; // We default the origin header to the production deployment of the client's URL public static final String DEFAULT_ORIGIN_HEADER = "https://app.appsmith.com"; } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/exceptions/AppsmithError.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/exceptions/AppsmithError.java index 4751d80c1f..a6a4fa0dd3 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/exceptions/AppsmithError.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/exceptions/AppsmithError.java @@ -150,6 +150,8 @@ public enum AppsmithError { ENV_FILE_NOT_FOUND(500, 5019, "Admin Settings is unavailable. Unable to read and write to Environment file.", AppsmithErrorAction.DEFAULT, null, ErrorType.CONFIGURATION_ERROR, null), PUBLIC_APP_NO_PERMISSION_GROUP(500, 5020, "Invalid state. Public application does not have the required roles set for public access. Please reach out to Appsmith customer support to resolve this.", AppsmithErrorAction.LOG_EXTERNALLY, null, ErrorType.INTERNAL_ERROR, null), RTS_SERVER_ERROR(500, 5021, "RTS server error while processing request: {0}", AppsmithErrorAction.LOG_EXTERNALLY, null, ErrorType.INTERNAL_ERROR, null), + SCHEMA_MISMATCH_ERROR(500, 5022, "Looks like you skipped some required update(s), please go back to the mandatory upgrade path {0}, or refer to ''https://docs.appsmith.com/'' for more info", AppsmithErrorAction.LOG_EXTERNALLY, null, ErrorType.INTERNAL_ERROR, null), + SCHEMA_VERSION_NOT_FOUND_ERROR(500, 5023, "Could not find mandatory instance schema version config. Please reach out to Appsmith customer support to resolve this.", AppsmithErrorAction.LOG_EXTERNALLY, null, ErrorType.INTERNAL_ERROR, null) ; private final Integer httpErrorCode; diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/DatabaseChangelog0.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/DatabaseChangelog0.java new file mode 100644 index 0000000000..9b60531b9a --- /dev/null +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/DatabaseChangelog0.java @@ -0,0 +1,52 @@ +package com.appsmith.server.migrations; + +import com.appsmith.server.configurations.CommonConfig; +import com.appsmith.server.constants.Appsmith; +import com.appsmith.server.domains.Config; +import com.appsmith.server.domains.QConfig; +import com.github.cloudyrock.mongock.ChangeLog; +import com.github.cloudyrock.mongock.ChangeSet; +import com.github.cloudyrock.mongock.driver.mongodb.springdata.v3.decorator.impl.MongockTemplate; +import lombok.extern.slf4j.Slf4j; +import net.minidev.json.JSONObject; + +import java.util.Map; + +import static com.appsmith.server.repositories.ce.BaseAppsmithRepositoryCEImpl.fieldName; +import static org.springframework.data.mongodb.core.query.Criteria.where; +import static org.springframework.data.mongodb.core.query.Query.query; + +@Slf4j +@ChangeLog(order = "000") +public class DatabaseChangelog0 { + + /** + * This migration initializes the correct version of instance schema migrations + * in the config collection for this Appsmith instance + * Future migrations that need to handle migrations for schema versions + * will check the current state and manage the migration accordingly + */ + @ChangeSet(order = "001", id = "initialize-schema-version", author = "") + public void initializeSchemaVersion(MongockTemplate mongockTemplate) { + + Config instanceIdConfig = mongockTemplate.findOne( + query(where(fieldName(QConfig.config1.name)).is("instance-id")), + Config.class); + + if (instanceIdConfig != null) { + // If instance id exists, this is an existing instance + // Instantiate with the first version so that we expect to go through all the migrations + mongockTemplate.insert(new Config( + new JSONObject(Map.of("value", 1)), + Appsmith.INSTANCE_SCHEMA_VERSION + )); + } else { + // Is no instance id exists, this is a new instance + // Instantiate with latest schema version that this Appsmith release shipped with + mongockTemplate.insert(new Config( + new JSONObject(Map.of("value", CommonConfig.LATEST_INSTANCE_SCHEMA_VERSION)), + Appsmith.INSTANCE_SCHEMA_VERSION + )); + } + } +} diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/DatabaseChangelog.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/DatabaseChangelog1.java similarity index 99% rename from app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/DatabaseChangelog.java rename to app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/DatabaseChangelog1.java index b6fcd60f05..7cdc55d560 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/DatabaseChangelog.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/DatabaseChangelog1.java @@ -139,7 +139,7 @@ import static org.springframework.data.mongodb.core.query.Update.update; @Slf4j @ChangeLog(order = "001") -public class DatabaseChangelog { +public class DatabaseChangelog1 { public static ObjectMapper objectMapper = new ObjectMapper(); private static final String AGGREGATE_LIMIT = "aggregate.limit"; @@ -2877,7 +2877,7 @@ public class DatabaseChangelog { } } - private Document getDocumentFromPath(Document document, String path) { + public static Document getDocumentFromPath(Document document, String path) { String[] pathKeys = path.split("\\."); Document documentPtr = document; diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/DatabaseChangelog2.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/DatabaseChangelog2.java index 8507808b28..a99e22c801 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/DatabaseChangelog2.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/DatabaseChangelog2.java @@ -1,5 +1,6 @@ package com.appsmith.server.migrations; +import com.appsmith.external.helpers.Stopwatch; import com.appsmith.external.models.ActionDTO; import com.appsmith.external.models.BaseDomain; import com.appsmith.external.models.Datasource; @@ -8,9 +9,12 @@ import com.appsmith.external.models.Policy; import com.appsmith.external.models.Property; import com.appsmith.external.models.QBaseDomain; import com.appsmith.external.models.QDatasource; +import com.appsmith.external.services.EncryptionService; import com.appsmith.server.acl.AclPermission; import com.appsmith.server.acl.AppsmithRole; import com.appsmith.server.acl.PolicyGenerator; +import com.appsmith.server.configurations.EncryptionConfig; +import com.appsmith.server.constants.Appsmith; import com.appsmith.server.constants.FieldName; import com.appsmith.server.domains.Action; import com.appsmith.server.domains.ActionCollection; @@ -63,14 +67,23 @@ import com.github.cloudyrock.mongock.ChangeLog; import com.github.cloudyrock.mongock.ChangeSet; import com.github.cloudyrock.mongock.driver.mongodb.springdata.v3.decorator.impl.MongockTemplate; import com.google.gson.Gson; +import com.mongodb.BasicDBObject; +import com.mongodb.client.MongoCollection; +import com.mongodb.client.MongoCursor; +import com.mongodb.client.model.Filters; import com.querydsl.core.types.Path; import io.changock.migration.api.annotations.NonLockGuarded; import lombok.extern.slf4j.Slf4j; import net.minidev.json.JSONObject; +import org.apache.commons.codec.binary.Hex; +import org.apache.commons.lang.ArrayUtils; +import org.bson.Document; +import org.bson.conversions.Bson; import org.bson.types.ObjectId; import org.springframework.core.io.DefaultResourceLoader; import org.springframework.dao.DuplicateKeyException; import org.springframework.data.domain.Sort; +import org.springframework.data.mongodb.core.CollectionCallback; import org.springframework.data.mongodb.core.aggregation.AggregationUpdate; import org.springframework.data.mongodb.core.aggregation.Fields; import org.springframework.data.mongodb.core.index.Index; @@ -79,6 +92,8 @@ import org.springframework.data.mongodb.core.query.Query; import org.springframework.data.mongodb.core.query.Update; import org.springframework.data.redis.core.ReactiveRedisOperations; import org.springframework.data.redis.core.script.RedisScript; +import org.springframework.security.crypto.encrypt.Encryptors; +import org.springframework.security.crypto.encrypt.TextEncryptor; import org.springframework.util.CollectionUtils; import org.springframework.util.ObjectUtils; import org.springframework.util.StreamUtils; @@ -117,16 +132,17 @@ import static com.appsmith.server.constants.EnvVariables.APPSMITH_ADMIN_EMAILS; import static com.appsmith.server.constants.FieldName.DEFAULT_PERMISSION_GROUP; import static com.appsmith.server.constants.FieldName.PERMISSION_GROUP_ID; import static com.appsmith.server.helpers.CollectionUtils.findSymmetricDiff; -import static com.appsmith.server.migrations.DatabaseChangelog.dropIndexIfExists; -import static com.appsmith.server.migrations.DatabaseChangelog.ensureIndexes; -import static com.appsmith.server.migrations.DatabaseChangelog.getUpdatedDynamicBindingPathList; -import static com.appsmith.server.migrations.DatabaseChangelog.installPluginToAllWorkspaces; -import static com.appsmith.server.migrations.DatabaseChangelog.makeIndex; +import static com.appsmith.server.migrations.DatabaseChangelog1.dropIndexIfExists; +import static com.appsmith.server.migrations.DatabaseChangelog1.ensureIndexes; +import static com.appsmith.server.migrations.DatabaseChangelog1.getUpdatedDynamicBindingPathList; +import static com.appsmith.server.migrations.DatabaseChangelog1.installPluginToAllWorkspaces; +import static com.appsmith.server.migrations.DatabaseChangelog1.makeIndex; import static com.appsmith.server.migrations.MigrationHelperMethods.evictPermissionCacheForUsers; import static com.appsmith.server.repositories.BaseAppsmithRepositoryImpl.fieldName; import static java.lang.Boolean.TRUE; import static org.springframework.data.mongodb.core.query.Criteria.where; import static org.springframework.data.mongodb.core.query.Query.query; +import static org.springframework.data.mongodb.core.query.Update.update; @Slf4j @@ -141,17 +157,17 @@ public class DatabaseChangelog2 { public void fixPluginTitleCasing(MongockTemplate mongockTemplate) { mongockTemplate.updateFirst( query(where(fieldName(QPlugin.plugin.packageName)).is("mysql-plugin")), - Update.update(fieldName(QPlugin.plugin.name), "MySQL"), + update(fieldName(QPlugin.plugin.name), "MySQL"), Plugin.class); mongockTemplate.updateFirst( query(where(fieldName(QPlugin.plugin.packageName)).is("mssql-plugin")), - Update.update(fieldName(QPlugin.plugin.name), "Microsoft SQL Server"), + update(fieldName(QPlugin.plugin.name), "Microsoft SQL Server"), Plugin.class); mongockTemplate.updateFirst( query(where(fieldName(QPlugin.plugin.packageName)).is("elasticsearch-plugin")), - Update.update(fieldName(QPlugin.plugin.name), "Elasticsearch"), + update(fieldName(QPlugin.plugin.name), "Elasticsearch"), Plugin.class); } @@ -756,7 +772,7 @@ public class DatabaseChangelog2 { public void setDefaultApplicationVersion(MongockTemplate mongockTemplate) { mongockTemplate.updateMulti( Query.query(where(fieldName(QApplication.application.deleted)).is(false)), - Update.update(fieldName(QApplication.application.applicationVersion), + update(fieldName(QApplication.application.applicationVersion), ApplicationVersion.EARLIEST_VERSION), Application.class); } @@ -908,7 +924,7 @@ public class DatabaseChangelog2 { if (!newName.equals(oldName)) { //Using strings in the field names instead of QSequence becauce Sequence is not a AppsmithDomain mongockTemplate.updateFirst(query(where("name").is(oldName)), - Update.update("name", newName), + update("name", newName), Sequence.class ); } @@ -1422,7 +1438,7 @@ public class DatabaseChangelog2 { @ChangeSet(order = "021", id = "flush-spring-redis-keys-2a", author = "") public void clearRedisCache2(ReactiveRedisOperations reactiveRedisOperations) { - DatabaseChangelog.doClearRedisKeys(reactiveRedisOperations); + DatabaseChangelog1.doClearRedisKeys(reactiveRedisOperations); } private List getCustomizedThemeIds(String fieldName, Function getThemeIdMethod, List systemThemeIds, MongockTemplate mongockTemplate) { @@ -2779,12 +2795,158 @@ public class DatabaseChangelog2 { dropIndexIfExists(mongockTemplate, Workspace.class, "tenantId_deleted"); ensureIndexes(mongockTemplate, Workspace.class, makeIndex("tenantId", "deleted").named("tenantId_deleted")); } - + @ChangeSet(order = "038", id = "add-unique-index-for-uidstring", author = "") public void addUniqueIndexOnUidString(MongockTemplate mongoTemplate) { Index uidStringUniqueness = makeIndex("uidString").unique() .named("customjslibs_uidstring_index"); ensureIndexes(mongoTemplate, CustomJSLib.class, uidStringUniqueness); } + + // TODO We'll be deleting this migration after upgrade to Spring 6.0 + @ChangeSet(order = "039", id = "deprecate-queryabletext-encryption", author = "") + public void deprecateQueryableTextEncryption(MongockTemplate mongockTemplate, + @NonLockGuarded EncryptionConfig encryptionConfig, + EncryptionService encryptionService) { + Stopwatch stopwatch = new Stopwatch("Instance Schema migration to v2"); + Config encryptionVersion = mongockTemplate.findOne( + query(where(fieldName(QConfig.config1.name)).is(Appsmith.INSTANCE_SCHEMA_VERSION)), + Config.class); + + if (encryptionVersion != null && (Integer) encryptionVersion.getConfig().get("value") < 2) { + String saltInHex = Hex.encodeHexString(encryptionConfig.getSalt().getBytes()); + TextEncryptor textEncryptor = Encryptors.queryableText(encryptionConfig.getPassword(), saltInHex); + + /** + * - List of attributes in datasources that need to be encoded. + * - Each path represents where the attribute exists in mongo db document. + */ + List datasourcePathList = new ArrayList<>(); + datasourcePathList.add("datasourceConfiguration.connection.ssl.keyFile.base64Content"); + datasourcePathList.add("datasourceConfiguration.connection.ssl.certificateFile.base64Content"); + datasourcePathList.add("datasourceConfiguration.connection.ssl.caCertificateFile.base64Content"); + datasourcePathList.add("datasourceConfiguration.connection.ssl.pemCertificate.file.base64Content"); + datasourcePathList.add("datasourceConfiguration.connection.ssl.pemCertificate.password"); + datasourcePathList.add("datasourceConfiguration.sshProxy.privateKey.keyFile.base64Content"); + datasourcePathList.add("datasourceConfiguration.sshProxy.privateKey.password"); + datasourcePathList.add("datasourceConfiguration.authentication.value"); + datasourcePathList.add("datasourceConfiguration.authentication.password"); + datasourcePathList.add("datasourceConfiguration.authentication.bearerToken"); + datasourcePathList.add("datasourceConfiguration.authentication.clientSecret"); + datasourcePathList.add("datasourceConfiguration.authentication.authenticationResponse.token"); + datasourcePathList.add("datasourceConfiguration.authentication.authenticationResponse.refreshToken"); + datasourcePathList.add("datasourceConfiguration.authentication.authenticationResponse.tokenResponse"); + List datasourcePathListExists = datasourcePathList + .stream() + .map(Filters::exists) + .collect(Collectors.toList()); + + List gitDeployKeysPathListExists = new ArrayList<>(); + ArrayList gitDeployKeysPathList = new ArrayList<>(); + gitDeployKeysPathList.add("gitAuth.privateKey"); + gitDeployKeysPathListExists.add(Filters.exists("gitAuth.privateKey")); + + List applicationPathListExists = new ArrayList<>(); + ArrayList applicationPathList = new ArrayList<>(); + applicationPathList.add("gitApplicationMetadata.gitAuth.privateKey"); + applicationPathListExists.add(Filters.exists("gitApplicationMetadata.gitAuth.privateKey")); + + mongockTemplate.execute("datasource", getNewEncryptionCallback(textEncryptor, encryptionService, datasourcePathListExists, datasourcePathList, stopwatch)); + mongockTemplate.execute("gitDeployKeys", getNewEncryptionCallback(textEncryptor, encryptionService, gitDeployKeysPathListExists, gitDeployKeysPathList, stopwatch)); + mongockTemplate.execute("application", getNewEncryptionCallback(textEncryptor, encryptionService, applicationPathListExists, applicationPathList, stopwatch)); + + mongockTemplate.upsert( + query(where(fieldName(QConfig.config1.name)).is(Appsmith.INSTANCE_SCHEMA_VERSION)), + update("config.value", 2), + Config.class); + } + stopwatch.stopAndLogTimeInMillis(); + } + + private CollectionCallback getNewEncryptionCallback( + TextEncryptor textEncryptor, + EncryptionService encryptionService, + Iterable collectionFilterIterable, + List pathList, + Stopwatch stopwatch) { + return new CollectionCallback() { + @Override + public String doInCollection(MongoCollection collection) { + MongoCursor cursor = collection + .find( + Filters.and( + Filters.or(collectionFilterIterable), + Filters.not(Filters.exists("encryptionVersion")))) + .cursor(); + + log.debug("collection callback start: {}ms", stopwatch.getExecutionTime()); + + List> documentPairList = new ArrayList<>(); + while (cursor.hasNext()) { + Document old = cursor.next(); + BasicDBObject query = new BasicDBObject(); + query.put("_id", old.getObjectId("_id")); + // This document will have the encrypted values. + BasicDBObject updated = new BasicDBObject(); + updated.put("$set", new BasicDBObject("encryptionVersion", 2)); + updated.put("$unset", new BasicDBObject()); + // Encrypt attributes + pathList.stream() + .forEach(path -> reapplyNewEncryptionToPathValueIfExists(old, updated, path, encryptionService, textEncryptor)); + documentPairList.add(List.of(query, updated)); + } + + log.debug("collection callback processing end: {}ms", stopwatch.getExecutionTime()); + log.debug("update will be run for {} documents", documentPairList.size()); + + /** + * - Replace old document with the updated document that has encrypted values. + * - Replacing here instead of the while loop above makes sure that we attempt replacement only if + * the encryption step succeeded without error for each selected document. + */ + documentPairList.stream().parallel() + .forEach(docPair -> collection.updateOne(docPair.get(0), docPair.get(1))); + + log.debug("collection callback update end: {}ms", stopwatch.getExecutionTime()); + + return null; + } + }; + } + + private void reapplyNewEncryptionToPathValueIfExists(Document document, BasicDBObject update, String path, + EncryptionService encryptionService, + TextEncryptor textEncryptor) { + String[] pathKeys = path.split("\\."); + /** + * - For attribute path "datasourceConfiguration.connection.ssl.keyFile.base64Content", first get the parent + * document that contains the attribute 'base64Content' i.e. fetch the document corresponding to path + * "datasourceConfiguration.connection.ssl.keyFile" + */ + String parentDocumentPath = org.apache.commons.lang.StringUtils.join(ArrayUtils.subarray(pathKeys, 0, pathKeys.length - 1), "."); + Document parentDocument = DatabaseChangelog1.getDocumentFromPath(document, parentDocumentPath); + + if (parentDocument != null) { + if (parentDocument.containsKey(pathKeys[pathKeys.length - 1])) { + String oldEncryptedValue = parentDocument.getString(pathKeys[pathKeys.length - 1]); + if (StringUtils.hasLength(String.valueOf(oldEncryptedValue))) { + String decryptedValue = null; + try { + decryptedValue = textEncryptor.decrypt(String.valueOf(oldEncryptedValue)); + } catch (IllegalArgumentException e) { + // This happens on release DB for some creds that are malformed + if ("Hex-encoded string must have an even number of characters".equals(e.getMessage())) { + decryptedValue = String.valueOf(oldEncryptedValue); + } + } + String newEncryptedValue = encryptionService.encryptString(decryptedValue); + ((BasicDBObject) update.get("$set")).put(path, newEncryptedValue); + if (path.startsWith("datasourceConfiguration.authentication")) { + ((BasicDBObject) update.get("$unset")).put("datasourceConfiguration.authentication.isEncrypted", 1); + } + } + } + } + } } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/ApplicationPageServiceCEImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/ApplicationPageServiceCEImpl.java index a152613483..7b31e5f9ee 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/ApplicationPageServiceCEImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/ApplicationPageServiceCEImpl.java @@ -23,8 +23,8 @@ import com.appsmith.server.domains.User; import com.appsmith.server.domains.Workspace; import com.appsmith.server.dtos.ActionCollectionDTO; import com.appsmith.server.dtos.ApplicationPagesDTO; -import com.appsmith.server.dtos.PageDTO; import com.appsmith.server.dtos.CustomJSLibApplicationDTO; +import com.appsmith.server.dtos.PageDTO; import com.appsmith.server.dtos.PageNameIdDTO; import com.appsmith.server.exceptions.AppsmithError; import com.appsmith.server.exceptions.AppsmithException; @@ -63,8 +63,8 @@ import java.nio.file.Paths; import java.time.Instant; import java.util.ArrayList; import java.util.HashMap; -import java.util.List; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -72,7 +72,6 @@ import java.util.stream.Collectors; import static com.appsmith.external.helpers.AppsmithBeanUtils.copyNestedNonNullProperties; import static org.apache.commons.lang.ObjectUtils.defaultIfNull; -import static com.appsmith.server.acl.AclPermission.MANAGE_APPLICATIONS; @Slf4j @@ -424,7 +423,7 @@ public class ApplicationPageServiceCEImpl implements ApplicationPageServiceCE { GitApplicationMetadata gitData = application.getGitApplicationMetadata(); if (gitData != null && !StringUtils.isEmpty(gitData.getDefaultApplicationId()) && !StringUtils.isEmpty(gitData.getRepoName())) { String repoName = gitData.getRepoName(); - Path repoPath = Paths.get(application.getOrganizationId(), gitData.getDefaultApplicationId(), repoName); + Path repoPath = Paths.get(application.getWorkspaceId(), gitData.getDefaultApplicationId(), repoName); // Delete git repo from local return gitFileUtils.deleteLocalRepo(repoPath) .then(Mono.just(application)); @@ -1076,14 +1075,15 @@ public class ApplicationPageServiceCEImpl implements ApplicationPageServiceCE { Mono> publishedActionsFlux, Mono> publishedActionsCollectionFlux, Mono> publishedJSLibDTOsMono, - String applicationId, boolean isPublishedManually) { return Mono.zip( + String applicationId, boolean isPublishedManually) { + return Mono.zip( publishApplicationAndPages, publishedActionsFlux, publishedActionsCollectionFlux, // not using existing applicationMono because we need the latest Application after published applicationService.findById(applicationId, applicationPermission.getEditPermission()), publishedJSLibDTOsMono - ) + ) .flatMap(objects -> { Application application = objects.getT4(); Map extraProperties = new HashMap<>(); diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/EncryptionServiceCEImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/EncryptionServiceCEImpl.java index b330e9b0dd..673c3db41c 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/EncryptionServiceCEImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/EncryptionServiceCEImpl.java @@ -18,8 +18,7 @@ public class EncryptionServiceCEImpl implements EncryptionServiceCE { public EncryptionServiceCEImpl(EncryptionConfig encryptionConfig) { this.encryptionConfig = encryptionConfig; String saltInHex = Hex.encodeHexString(encryptionConfig.getSalt().getBytes()); - this.textEncryptor = Encryptors.queryableText(encryptionConfig.getPassword(), - saltInHex); + this.textEncryptor = Encryptors.delux(encryptionConfig.getPassword(), saltInHex); } @Override diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/UserServiceCEImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/UserServiceCEImpl.java index d3d915d25b..def4b785d6 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/UserServiceCEImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/UserServiceCEImpl.java @@ -237,7 +237,8 @@ public class UserServiceCEImpl extends BaseService resetToken.setTokenHash(passwordEncoder.encode(token)); return resetToken; }); - }).flatMap(passwordResetTokenRepository::save) + }) + .flatMap(passwordResetTokenRepository::save) .flatMap(passwordResetToken -> { log.debug("Password reset Token: {} for email: {}", token, passwordResetToken.getEmail()); @@ -302,7 +303,7 @@ public class UserServiceCEImpl extends BaseService EmailTokenDTO emailTokenDTO; try { emailTokenDTO = parseValueFromEncryptedToken(encryptedToken); - } catch (IllegalStateException e) { + } catch (ArrayIndexOutOfBoundsException | IllegalStateException e) { return Mono.error(new AppsmithException(AppsmithError.INVALID_PARAMETER, FieldName.TOKEN)); } @@ -325,7 +326,7 @@ public class UserServiceCEImpl extends BaseService EmailTokenDTO emailTokenDTO; try { emailTokenDTO = parseValueFromEncryptedToken(encryptedToken); - } catch (IllegalStateException e) { + } catch (ArrayIndexOutOfBoundsException | IllegalStateException e) { return Mono.error(new AppsmithException(AppsmithError.INVALID_PARAMETER, FieldName.TOKEN)); } @@ -599,10 +600,9 @@ public class UserServiceCEImpl extends BaseService } - @Override public Mono createNewUserAndSendInviteEmail(String email, String originHeader, - Workspace workspace, User inviter, String role) { + Workspace workspace, User inviter, String role) { User newUser = new User(); newUser.setEmail(email.toLowerCase()); diff --git a/app/server/appsmith-server/src/main/resources/application.properties b/app/server/appsmith-server/src/main/resources/application.properties index f47a56adb5..93614c238f 100644 --- a/app/server/appsmith-server/src/main/resources/application.properties +++ b/app/server/appsmith-server/src/main/resources/application.properties @@ -1,4 +1,8 @@ server.port=${PORT:8080} +# Allow the Spring context to close all active requests before shutting down the server +# Please ref: https://docs.spring.io/spring-boot/docs/2.3.0.RELEASE/reference/html/spring-boot-features.html#boot-features-graceful-shutdown +server.shutdown=graceful +spring.lifecycle.timeout-per-shutdown-phase=20s # This property allows us to override beans during testing. This is useful when we want to set different configurations # and different parameters during test as compared to production. If this property is disabled, some tests will fail. diff --git a/app/server/appsmith-server/src/test/java/com/appsmith/server/services/DatasourceContextServiceTest.java b/app/server/appsmith-server/src/test/java/com/appsmith/server/services/DatasourceContextServiceTest.java index ae700ce883..58cfc1b817 100644 --- a/app/server/appsmith-server/src/test/java/com/appsmith/server/services/DatasourceContextServiceTest.java +++ b/app/server/appsmith-server/src/test/java/com/appsmith/server/services/DatasourceContextServiceTest.java @@ -166,7 +166,7 @@ public class DatasourceContextServiceTest { DBAuth authentication = (DBAuth) savedDatasource.getDatasourceConfiguration().getAuthentication(); assertEquals(password, authentication.getPassword()); DBAuth encryptedAuthentication = (DBAuth) createdDatasource.getDatasourceConfiguration().getAuthentication(); - assertEquals(encryptionService.encryptString(password), encryptedAuthentication.getPassword()); + assertEquals(password, encryptionService.decryptString(encryptedAuthentication.getPassword())); }) .verifyComplete(); } diff --git a/app/server/appsmith-server/src/test/java/com/appsmith/server/services/DatasourceServiceTest.java b/app/server/appsmith-server/src/test/java/com/appsmith/server/services/DatasourceServiceTest.java index 42c991f454..9cf43ac8aa 100644 --- a/app/server/appsmith-server/src/test/java/com/appsmith/server/services/DatasourceServiceTest.java +++ b/app/server/appsmith-server/src/test/java/com/appsmith/server/services/DatasourceServiceTest.java @@ -862,7 +862,7 @@ public class DatasourceServiceTest { .assertNext(savedDatasource -> { DBAuth authentication = (DBAuth) savedDatasource.getDatasourceConfiguration().getAuthentication(); assertThat(authentication.getUsername()).isEqualTo(username); - assertThat(authentication.getPassword()).isEqualTo(encryptionService.encryptString(password)); + assertThat(encryptionService.decryptString(authentication.getPassword())).isEqualTo(password); }) .verifyComplete(); } @@ -961,7 +961,7 @@ public class DatasourceServiceTest { DBAuth authentication = (DBAuth) updatedDatasource.getDatasourceConfiguration().getAuthentication(); assertThat(authentication.getUsername()).isEqualTo(username); - assertThat(encryptionService.encryptString(password)).isEqualTo(authentication.getPassword()); + assertThat(password).isEqualTo(encryptionService.decryptString(authentication.getPassword())); }) .verifyComplete(); } diff --git a/app/server/pom.xml b/app/server/pom.xml index 41f1134f3d..11bc53fbde 100644 --- a/app/server/pom.xml +++ b/app/server/pom.xml @@ -5,7 +5,7 @@ org.springframework.boot spring-boot-starter-parent - 2.6.5 + 2.7.1 @@ -24,6 +24,7 @@ 1.0-SNAPSHOT true + 5.8.0 2.17.1 2.1.210 1.17.3