diff --git a/app/server/appsmith-server/pom.xml b/app/server/appsmith-server/pom.xml index 96fe6643b4..0ca001dd78 100644 --- a/app/server/appsmith-server/pom.xml +++ b/app/server/appsmith-server/pom.xml @@ -188,6 +188,13 @@ httpclient 4.5.10 + + + + com.github.mongobee + mongobee + 0.13 + diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/configurations/MongoConfig.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/configurations/MongoConfig.java index fbe76b0476..86113aa79e 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/configurations/MongoConfig.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/configurations/MongoConfig.java @@ -1,10 +1,22 @@ package com.appsmith.server.configurations; import com.appsmith.server.configurations.mongo.SoftDeleteMongoRepositoryFactoryBean; +import com.appsmith.server.migrations.DatabaseChangelog; import com.appsmith.server.repositories.BaseRepositoryImpl; +import com.github.mongobee.Mongobee; +import com.github.mongobee.exception.MongobeeException; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Configuration; +import org.springframework.core.env.Environment; +import org.springframework.core.env.Profiles; import org.springframework.data.mongodb.config.EnableMongoAuditing; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.data.mongodb.core.mapping.MongoMappingContext; import org.springframework.data.mongodb.repository.config.EnableReactiveMongoRepositories; +import org.springframework.util.StringUtils; + +import javax.annotation.PostConstruct; /** * This configures the JPA Mongo repositories. The default base implementation is defined in {@link BaseRepositoryImpl}. @@ -13,6 +25,7 @@ import org.springframework.data.mongodb.repository.config.EnableReactiveMongoRep * The factoryBean class is also custom defined in order to add default clauses for soft delete for all custom JPA queries. * {@link SoftDeleteMongoRepositoryFactoryBean} for details. */ +@Slf4j @Configuration @EnableMongoAuditing @EnableReactiveMongoRepositories(repositoryFactoryBeanClass = SoftDeleteMongoRepositoryFactoryBean.class, @@ -20,4 +33,74 @@ import org.springframework.data.mongodb.repository.config.EnableReactiveMongoRep repositoryBaseClass = BaseRepositoryImpl.class ) public class MongoConfig { + + @Value("${spring.data.mongodb.uri:}") + private String mongoDbUri; + + @Value("${spring.data.mongodb.username:}") + private String mongoDbUsername; + + @Value("${spring.data.mongodb.password:}") + private String mongoDbPassword; + + @Value("${spring.data.mongodb.host:}:${spring.data.mongodb.port:}/${spring.data.mongodb.database:}") + private String mongoDbResource; + + private final MongoTemplate mongoTemplate; + + private MongoMappingContext mongoMappingContext; + + private Environment environment; + + public MongoConfig(MongoTemplate mongoTemplate, MongoMappingContext mongoMappingContext, Environment environment) { + this.mongoTemplate = mongoTemplate; + this.mongoMappingContext = mongoMappingContext; + this.environment = environment; + } + + @PostConstruct + public void init() { + if (!environment.acceptsProfiles(Profiles.of("test"))) { + try { + runMigrations(); + } catch (MongobeeException e) { + log.error("Failed running migrations automatically at startup.", e); + } + } + } + + public void runMigrations() throws MongobeeException { + String uri; + + if (StringUtils.isEmpty(mongoDbUri)) { + uri = "mongodb://" + + mongoDbUsername + + (mongoDbPassword.isEmpty() ? "" : ":") + + mongoDbPassword + + (mongoDbUsername.isEmpty() ? "" : "@") + + mongoDbResource; + } else { + uri = mongoDbUri; + } + + runMigrations(uri); + } + + public void runMigrations(String uri) throws MongobeeException { + // Mongobee creates its own connection to MongoDB and (hopefully) closes it when migration execution is done. + // If there's no new migrations to be applied, no connection is made. However, note that the `autoCreateIndexes` + // migration method is always run, so a connection is always made. + // Also, this Mongobee runner cannot be a separate bean, since then the `.execute` will be called automatically. + // We need to run it manually so we can control it during tests. + Mongobee runner = new Mongobee(uri); + runner.setSpringEnvironment(environment); + runner.setChangeLogsScanPackage(getClass().getPackageName().replaceFirst("\\.[^.]+$", ".migrations")); + runner.setMongoTemplate(mongoTemplate); + + DatabaseChangelog.mongoMappingContext = mongoMappingContext; + + log.info("Executing MongoDB migrations"); + runner.execute(); + } + } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/domains/Organization.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/domains/Organization.java index 0228a446ab..3762bed455 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/domains/Organization.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/domains/Organization.java @@ -22,7 +22,6 @@ public class Organization extends BaseDomain { private String domain; @NotNull - @Indexed(unique = true) private String name; private String website; @@ -31,4 +30,16 @@ public class Organization extends BaseDomain { private List plugins; + @NotNull + @Indexed(unique = true) + private String slug; + + public String getSlug() { + return slug == null ? toSlug(name) : slug; + } + + public static String toSlug(String text) { + return text == null ? null : text.replaceAll("[^\\w\\d]+", "-").toLowerCase(); + } + } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/DatabaseChangelog.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/DatabaseChangelog.java new file mode 100644 index 0000000000..464ad28698 --- /dev/null +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/DatabaseChangelog.java @@ -0,0 +1,189 @@ +package com.appsmith.server.migrations; + +import com.appsmith.server.domains.*; +import com.github.mongobee.changeset.ChangeLog; +import com.github.mongobee.changeset.ChangeSet; +import com.mongodb.DuplicateKeyException; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.domain.Sort; +import org.springframework.data.mongodb.UncategorizedMongoDbException; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.data.mongodb.core.index.CompoundIndexDefinition; +import org.springframework.data.mongodb.core.index.Index; +import org.springframework.data.mongodb.core.index.MongoPersistentEntityIndexResolver; +import org.springframework.data.mongodb.core.mapping.BasicMongoPersistentEntity; +import org.springframework.data.mongodb.core.mapping.Document; +import org.springframework.data.mongodb.core.mapping.MongoMappingContext; + +import java.util.concurrent.TimeUnit; + +@Slf4j +@ChangeLog(order = "001") +public class DatabaseChangelog { + + // This value is set from ServerApplication before the migrations run. This appears to be the only reliable way to + // inject Spring beans into this class. May be there will be a better way to do this once Mongobee's Spring + // integration is improved. + // - If we add @Autowired to the `autoCreateIndexes` method to get this, then the migration method will run but + // Mongobee won't save that fact in it's own collection, and will run this migration method at every server startup. + // - If we add @Autowired to this field here, we have to turn the class into a Spring Component, but then Mongobee + // creates its own instance of this class, which would have this field set to `null`. + public static MongoMappingContext mongoMappingContext; + + /** + * A private, pure utility function to create instances of Index objects to pass to `IndexOps.ensureIndex` method. + */ + private static Index makeIndex(String... fields) { + if (fields.length == 1) { + return new Index(fields[0], Sort.Direction.ASC).named(fields[0]); + } else { + org.bson.Document doc = new org.bson.Document(); + for (String field : fields) { + doc.put(field, 1); + } + return new CompoundIndexDefinition(doc); + } + } + + /** + * Given a MongoTemplate, a domain class and a bunch of Index definitions, this pure utility function will ensure + * those indexes on the database behind the MongoTemplate instance. + */ + private static void ensureIndexes(MongoTemplate mongoTemplate, Class entityClass, Index... indexes) { + var indexOps = mongoTemplate.indexOps(entityClass); + for (Index index : indexes) { + indexOps.ensureIndex(index); + } + } + + @ChangeSet(order = "001", id = "initial-plugins", author = "") + public void initialPlugins(MongoTemplate mongoTemplate) { + Plugin plugin1 = new Plugin(); + plugin1.setName("PostgresDbPlugin"); + plugin1.setType(PluginType.DB); + plugin1.setPackageName("postgres-plugin"); + plugin1.setUiComponent("DbEditorForm"); + plugin1.setDefaultInstall(true); + try { + mongoTemplate.insert(plugin1); + } catch (DuplicateKeyException e) { + log.warn("postgres-plugin already present in database.", e); + } + + Plugin plugin2 = new Plugin(); + plugin2.setName("RestTemplatePluginExecutor"); + plugin2.setType(PluginType.API); + plugin2.setPackageName("restapi-plugin"); + plugin2.setUiComponent("ApiEditorForm"); + plugin2.setDefaultInstall(true); + try { + mongoTemplate.insert(plugin2); + } catch (DuplicateKeyException e) { + log.warn("restapi-plugin already present in database.", e); + } + } + + @ChangeSet(order = "002", id = "remove-org-name-index", author = "") + public void removeOrgNameIndex(MongoTemplate mongoTemplate) { + try { + mongoTemplate.indexOps(Organization.class).dropIndex("name"); + } catch (UncategorizedMongoDbException ignored) { + // The index probably doesn't exist. This happens if the database is created after the @Indexed annotation + // has been removed. + } + } + + @ChangeSet(order = "003", id = "add-org-slugs", author = "") + public void addOrgSlugs(MongoTemplate mongoTemplate) { + // For all existing organizations, add a slug field, which should be unique. + for (Organization organization : mongoTemplate.findAll(Organization.class)) { + organization.setSlug(organization.getSlug()); + mongoTemplate.save(organization); + } + } + + /** + * We are creating indexes manually because Spring's index resolver creates indexes on fields as well. + * See https://stackoverflow.com/questions/60867491/ for an explanation of the problem. We have that problem with + * the `Action.datasource` field. + */ + @ChangeSet(order = "004", id = "initial-indexes", author = "") + public void addInitialIndexes(MongoTemplate mongoTemplate) { + var createdAtIndex = makeIndex("createdAt"); + + ensureIndexes(mongoTemplate, Action.class, + createdAtIndex, + makeIndex("pageId", "name").unique().named("action_page_compound_index") + ); + + ensureIndexes(mongoTemplate, Application.class, + createdAtIndex, + makeIndex("name", "organizationId").unique().named("organization_application_compound_index") + ); + + ensureIndexes(mongoTemplate, Collection.class, + createdAtIndex + ); + + ensureIndexes(mongoTemplate, Config.class, + createdAtIndex, + makeIndex("name").unique() + ); + + ensureIndexes(mongoTemplate, Datasource.class, + createdAtIndex, + makeIndex("name", "organizationId").unique().named("organization_datasource_compound_index") + ); + + ensureIndexes(mongoTemplate, InviteUser.class, + createdAtIndex, + makeIndex("token").unique().expire(3600, TimeUnit.SECONDS), + makeIndex("email").unique() + ); + + ensureIndexes(mongoTemplate, Organization.class, + createdAtIndex, + makeIndex("slug").unique() + ); + + ensureIndexes(mongoTemplate, Page.class, + createdAtIndex, + makeIndex("applicationId", "name").unique().named("application_page_compound_index") + ); + + ensureIndexes(mongoTemplate, PasswordResetToken.class, + createdAtIndex, + makeIndex("email").unique().expire(3600, TimeUnit.SECONDS) + ); + + ensureIndexes(mongoTemplate, Permission.class, + createdAtIndex + ); + + ensureIndexes(mongoTemplate, Plugin.class, + createdAtIndex, + makeIndex("type"), + makeIndex("packageName").unique() + ); + + ensureIndexes(mongoTemplate, Query.class, + createdAtIndex, + makeIndex("name").unique() + ); + + ensureIndexes(mongoTemplate, Role.class, + createdAtIndex + ); + + ensureIndexes(mongoTemplate, Setting.class, + createdAtIndex, + makeIndex("key").unique() + ); + + ensureIndexes(mongoTemplate, User.class, + createdAtIndex, + makeIndex("email").unique() + ); + } + +} diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/OrganizationRepository.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/OrganizationRepository.java index 2fd02bb518..6a4b591dd0 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/OrganizationRepository.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/OrganizationRepository.java @@ -1,12 +1,16 @@ package com.appsmith.server.repositories; import com.appsmith.server.domains.Organization; +import org.springframework.data.mongodb.repository.Query; import org.springframework.stereotype.Repository; import reactor.core.publisher.Mono; @Repository public interface OrganizationRepository extends BaseRepository { - Mono findByName(String name); + Mono findBySlug(String slug); Mono findByIdAndPluginsPluginId(String organizationId, String pluginId); + + @Query(value = "{slug: {$regex: ?0}}", count = true) + Mono countSlugsByPrefix(String keyword); } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/OrganizationService.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/OrganizationService.java index 5dcfd883b7..c24f66c909 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/OrganizationService.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/OrganizationService.java @@ -6,10 +6,12 @@ import reactor.core.publisher.Mono; public interface OrganizationService extends CrudService { - Mono getByName(String name); - Mono create(Organization organization); + Mono getBySlug(String slug); + + Mono getNextUniqueSlug(String initialSlug); + Mono create(Organization organization, User user); Mono findById(String id); diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/OrganizationServiceImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/OrganizationServiceImpl.java index 0b1acff63d..32dcd24545 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/OrganizationServiceImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/OrganizationServiceImpl.java @@ -73,8 +73,14 @@ public class OrganizationServiceImpl extends BaseService getByName(String name) { - return repository.findByName(name); + public Mono getBySlug(String slug) { + return repository.findBySlug(slug); + } + + @Override + public Mono getNextUniqueSlug(String initialSlug) { + return repository.countSlugsByPrefix(initialSlug) + .map(max -> initialSlug + (max == 0 ? "" : (max + 1))); } /** @@ -95,7 +101,13 @@ public class OrganizationServiceImpl extends BaseService organizationMono = Mono.just(organization) + Mono setSlugMono = getNextUniqueSlug(organization.getSlug()) + .map(slug -> { + organization.setSlug(slug); + return organization; + }); + + Mono organizationMono = setSlugMono .flatMap(this::validateObject) //transform the organization data to embed setting object in each object in organizationSetting list. .flatMap(this::enhanceOrganizationSettingList) diff --git a/app/server/appsmith-server/src/test/java/com/appsmith/server/configurations/SeedMongoData.java b/app/server/appsmith-server/src/test/java/com/appsmith/server/configurations/SeedMongoData.java index 7a0e4660ed..ae468249c8 100644 --- a/app/server/appsmith-server/src/test/java/com/appsmith/server/configurations/SeedMongoData.java +++ b/app/server/appsmith-server/src/test/java/com/appsmith/server/configurations/SeedMongoData.java @@ -13,7 +13,9 @@ import com.appsmith.server.repositories.OrganizationRepository; import com.appsmith.server.repositories.PageRepository; import com.appsmith.server.repositories.PluginRepository; import com.appsmith.server.repositories.UserRepository; +import com.github.mongobee.exception.MongobeeException; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.ApplicationRunner; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -27,12 +29,18 @@ import java.util.List; @Configuration public class SeedMongoData { + @Autowired + private MongoConfig mongoConfig; + @Bean ApplicationRunner init(UserRepository userRepository, OrganizationRepository organizationRepository, ApplicationRepository applicationRepository, PageRepository pageRepository, - PluginRepository pluginRepository) { + PluginRepository pluginRepository) + throws MongobeeException { + + mongoConfig.runMigrations(); log.info("Seeding the data"); Object[][] userData = { @@ -40,7 +48,7 @@ public class SeedMongoData { {"api_user", "api_user", UserState.ACTIVATED}, }; Object[][] orgData = { - {"Spring Test Organization", "appsmith-spring-test.com", "appsmith.com"} + {"Spring Test Organization", "appsmith-spring-test.com", "appsmith.com", "spring-test-organization"} }; Object[][] appData = { {"LayoutServiceTest TestApplications"} @@ -50,8 +58,7 @@ public class SeedMongoData { }; Object[][] pluginData = { {"Installed Plugin Name", PluginType.API, "installed-plugin"}, - {"Not Installed Plugin Name", PluginType.API, "not-installed-plugin"}, - {"RestTemplatePluginExecutor", PluginType.API, "restapi-plugin"} + {"Not Installed Plugin Name", PluginType.API, "not-installed-plugin"} }; return args -> { organizationRepository.deleteAll() @@ -76,6 +83,7 @@ public class SeedMongoData { organization.setName((String) array[0]); organization.setDomain((String) array[1]); organization.setWebsite((String) array[2]); + organization.setSlug((String) array[3]); OrganizationPlugin orgPlugin = new OrganizationPlugin(); orgPlugin.setPluginId(pluginId); List orgPlugins = new ArrayList<>(); @@ -85,7 +93,7 @@ public class SeedMongoData { }).flatMap(organizationRepository::save) ) // Query the seed data to get the organizationId (required for application creation) - .then(organizationRepository.findByName((String) orgData[0][0])) + .then(organizationRepository.findBySlug((String) orgData[0][3])) .map(org -> org.getId()) // Seed the user data into the DB .flatMapMany(orgId -> Flux.just(userData) diff --git a/app/server/appsmith-server/src/test/java/com/appsmith/server/services/ActionServiceTest.java b/app/server/appsmith-server/src/test/java/com/appsmith/server/services/ActionServiceTest.java index 8c85592cd2..f2d7f3e4f9 100644 --- a/app/server/appsmith-server/src/test/java/com/appsmith/server/services/ActionServiceTest.java +++ b/app/server/appsmith-server/src/test/java/com/appsmith/server/services/ActionServiceTest.java @@ -53,7 +53,7 @@ public class ActionServiceTest { @WithUserDetails(value = "api_user") public void createValidActionNullActionConfiguration() { Action action = new Action(); - action.setName("randomActionName"); + action.setName("randomActionName2"); action.setPageId("randomPageId"); Mono actionMono = Mono.just(action) .flatMap(actionService::create); diff --git a/app/server/appsmith-server/src/test/java/com/appsmith/server/services/OrganizationServiceTest.java b/app/server/appsmith-server/src/test/java/com/appsmith/server/services/OrganizationServiceTest.java index eb2d6e8e04..39a6072001 100644 --- a/app/server/appsmith-server/src/test/java/com/appsmith/server/services/OrganizationServiceTest.java +++ b/app/server/appsmith-server/src/test/java/com/appsmith/server/services/OrganizationServiceTest.java @@ -34,6 +34,7 @@ public class OrganizationServiceTest { organization.setName("Test Name"); organization.setDomain("example.com"); organization.setWebsite("https://example.com"); + organization.setSlug("test-name"); } /* Tests for the Create Organization Flow */ @@ -100,6 +101,7 @@ public class OrganizationServiceTest { organization.setName("Test For Get Name"); organization.setDomain("example.com"); organization.setWebsite("https://example.com"); + organization.setSlug("test-for-get-name"); Mono createOrganization = organizationService.create(organization); Mono getOrganization = createOrganization.flatMap(t -> organizationService.getById(t.getId())); StepVerifier.create(getOrganization) @@ -118,6 +120,7 @@ public class OrganizationServiceTest { organization.setName("Test Update Name"); organization.setDomain("example.com"); organization.setWebsite("https://example.com"); + organization.setSlug("test-update-name"); Mono createOrganization = organizationService.create(organization); Mono updateOrganization = createOrganization @@ -137,4 +140,51 @@ public class OrganizationServiceTest { }) .verifyComplete(); } + + @Test + @WithUserDetails(value = "api_user") + public void uniqueSlugs() { + Organization organization = new Organization(); + organization.setName("First slug org"); + organization.setDomain("example.com"); + organization.setWebsite("https://example.com"); + + Mono uniqueSlug = organizationService.getNextUniqueSlug("slug-org") + .map(slug -> { + organization.setSlug(slug); + return organization; + }) + .flatMap(organizationService::create) + .then(organizationService.getNextUniqueSlug("slug-org")); + + StepVerifier.create(uniqueSlug) + .assertNext(slug -> { + assertThat(slug).isNotEqualTo("slug-org"); + }) + .verifyComplete(); + } + + @Test + @WithUserDetails(value = "api_user") + public void createDuplicateNameOrganization() { + Organization firstOrg = new Organization(); + firstOrg.setName("Really good org"); + firstOrg.setDomain("example.com"); + firstOrg.setWebsite("https://example.com"); + + Organization secondOrg = new Organization(); + secondOrg.setName(firstOrg.getName()); + secondOrg.setDomain(firstOrg.getDomain()); + secondOrg.setWebsite(firstOrg.getWebsite()); + + Mono firstOrgCreation = organizationService.create(firstOrg).cache(); + Mono secondOrgCreation = firstOrgCreation.then(organizationService.create(secondOrg)); + + StepVerifier.create(Mono.zip(firstOrgCreation, secondOrgCreation)) + .assertNext(orgsTuple -> { + assertThat(orgsTuple.getT1().getSlug()).isEqualTo("really-good-org"); + assertThat(orgsTuple.getT2().getSlug()).isEqualTo("really-good-org2"); + }) + .verifyComplete(); + } } diff --git a/app/server/appsmith-server/src/test/java/com/appsmith/server/services/UserServiceTest.java b/app/server/appsmith-server/src/test/java/com/appsmith/server/services/UserServiceTest.java index a3162f5872..43935bc20d 100644 --- a/app/server/appsmith-server/src/test/java/com/appsmith/server/services/UserServiceTest.java +++ b/app/server/appsmith-server/src/test/java/com/appsmith/server/services/UserServiceTest.java @@ -32,9 +32,8 @@ public class UserServiceTest { @Before public void setup() { - userMono = userService.findByEmail("usertest@usertest.com"); - organizationMono = organizationService.getByName("Spring Test Organization"); + organizationMono = organizationService.getBySlug("spring-test-organization"); } //Test the update organization flow. diff --git a/app/server/mongo-seed/seed.js b/app/server/mongo-seed/seed.js index 42173235a7..aeb5e888cb 100644 --- a/app/server/mongo-seed/seed.js +++ b/app/server/mongo-seed/seed.js @@ -2,29 +2,6 @@ let error = false print("**** Going to start Mongo seed ****") let res = [ - db.plugins.insertMany([ - { - "_id" : ObjectId("5c9f512f96c1a50004819786"), - "name" : "PostgresDbPlugin", - "type" : "DB", - "packageName" : "postgres-plugin", - "deleted" : false, - "uiComponent" : "DbEditorForm", - "defaultInstall" : true, - "_class" : "com.appsmith.server.domains.Plugin" - }, - { - "_id" : ObjectId("5ca385dc81b37f0004b4db85"), - "name" : "RestTemplatePluginExecutor", - "type" : "API", - "packageName" : "restapi-plugin", - "deleted" : false, - "uiComponent" : "ApiEditorForm", - "defaultInstall" : true, - "_class" : "com.appsmith.server.domains.Plugin" - } - ]), - db.organization.insert({ "_id": ObjectId("5da151714a020300041ae8fd"), "name": "Test Organization",