Protei/PromucFlow_constructor
2
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Merge branch 'feature/mongodb-migrations' into 'release'

Browse Source 
MongoDB Migrations and Organization Slug

See merge request theappsmith/internal-tools-server!241
This commit is contained in:
Shrikant Kandula 2020-03-27 14:52:25 +00:00
commit bfa0453a27
12 changed files with 380 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
app/server/appsmith-server/pom.xml
View File

@ -188,6 +188,13 @@
<artifactId>httpclient</artifactId>
<version>4.5.10</version>
</dependency>
<!-- MongoDB Migrations: https://github.com/mongobee/mongobee -->
<dependency>
<groupId>com.github.mongobee</groupId>
<artifactId>mongobee</artifactId>
<version>0.13</version>
</dependency>
</dependencies>
<build>

83
app/server/appsmith-server/src/main/java/com/appsmith/server/configurations/MongoConfig.java
View File

@ -1,10 +1,22 @@
package com.appsmith.server.configurations;
import com.appsmith.server.configurations.mongo.SoftDeleteMongoRepositoryFactoryBean;
import com.appsmith.server.migrations.DatabaseChangelog;
import com.appsmith.server.repositories.BaseRepositoryImpl;
import com.github.mongobee.Mongobee;
import com.github.mongobee.exception.MongobeeException;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.env.Environment;
import org.springframework.core.env.Profiles;
import org.springframework.data.mongodb.config.EnableMongoAuditing;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.mapping.MongoMappingContext;
import org.springframework.data.mongodb.repository.config.EnableReactiveMongoRepositories;
import org.springframework.util.StringUtils;
import javax.annotation.PostConstruct;
/**
* This configures the JPA Mongo repositories. The default base implementation is defined in {@link BaseRepositoryImpl}.
@ -13,6 +25,7 @@ import org.springframework.data.mongodb.repository.config.EnableReactiveMongoRep
* The factoryBean class is also custom defined in order to add default clauses for soft delete for all custom JPA queries.
* {@link SoftDeleteMongoRepositoryFactoryBean} for details.
*/
@Slf4j
@Configuration
@EnableMongoAuditing
@EnableReactiveMongoRepositories(repositoryFactoryBeanClass = SoftDeleteMongoRepositoryFactoryBean.class,
@ -20,4 +33,74 @@ import org.springframework.data.mongodb.repository.config.EnableReactiveMongoRep
repositoryBaseClass = BaseRepositoryImpl.class
)
public class MongoConfig {
@Value("${spring.data.mongodb.uri:}")
private String mongoDbUri;
@Value("${spring.data.mongodb.username:}")
private String mongoDbUsername;
@Value("${spring.data.mongodb.password:}")
private String mongoDbPassword;
@Value("${spring.data.mongodb.host:}:${spring.data.mongodb.port:}/${spring.data.mongodb.database:}")
private String mongoDbResource;
private final MongoTemplate mongoTemplate;
private MongoMappingContext mongoMappingContext;
private Environment environment;
public MongoConfig(MongoTemplate mongoTemplate, MongoMappingContext mongoMappingContext, Environment environment) {
this.mongoTemplate = mongoTemplate;
this.mongoMappingContext = mongoMappingContext;
this.environment = environment;
}
@PostConstruct
public void init() {
if (!environment.acceptsProfiles(Profiles.of("test"))) {
try {
runMigrations();
} catch (MongobeeException e) {
log.error("Failed running migrations automatically at startup.", e);
}
}
}
public void runMigrations() throws MongobeeException {
String uri;
if (StringUtils.isEmpty(mongoDbUri)) {
uri = "mongodb://" +
mongoDbUsername +
(mongoDbPassword.isEmpty() ? "" : ":") +
mongoDbPassword +
(mongoDbUsername.isEmpty() ? "" : "@") +
mongoDbResource;
} else {
uri = mongoDbUri;
}
runMigrations(uri);
}
public void runMigrations(String uri) throws MongobeeException {
// Mongobee creates its own connection to MongoDB and (hopefully) closes it when migration execution is done.
// If there's no new migrations to be applied, no connection is made. However, note that the `autoCreateIndexes`
// migration method is always run, so a connection is always made.
// Also, this Mongobee runner cannot be a separate bean, since then the `.execute` will be called automatically.
// We need to run it manually so we can control it during tests.
Mongobee runner = new Mongobee(uri);
runner.setSpringEnvironment(environment);
runner.setChangeLogsScanPackage(getClass().getPackageName().replaceFirst("\\.[^.]+$", ".migrations"));
runner.setMongoTemplate(mongoTemplate);
DatabaseChangelog.mongoMappingContext = mongoMappingContext;
log.info("Executing MongoDB migrations");
runner.execute();
}
}

13
app/server/appsmith-server/src/main/java/com/appsmith/server/domains/Organization.java
View File

@ -22,7 +22,6 @@ public class Organization extends BaseDomain {
private String domain;
@NotNull
@Indexed(unique = true)
private String name;
private String website;
@ -31,4 +30,16 @@ public class Organization extends BaseDomain {
private List<OrganizationPlugin> plugins;
@NotNull
@Indexed(unique = true)
private String slug;
public String getSlug() {
return slug == null ? toSlug(name) : slug;
}
public static String toSlug(String text) {
return text == null ? null : text.replaceAll("[^\\w\\d]+", "-").toLowerCase();
}
}

189
app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/DatabaseChangelog.java Normal file
View File

@ -0,0 +1,189 @@
package com.appsmith.server.migrations;
import com.appsmith.server.domains.*;
import com.github.mongobee.changeset.ChangeLog;
import com.github.mongobee.changeset.ChangeSet;
import com.mongodb.DuplicateKeyException;
import lombok.extern.slf4j.Slf4j;
import org.springframework.data.domain.Sort;
import org.springframework.data.mongodb.UncategorizedMongoDbException;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.index.CompoundIndexDefinition;
import org.springframework.data.mongodb.core.index.Index;
import org.springframework.data.mongodb.core.index.MongoPersistentEntityIndexResolver;
import org.springframework.data.mongodb.core.mapping.BasicMongoPersistentEntity;
import org.springframework.data.mongodb.core.mapping.Document;
import org.springframework.data.mongodb.core.mapping.MongoMappingContext;
import java.util.concurrent.TimeUnit;
@Slf4j
@ChangeLog(order = "001")
public class DatabaseChangelog {
// This value is set from ServerApplication before the migrations run. This appears to be the only reliable way to
// inject Spring beans into this class. May be there will be a better way to do this once Mongobee's Spring
// integration is improved.
// - If we add @Autowired to the `autoCreateIndexes` method to get this, then the migration method will run but
// Mongobee won't save that fact in it's own collection, and will run this migration method at every server startup.
// - If we add @Autowired to this field here, we have to turn the class into a Spring Component, but then Mongobee
// creates its own instance of this class, which would have this field set to `null`.
public static MongoMappingContext mongoMappingContext;
/**
* A private, pure utility function to create instances of Index objects to pass to `IndexOps.ensureIndex` method.
*/
private static Index makeIndex(String... fields) {
if (fields.length == 1) {
return new Index(fields[0], Sort.Direction.ASC).named(fields[0]);
} else {
org.bson.Document doc = new org.bson.Document();
for (String field : fields) {
doc.put(field, 1);
}
return new CompoundIndexDefinition(doc);
}
}
/**
* Given a MongoTemplate, a domain class and a bunch of Index definitions, this pure utility function will ensure
* those indexes on the database behind the MongoTemplate instance.
*/
private static void ensureIndexes(MongoTemplate mongoTemplate, Class<?> entityClass, Index... indexes) {
var indexOps = mongoTemplate.indexOps(entityClass);
for (Index index : indexes) {
indexOps.ensureIndex(index);
}
}
@ChangeSet(order = "001", id = "initial-plugins", author = "")
public void initialPlugins(MongoTemplate mongoTemplate) {
Plugin plugin1 = new Plugin();
plugin1.setName("PostgresDbPlugin");
plugin1.setType(PluginType.DB);
plugin1.setPackageName("postgres-plugin");
plugin1.setUiComponent("DbEditorForm");
plugin1.setDefaultInstall(true);
try {
mongoTemplate.insert(plugin1);
} catch (DuplicateKeyException e) {
log.warn("postgres-plugin already present in database.", e);
}
Plugin plugin2 = new Plugin();
plugin2.setName("RestTemplatePluginExecutor");
plugin2.setType(PluginType.API);
plugin2.setPackageName("restapi-plugin");
plugin2.setUiComponent("ApiEditorForm");
plugin2.setDefaultInstall(true);
try {
mongoTemplate.insert(plugin2);
} catch (DuplicateKeyException e) {
log.warn("restapi-plugin already present in database.", e);
}
}
@ChangeSet(order = "002", id = "remove-org-name-index", author = "")
public void removeOrgNameIndex(MongoTemplate mongoTemplate) {
try {
mongoTemplate.indexOps(Organization.class).dropIndex("name");
} catch (UncategorizedMongoDbException ignored) {
// The index probably doesn't exist. This happens if the database is created after the @Indexed annotation
// has been removed.
}
}
@ChangeSet(order = "003", id = "add-org-slugs", author = "")
public void addOrgSlugs(MongoTemplate mongoTemplate) {
// For all existing organizations, add a slug field, which should be unique.
for (Organization organization : mongoTemplate.findAll(Organization.class)) {
organization.setSlug(organization.getSlug());
mongoTemplate.save(organization);
}
}
/**
* We are creating indexes manually because Spring's index resolver creates indexes on fields as well.
* See https://stackoverflow.com/questions/60867491/ for an explanation of the problem. We have that problem with
* the `Action.datasource` field.
*/
@ChangeSet(order = "004", id = "initial-indexes", author = "")
public void addInitialIndexes(MongoTemplate mongoTemplate) {
var createdAtIndex = makeIndex("createdAt");
ensureIndexes(mongoTemplate, Action.class,
createdAtIndex,
makeIndex("pageId", "name").unique().named("action_page_compound_index")
);
ensureIndexes(mongoTemplate, Application.class,
createdAtIndex,
makeIndex("name", "organizationId").unique().named("organization_application_compound_index")
);
ensureIndexes(mongoTemplate, Collection.class,
createdAtIndex
);
ensureIndexes(mongoTemplate, Config.class,
createdAtIndex,
makeIndex("name").unique()
);
ensureIndexes(mongoTemplate, Datasource.class,
createdAtIndex,
makeIndex("name", "organizationId").unique().named("organization_datasource_compound_index")
);
ensureIndexes(mongoTemplate, InviteUser.class,
createdAtIndex,
makeIndex("token").unique().expire(3600, TimeUnit.SECONDS),
makeIndex("email").unique()
);
ensureIndexes(mongoTemplate, Organization.class,
createdAtIndex,
makeIndex("slug").unique()
);
ensureIndexes(mongoTemplate, Page.class,
createdAtIndex,
makeIndex("applicationId", "name").unique().named("application_page_compound_index")
);
ensureIndexes(mongoTemplate, PasswordResetToken.class,
createdAtIndex,
makeIndex("email").unique().expire(3600, TimeUnit.SECONDS)
);
ensureIndexes(mongoTemplate, Permission.class,
createdAtIndex
);
ensureIndexes(mongoTemplate, Plugin.class,
createdAtIndex,
makeIndex("type"),
makeIndex("packageName").unique()
);
ensureIndexes(mongoTemplate, Query.class,
createdAtIndex,
makeIndex("name").unique()
);
ensureIndexes(mongoTemplate, Role.class,
createdAtIndex
);
ensureIndexes(mongoTemplate, Setting.class,
createdAtIndex,
makeIndex("key").unique()
);
ensureIndexes(mongoTemplate, User.class,
createdAtIndex,
makeIndex("email").unique()
);
}
}

6
app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/OrganizationRepository.java
View File

@ -1,12 +1,16 @@
package com.appsmith.server.repositories;
import com.appsmith.server.domains.Organization;
import org.springframework.data.mongodb.repository.Query;
import org.springframework.stereotype.Repository;
import reactor.core.publisher.Mono;
@Repository
public interface OrganizationRepository extends BaseRepository<Organization, String> {
Mono<Organization> findByName(String name);
Mono<Organization> findBySlug(String slug);
Mono<Organization> findByIdAndPluginsPluginId(String organizationId, String pluginId);
@Query(value = "{slug: {$regex: ?0}}", count = true)
Mono<Long> countSlugsByPrefix(String keyword);
}

6
app/server/appsmith-server/src/main/java/com/appsmith/server/services/OrganizationService.java
View File

@ -6,10 +6,12 @@ import reactor.core.publisher.Mono;
public interface OrganizationService extends CrudService<Organization, String> {
Mono<Organization> getByName(String name);
Mono<Organization> create(Organization organization);
Mono<Organization> getBySlug(String slug);
Mono<String> getNextUniqueSlug(String initialSlug);
Mono<Organization> create(Organization organization, User user);
Mono<Organization> findById(String id);

18
app/server/appsmith-server/src/main/java/com/appsmith/server/services/OrganizationServiceImpl.java
View File

@ -73,8 +73,14 @@ public class OrganizationServiceImpl extends BaseService<OrganizationRepository,
}
@Override
public Mono<Organization> getByName(String name) {
return repository.findByName(name);
public Mono<Organization> getBySlug(String slug) {
return repository.findBySlug(slug);
}
@Override
public Mono<String> getNextUniqueSlug(String initialSlug) {
return repository.countSlugsByPrefix(initialSlug)
.map(max -> initialSlug + (max == 0 ? "" : (max + 1)));
}
/**
@ -95,7 +101,13 @@ public class OrganizationServiceImpl extends BaseService<OrganizationRepository,
return Mono.error(new AppsmithException(AppsmithError.INVALID_PARAMETER, FieldName.ORGANIZATION));
}
Mono<Organization> organizationMono = Mono.just(organization)
Mono<Organization> setSlugMono = getNextUniqueSlug(organization.getSlug())
.map(slug -> {
organization.setSlug(slug);
return organization;
});
Mono<Organization> organizationMono = setSlugMono
.flatMap(this::validateObject)
//transform the organization data to embed setting object in each object in organizationSetting list.
.flatMap(this::enhanceOrganizationSettingList)

18
app/server/appsmith-server/src/test/java/com/appsmith/server/configurations/SeedMongoData.java
View File

@ -13,7 +13,9 @@ import com.appsmith.server.repositories.OrganizationRepository;
import com.appsmith.server.repositories.PageRepository;
import com.appsmith.server.repositories.PluginRepository;
import com.appsmith.server.repositories.UserRepository;
import com.github.mongobee.exception.MongobeeException;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.ApplicationRunner;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@ -27,12 +29,18 @@ import java.util.List;
@Configuration
public class SeedMongoData {
@Autowired
private MongoConfig mongoConfig;
@Bean
ApplicationRunner init(UserRepository userRepository,
OrganizationRepository organizationRepository,
ApplicationRepository applicationRepository,
PageRepository pageRepository,
PluginRepository pluginRepository) {
PluginRepository pluginRepository)
throws MongobeeException {
mongoConfig.runMigrations();
log.info("Seeding the data");
Object[][] userData = {
@ -40,7 +48,7 @@ public class SeedMongoData {
{"api_user", "api_user", UserState.ACTIVATED},
};
Object[][] orgData = {
{"Spring Test Organization", "appsmith-spring-test.com", "appsmith.com"}
{"Spring Test Organization", "appsmith-spring-test.com", "appsmith.com", "spring-test-organization"}
};
Object[][] appData = {
{"LayoutServiceTest TestApplications"}
@ -50,8 +58,7 @@ public class SeedMongoData {
};
Object[][] pluginData = {
{"Installed Plugin Name", PluginType.API, "installed-plugin"},
{"Not Installed Plugin Name", PluginType.API, "not-installed-plugin"},
{"RestTemplatePluginExecutor", PluginType.API, "restapi-plugin"}
{"Not Installed Plugin Name", PluginType.API, "not-installed-plugin"}
};
return args -> {
organizationRepository.deleteAll()
@ -76,6 +83,7 @@ public class SeedMongoData {
organization.setName((String) array[0]);
organization.setDomain((String) array[1]);
organization.setWebsite((String) array[2]);
organization.setSlug((String) array[3]);
OrganizationPlugin orgPlugin = new OrganizationPlugin();
orgPlugin.setPluginId(pluginId);
List<OrganizationPlugin> orgPlugins = new ArrayList<>();
@ -85,7 +93,7 @@ public class SeedMongoData {
}).flatMap(organizationRepository::save)
)
// Query the seed data to get the organizationId (required for application creation)
.then(organizationRepository.findByName((String) orgData[0][0]))
.then(organizationRepository.findBySlug((String) orgData[0][3]))
.map(org -> org.getId())
// Seed the user data into the DB
.flatMapMany(orgId -> Flux.just(userData)

2
app/server/appsmith-server/src/test/java/com/appsmith/server/services/ActionServiceTest.java
View File

@ -53,7 +53,7 @@ public class ActionServiceTest {
@WithUserDetails(value = "api_user")
public void createValidActionNullActionConfiguration() {
Action action = new Action();
action.setName("randomActionName");
action.setName("randomActionName2");
action.setPageId("randomPageId");
Mono<Action> actionMono = Mono.just(action)
.flatMap(actionService::create);

50
app/server/appsmith-server/src/test/java/com/appsmith/server/services/OrganizationServiceTest.java
View File

@ -34,6 +34,7 @@ public class OrganizationServiceTest {
organization.setName("Test Name");
organization.setDomain("example.com");
organization.setWebsite("https://example.com");
organization.setSlug("test-name");
}
/* Tests for the Create Organization Flow */
@ -100,6 +101,7 @@ public class OrganizationServiceTest {
organization.setName("Test For Get Name");
organization.setDomain("example.com");
organization.setWebsite("https://example.com");
organization.setSlug("test-for-get-name");
Mono<Organization> createOrganization = organizationService.create(organization);
Mono<Organization> getOrganization = createOrganization.flatMap(t -> organizationService.getById(t.getId()));
StepVerifier.create(getOrganization)
@ -118,6 +120,7 @@ public class OrganizationServiceTest {
organization.setName("Test Update Name");
organization.setDomain("example.com");
organization.setWebsite("https://example.com");
organization.setSlug("test-update-name");
Mono<Organization> createOrganization = organizationService.create(organization);
Mono<Organization> updateOrganization = createOrganization
@ -137,4 +140,51 @@ public class OrganizationServiceTest {
})
.verifyComplete();
}
@Test
@WithUserDetails(value = "api_user")
public void uniqueSlugs() {
Organization organization = new Organization();
organization.setName("First slug org");
organization.setDomain("example.com");
organization.setWebsite("https://example.com");
Mono<String> uniqueSlug = organizationService.getNextUniqueSlug("slug-org")
.map(slug -> {
organization.setSlug(slug);
return organization;
})
.flatMap(organizationService::create)
.then(organizationService.getNextUniqueSlug("slug-org"));
StepVerifier.create(uniqueSlug)
.assertNext(slug -> {
assertThat(slug).isNotEqualTo("slug-org");
})
.verifyComplete();
}
@Test
@WithUserDetails(value = "api_user")
public void createDuplicateNameOrganization() {
Organization firstOrg = new Organization();
firstOrg.setName("Really good org");
firstOrg.setDomain("example.com");
firstOrg.setWebsite("https://example.com");
Organization secondOrg = new Organization();
secondOrg.setName(firstOrg.getName());
secondOrg.setDomain(firstOrg.getDomain());
secondOrg.setWebsite(firstOrg.getWebsite());
Mono<Organization> firstOrgCreation = organizationService.create(firstOrg).cache();
Mono<Organization> secondOrgCreation = firstOrgCreation.then(organizationService.create(secondOrg));
StepVerifier.create(Mono.zip(firstOrgCreation, secondOrgCreation))
.assertNext(orgsTuple -> {
assertThat(orgsTuple.getT1().getSlug()).isEqualTo("really-good-org");
assertThat(orgsTuple.getT2().getSlug()).isEqualTo("really-good-org2");
})
.verifyComplete();
}
}

3
app/server/appsmith-server/src/test/java/com/appsmith/server/services/UserServiceTest.java
View File

@ -32,9 +32,8 @@ public class UserServiceTest {
@Before
public void setup() {
userMono = userService.findByEmail("usertest@usertest.com");
organizationMono = organizationService.getByName("Spring Test Organization");
organizationMono = organizationService.getBySlug("spring-test-organization");
}
//Test the update organization flow.

23
app/server/mongo-seed/seed.js
View File

@ -2,29 +2,6 @@ let error = false
print("**** Going to start Mongo seed ****")
let res = [
db.plugins.insertMany([
{
"_id" : ObjectId("5c9f512f96c1a50004819786"),
"name" : "PostgresDbPlugin",
"type" : "DB",
"packageName" : "postgres-plugin",
"deleted" : false,
"uiComponent" : "DbEditorForm",
"defaultInstall" : true,
"_class" : "com.appsmith.server.domains.Plugin"
},
{
"_id" : ObjectId("5ca385dc81b37f0004b4db85"),
"name" : "RestTemplatePluginExecutor",
"type" : "API",
"packageName" : "restapi-plugin",
"deleted" : false,
"uiComponent" : "ApiEditorForm",
"defaultInstall" : true,
"_class" : "com.appsmith.server.domains.Plugin"
}
]),
db.organization.insert({
"_id": ObjectId("5da151714a020300041ae8fd"),
"name": "Test Organization",