chore: Add mover script for Mongo to postgres migration (#36458)
## Description PR to add: 1. Mover script : move-to-postgres.sh 2. Baseline data generation: regen-baseline.sh /test Sanity ### 🔍 Cypress test results <!-- This is an auto-generated comment: Cypress test results --> > [!TIP] > 🟢 🟢 🟢 All cypress tests have passed! 🎉 🎉 🎉 > Workflow run: <https://github.com/appsmithorg/appsmith/actions/runs/11027514472> > Commit: c565b7a4d832b0d0e75df2f3122a1c0679235e88 > <a href="https://internal.appsmith.com/app/cypress-dashboard/rundetails-65890b3c81d7400d08fa9ee5?branch=master&workflowId=11027514472&attempt=1" target="_blank">Cypress dashboard</a>. > Tags: `@tag.Sanity` > Spec: > <hr>Wed, 25 Sep 2024 06:54:08 UTC <!-- end of auto-generated comment: Cypress test results --> ## Communication Should the DevRel and Marketing teams inform users about this change? - [ ] Yes - [x] No <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit - **New Features** - Introduced a new configuration file for instance management and permission groups. - Added a custom JavaScript library for XML parsing. - Implemented multiple database and API plugins, enhancing integration capabilities. - Established a tenant configuration for managing tenant-specific settings. - Added an anonymous user entity for improved user management. - **Bug Fixes** - Improved data migration script from MongoDB to Postgres. - **Documentation** - Updated scripts for baseline data generation and migration processes. <!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
Abhijeet
GitHub
parent
842443c267
commit
1210104575
4
deploy/docker/fs/opt/appsmith/baseline-ce/config.jsonl
Normal file
4
deploy/docker/fs/opt/appsmith/baseline-ce/config.jsonl
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
{"_class":"com.appsmith.server.domains.Config","config":{"value":"66f2ec65704a8e568824bda7"},"deleted":false,"id":"66f2ec65704a8e568824bda8","name":"instance-id","policies":[],"policyMap":{}}
|
||||
{"_class":"com.appsmith.server.domains.Config","config":{"defaultPermissionGroup":"66f2ec66704a8e568824bdb9"},"deleted":false,"id":"66f2ec66704a8e568824bdb8","name":"instanceConfig","policies":[{"permission":"readInstanceConfiguration:config","permissionGroups":["66f2ec66704a8e568824bdb9"]},{"permission":"manageInstanceConfiguration:config","permissionGroups":["66f2ec66704a8e568824bdb9"]}],"policyMap":{"manageInstanceConfiguration:config":{"permission":"manageInstanceConfiguration:config","permissionGroups":["66f2ec66704a8e568824bdb9"]},"readInstanceConfiguration:config":{"permission":"readInstanceConfiguration:config","permissionGroups":["66f2ec66704a8e568824bdb9"]}}}
|
||||
{"_class":"com.appsmith.server.domains.Config","config":{"permissionGroupId":"66f2ec66704a8e568824bdba"},"deleted":false,"id":"66f2ec66704a8e568824bdbb","name":"publicPermissionGroup","policies":[],"policyMap":{}}
|
||||
{"_class":"com.appsmith.server.domains.Config","config":{"value":2},"deleted":false,"id":"66f2ec64704a8e568824bda2","name":"schemaVersion","policies":[],"policyMap":{}}
|
||||
|
|
@ -0,0 +1 @@
|
|||
{"_class":"com.appsmith.server.domains.CustomJSLib","accessor":["xmlParser"],"defs":"{\"!name\":\"LIB/xmlParser\",\"xmlParser\":{\"parse\":{\"!type\":\"fn()\",\"prototype\":{}},\"convertTonimn\":{\"!type\":\"fn()\",\"prototype\":{}},\"getTraversalObj\":{\"!type\":\"fn()\",\"prototype\":{}},\"convertToJson\":{\"!type\":\"fn()\",\"prototype\":{}},\"convertToJsonString\":{\"!type\":\"fn()\",\"prototype\":{}},\"validate\":{\"!type\":\"fn()\",\"prototype\":{}},\"j2xParser\":{\"!type\":\"fn()\",\"prototype\":{\"parse\":{\"!type\":\"fn()\",\"prototype\":{}},\"j2x\":{\"!type\":\"fn()\",\"prototype\":{}}}},\"parseToNimn\":{\"!type\":\"fn()\",\"prototype\":{}}}}","deleted":false,"id":"66f2ec6a704a8e568824bdc7","name":"xmlParser","policies":[],"policyMap":{},"uidString":"xmlParser_https://cdnjs.cloudflare.com/ajax/libs/fast-xml-parser/3.17.5/parser.min.js","url":"https://cdnjs.cloudflare.com/ajax/libs/fast-xml-parser/3.17.5/parser.min.js","version":"3.17.5"}
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
{"_class":"com.appsmith.server.domains.PermissionGroup","assignedToGroupIds":[],"assignedToUserIds":[],"deleted":false,"id":"66f2ec66704a8e568824bdb9","name":"Instance Administrator Role","permissions":[],"policies":[{"permission":"read:permissionGroupMembers","permissionGroups":["66f2ec66704a8e568824bdb9"]},{"permission":"assign:permissionGroups","permissionGroups":["66f2ec66704a8e568824bdb9"]},{"permission":"unassign:permissionGroups","permissionGroups":["66f2ec66704a8e568824bdb9"]}],"policyMap":{"assign:permissionGroups":{"permission":"assign:permissionGroups","permissionGroups":["66f2ec66704a8e568824bdb9"]},"read:permissionGroupMembers":{"permission":"read:permissionGroupMembers","permissionGroups":["66f2ec66704a8e568824bdb9"]},"unassign:permissionGroups":{"permission":"unassign:permissionGroups","permissionGroups":["66f2ec66704a8e568824bdb9"]}}}
|
||||
{"_class":"com.appsmith.server.domains.PermissionGroup","assignedToGroupIds":[],"assignedToUserIds":["66f2ec66704a8e568824bdb7"],"deleted":false,"description":"Role for giving accesses for all objects to anonymous users","id":"66f2ec66704a8e568824bdba","name":"publicPermissionGroup","permissions":[],"policies":[],"policyMap":{}}
|
||||
24
deploy/docker/fs/opt/appsmith/baseline-ce/plugin.jsonl
Normal file
24
deploy/docker/fs/opt/appsmith/baseline-ce/plugin.jsonl
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
{"_class":"com.appsmith.server.domains.Plugin","allowUserDatasources":true,"datasourceComponent":"AutoForm","defaultInstall":true,"deleted":false,"documentationLink":"https://docs.appsmith.com/reference/datasources/querying-postgres#create-crud-queries","generateCRUDPageComponent":"PostgreSQL","iconLocation":"https://assets.appsmith.com/logo/postgresql.svg","id":"66f2ec64704a8e568824bda3","isRemotePlugin":false,"isSupportedForAirGap":true,"name":"PostgreSQL","packageName":"postgres-plugin","policies":[],"policyMap":{},"responseType":"TABLE","type":"DB","uiComponent":"DbEditorForm"}
|
||||
{"_class":"com.appsmith.server.domains.Plugin","allowUserDatasources":true,"datasourceComponent":"RestAPIDatasourceForm","defaultInstall":true,"deleted":false,"iconLocation":"https://assets.appsmith.com/RestAPI.png","id":"66f2ec64704a8e568824bda4","isRemotePlugin":false,"isSupportedForAirGap":true,"name":"REST API","packageName":"restapi-plugin","policies":[],"policyMap":{},"type":"API","uiComponent":"ApiEditorForm"}
|
||||
{"_class":"com.appsmith.server.domains.Plugin","allowUserDatasources":true,"datasourceComponent":"AutoForm","defaultInstall":true,"deleted":false,"documentationLink":"https://docs.appsmith.com/reference/datasources/querying-mongodb#create-queries","generateCRUDPageComponent":"MongoDB","iconLocation":"https://assets.appsmith.com/logo/mongodb.svg","id":"66f2ec64704a8e568824bda5","isRemotePlugin":false,"isSupportedForAirGap":true,"name":"MongoDB","packageName":"mongo-plugin","policies":[],"policyMap":{},"responseType":"JSON","type":"DB","uiComponent":"UQIDbEditorForm"}
|
||||
{"_class":"com.appsmith.server.domains.Plugin","allowUserDatasources":true,"datasourceComponent":"AutoForm","defaultInstall":true,"deleted":false,"documentationLink":"https://docs.appsmith.com/reference/datasources/querying-mysql#create-queries","generateCRUDPageComponent":"SQL","iconLocation":"https://assets.appsmith.com/logo/mysql.svg","id":"66f2ec65704a8e568824bda6","isRemotePlugin":false,"isSupportedForAirGap":true,"name":"MySQL","packageName":"mysql-plugin","policies":[],"policyMap":{},"responseType":"TABLE","type":"DB","uiComponent":"DbEditorForm"}
|
||||
{"_class":"com.appsmith.server.domains.Plugin","allowUserDatasources":true,"datasourceComponent":"AutoForm","defaultInstall":true,"deleted":false,"documentationLink":"https://docs.appsmith.com/reference/datasources/querying-elasticsearch#querying-elasticsearch","iconLocation":"https://assets.appsmith.com/logo/elastic.svg","id":"66f2ec65704a8e568824bda9","isRemotePlugin":false,"isSupportedForAirGap":true,"name":"Elasticsearch","packageName":"elasticsearch-plugin","policies":[],"policyMap":{},"responseType":"JSON","type":"DB","uiComponent":"DbEditorForm"}
|
||||
{"_class":"com.appsmith.server.domains.Plugin","allowUserDatasources":true,"datasourceComponent":"AutoForm","defaultInstall":true,"deleted":false,"documentationLink":"https://docs.appsmith.com/reference/datasources/querying-dynamodb#create-queries","iconLocation":"https://assets.appsmith.com/logo/aws-dynamodb.svg","id":"66f2ec65704a8e568824bdaa","isRemotePlugin":false,"isSupportedForAirGap":true,"name":"DynamoDB","packageName":"dynamo-plugin","policies":[],"policyMap":{},"responseType":"JSON","type":"DB","uiComponent":"DbEditorForm"}
|
||||
{"_class":"com.appsmith.server.domains.Plugin","allowUserDatasources":true,"datasourceComponent":"AutoForm","defaultInstall":true,"deleted":false,"documentationLink":"https://docs.appsmith.com/reference/datasources/querying-redis#querying-redis","iconLocation":"https://assets.appsmith.com/logo/redis.svg","id":"66f2ec65704a8e568824bdab","isRemotePlugin":false,"isSupportedForAirGap":true,"name":"Redis","packageName":"redis-plugin","policies":[],"policyMap":{},"responseType":"TABLE","type":"DB","uiComponent":"DbEditorForm"}
|
||||
{"_class":"com.appsmith.server.domains.Plugin","allowUserDatasources":true,"datasourceComponent":"AutoForm","defaultInstall":true,"deleted":false,"documentationLink":"https://docs.appsmith.com/reference/datasources/querying-mssql#create-queries","generateCRUDPageComponent":"SQL","iconLocation":"https://assets.appsmith.com/logo/mssql.svg","id":"66f2ec65704a8e568824bdac","isRemotePlugin":false,"isSupportedForAirGap":true,"name":"Microsoft SQL Server","packageName":"mssql-plugin","policies":[],"policyMap":{},"responseType":"TABLE","type":"DB","uiComponent":"DbEditorForm"}
|
||||
{"_class":"com.appsmith.server.domains.Plugin","allowUserDatasources":true,"datasourceComponent":"AutoForm","defaultInstall":true,"deleted":false,"documentationLink":"https://docs.appsmith.com/reference/datasources/querying-firestore#understanding-commands","iconLocation":"https://assets.appsmith.com/logo/firestore.svg","id":"66f2ec65704a8e568824bdad","isRemotePlugin":false,"isSupportedForAirGap":true,"name":"Firestore","packageName":"firestore-plugin","policies":[],"policyMap":{},"responseType":"JSON","type":"DB","uiComponent":"UQIDbEditorForm"}
|
||||
{"_class":"com.appsmith.server.domains.Plugin","allowUserDatasources":true,"datasourceComponent":"AutoForm","defaultInstall":true,"deleted":false,"documentationLink":"https://docs.appsmith.com/reference/datasources/querying-redshift#querying-redshift","generateCRUDPageComponent":"SQL","iconLocation":"https://assets.appsmith.com/logo/aws-redshift.svg","id":"66f2ec65704a8e568824bdae","isRemotePlugin":false,"isSupportedForAirGap":true,"name":"Redshift","packageName":"redshift-plugin","policies":[],"policyMap":{},"responseType":"TABLE","type":"DB","uiComponent":"DbEditorForm"}
|
||||
{"_class":"com.appsmith.server.domains.Plugin","allowUserDatasources":true,"datasourceComponent":"AutoForm","defaultInstall":true,"deleted":false,"documentationLink":"https://docs.appsmith.com/reference/datasources/querying-amazon-s3#list-files","generateCRUDPageComponent":"S3","iconLocation":"https://assets.appsmith.com/logo/aws-s3.svg","id":"66f2ec65704a8e568824bdaf","isRemotePlugin":false,"isSupportedForAirGap":true,"name":"S3","packageName":"amazons3-plugin","policies":[],"policyMap":{},"responseType":"JSON","type":"DB","uiComponent":"UQIDbEditorForm"}
|
||||
{"_class":"com.appsmith.server.domains.Plugin","allowUserDatasources":true,"datasourceComponent":"OAuth2DatasourceForm","defaultInstall":true,"deleted":false,"documentationLink":"https://docs.appsmith.com/reference/datasources/querying-google-sheets#create-queries","generateCRUDPageComponent":"Google Sheets","iconLocation":"https://assets.appsmith.com/GoogleSheets.svg","id":"66f2ec65704a8e568824bdb0","isDependentOnCS":true,"isRemotePlugin":false,"isSupportedForAirGap":false,"name":"Google Sheets","packageName":"google-sheets-plugin","pluginName":"google-sheets-plugin","policies":[],"policyMap":{},"responseType":"JSON","type":"SAAS","uiComponent":"UQIDbEditorForm"}
|
||||
{"_class":"com.appsmith.server.domains.Plugin","allowUserDatasources":true,"datasourceComponent":"AutoForm","defaultInstall":true,"deleted":false,"documentationLink":"https://docs.appsmith.com/reference/datasources/querying-snowflake-db#querying-snowflake","generateCRUDPageComponent":"SQL","iconLocation":"https://assets.appsmith.com/logo/snowflake.svg","id":"66f2ec65704a8e568824bdb2","isRemotePlugin":false,"isSupportedForAirGap":true,"name":"Snowflake","packageName":"snowflake-plugin","policies":[],"policyMap":{},"responseType":"TABLE","type":"DB","uiComponent":"DbEditorForm"}
|
||||
{"_class":"com.appsmith.server.domains.Plugin","allowUserDatasources":true,"datasourceComponent":"AutoForm","defaultInstall":true,"deleted":false,"documentationLink":"https://docs.appsmith.com/reference/datasources/querying-arango-db#using-queries-in-applications","iconLocation":"https://assets.appsmith.com/logo/arangodb.svg","id":"66f2ec65704a8e568824bdb3","isRemotePlugin":false,"isSupportedForAirGap":true,"name":"ArangoDB","packageName":"arangodb-plugin","policies":[],"policyMap":{},"responseType":"TABLE","type":"DB","uiComponent":"DbEditorForm"}
|
||||
{"_class":"com.appsmith.server.domains.Plugin","allowUserDatasources":true,"defaultInstall":true,"deleted":false,"documentationLink":"https://docs.appsmith.com/v/v1.2.1/js-reference/using-js","iconLocation":"https://assets.appsmith.com/js-yellow.svg","id":"66f2ec65704a8e568824bdb4","isRemotePlugin":false,"isSupportedForAirGap":true,"name":"JS Functions","packageName":"js-plugin","policies":[],"policyMap":{},"responseType":"JSON","type":"JS","uiComponent":"JsEditorForm"}
|
||||
{"_class":"com.appsmith.server.domains.Plugin","allowUserDatasources":true,"datasourceComponent":"AutoForm","defaultInstall":true,"deleted":false,"documentationLink":"https://docs.appsmith.com/reference/datasources/using-smtp","iconLocation":"https://assets.appsmith.com/smtp-icon.svg","id":"66f2ec66704a8e568824bdb5","isRemotePlugin":false,"isSupportedForAirGap":true,"name":"SMTP","packageName":"smtp-plugin","policies":[],"policyMap":{},"responseType":"JSON","type":"DB","uiComponent":"UQIDbEditorForm"}
|
||||
{"_class":"com.appsmith.server.domains.Plugin","allowUserDatasources":true,"datasourceComponent":"RestAPIDatasourceForm","defaultInstall":true,"deleted":false,"documentationLink":"https://docs.appsmith.com/reference/datasources/graphql#create-queries","iconLocation":"https://s3.us-east-2.amazonaws.com/assets.appsmith.com/logo/graphql.svg","id":"66f2ec66704a8e568824bdc5","isRemotePlugin":false,"isSupportedForAirGap":true,"name":"Authenticated GraphQL API","packageName":"graphql-plugin","policies":[],"policyMap":{},"responseType":"JSON","type":"API","uiComponent":"GraphQLEditorForm"}
|
||||
{"_class":"com.appsmith.server.domains.Plugin","allowUserDatasources":true,"defaultInstall":true,"deleted":false,"documentationLink":"https://docs.appsmith.com/reference/datasources/querying-oracle#create-queries","iconLocation":"https://s3.us-east-2.amazonaws.com/assets.appsmith.com/oracle.svg","id":"66f2ec67704a8e568824bdc6","isRemotePlugin":false,"isSupportedForAirGap":true,"name":"Oracle","packageName":"oracle-plugin","policies":[],"policyMap":{},"responseType":"TABLE","type":"DB","uiComponent":"DbEditorForm"}
|
||||
{"_class":"com.appsmith.server.domains.Plugin","allowUserDatasources":true,"datasourceComponent":"DbEditorForm","defaultInstall":true,"deleted":false,"documentationLink":"https://docs.appsmith.com/connect-data/reference/open-ai","iconLocation":"https://assets.appsmith.com/logo/open-ai.svg","id":"66f2ec6a704a8e568824bdc8","isRemotePlugin":false,"isSupportedForAirGap":true,"name":"Open AI","packageName":"openai-plugin","pluginName":"Open AI","policies":[],"policyMap":{},"responseType":"JSON","type":"AI","uiComponent":"UQIDbEditorForm"}
|
||||
{"_class":"com.appsmith.server.domains.Plugin","allowUserDatasources":true,"datasourceComponent":"DbEditorForm","defaultInstall":true,"deleted":false,"documentationLink":"https://docs.appsmith.com/connect-data/reference/anthropic","iconLocation":"https://assets.appsmith.com/logo/anthropic.svg","id":"66f2ec6b704a8e568824bdc9","isRemotePlugin":false,"isSupportedForAirGap":true,"name":"Anthropic","packageName":"anthropic-plugin","pluginName":"Anthropic","policies":[],"policyMap":{},"responseType":"JSON","type":"AI","uiComponent":"UQIDbEditorForm"}
|
||||
{"_class":"com.appsmith.server.domains.Plugin","allowUserDatasources":true,"datasourceComponent":"DbEditorForm","defaultInstall":true,"deleted":false,"documentationLink":"https://docs.appsmith.com/connect-data/reference/google-ai","iconLocation":"https://assets.appsmith.com/google-ai.svg","id":"66f2ec6b704a8e568824bdca","isRemotePlugin":false,"isSupportedForAirGap":true,"name":"Google AI","packageName":"googleai-plugin","pluginName":"Google AI","policies":[],"policyMap":{},"responseType":"JSON","type":"AI","uiComponent":"UQIDbEditorForm"}
|
||||
{"_class":"com.appsmith.server.domains.Plugin","allowUserDatasources":true,"datasourceComponent":"DbEditorForm","defaultInstall":true,"deleted":false,"documentationLink":"https://docs.appsmith.com/connect-data/reference/databricks","iconLocation":"https://assets.appsmith.com/databricks-logo.svg","id":"66f2ec6c704a8e568824bdcb","isRemotePlugin":false,"isSupportedForAirGap":true,"name":"Databricks","packageName":"databricks-plugin","pluginName":"Databricks","policies":[],"policyMap":{},"responseType":"JSON","type":"DB","uiComponent":"UQIDbEditorForm"}
|
||||
{"_class":"com.appsmith.server.domains.Plugin","allowUserDatasources":true,"datasourceComponent":"DbEditorForm","defaultInstall":true,"deleted":false,"documentationLink":"https://docs.appsmith.com/connect-data/reference/aws-lambda","iconLocation":"https://assets.appsmith.com/aws-lambda-logo.svg","id":"66f2ec6c704a8e568824bdcc","isRemotePlugin":false,"isSupportedForAirGap":true,"name":"AWS Lambda","packageName":"aws-lambda-plugin","pluginName":"AWS Lambda","policies":[],"policyMap":{},"responseType":"JSON","type":"REMOTE","uiComponent":"UQIDbEditorForm"}
|
||||
{"_class":"com.appsmith.server.domains.Plugin","allowUserDatasources":true,"datasourceComponent":"DbEditorForm","defaultInstall":true,"deleted":false,"documentationLink":"https://docs.appsmith.com/connect-data/reference/appsmith-ai","iconLocation":"https://assets.appsmith.com/logo/appsmith-ai.svg","id":"66f2ec6c704a8e568824bdcd","isRemotePlugin":false,"isSupportedForAirGap":true,"name":"Appsmith AI","packageName":"appsmithai-plugin","pluginName":"Appsmith AI","policies":[],"policyMap":{},"responseType":"JSON","type":"AI","uiComponent":"UQIDbEditorForm"}
|
||||
1
deploy/docker/fs/opt/appsmith/baseline-ce/sequence.jsonl
Normal file
1
deploy/docker/fs/opt/appsmith/baseline-ce/sequence.jsonl
Normal file
|
|
@ -0,0 +1 @@
|
|||
{"id":"66f2ec65ebbcb7fe2c1b26ef","name":"datasource","nextNumber":1,"policyMap":{}}
|
||||
1
deploy/docker/fs/opt/appsmith/baseline-ce/tenant.jsonl
Normal file
1
deploy/docker/fs/opt/appsmith/baseline-ce/tenant.jsonl
Normal file
|
|
@ -0,0 +1 @@
|
|||
{"_class":"com.appsmith.server.domains.Tenant","deleted":false,"displayName":"Default","id":"66f2ec66704a8e568824bdb6","policies":[{"permission":"manage:tenants","permissionGroups":["66f2ec66704a8e568824bdb9"]}],"policyMap":{"manage:tenants":{"permission":"manage:tenants","permissionGroups":["66f2ec66704a8e568824bdb9"]}},"pricingPlan":"FREE","slug":"default","tenantConfiguration":{"instanceName":"Appsmith","isAtomicPushAllowed":false,"migrationStatus":"COMPLETED"}}
|
||||
9
deploy/docker/fs/opt/appsmith/baseline-ce/theme.jsonl
Normal file
9
deploy/docker/fs/opt/appsmith/baseline-ce/theme.jsonl
Normal file
File diff suppressed because one or more lines are too long
1
deploy/docker/fs/opt/appsmith/baseline-ce/user.jsonl
Normal file
1
deploy/docker/fs/opt/appsmith/baseline-ce/user.jsonl
Normal file
|
|
@ -0,0 +1 @@
|
|||
{"_class":"com.appsmith.server.domains.User","deleted":false,"email":"anonymousUser","id":"66f2ec66704a8e568824bdb7","isAnonymous":true,"isEnabled":true,"isSystemGenerated":true,"name":"anonymousUser","passwordResetInitiated":false,"policies":[],"policyMap":{},"source":"FORM","tenantId":"66f2ec66704a8e568824bdb6","workspaceIds":[]}
|
||||
162
deploy/docker/fs/opt/appsmith/utils/bin/move-to-postgres.mjs
Normal file
162
deploy/docker/fs/opt/appsmith/utils/bin/move-to-postgres.mjs
Normal file
|
|
@ -0,0 +1,162 @@
|
|||
/**
|
||||
* Moves data from MongoDB to Postgres.
|
||||
*
|
||||
* @param {string} mongoDbUrl - The URL of the MongoDB.
|
||||
* @param {string} mongoDumpFile - The path to the MongoDB dump file.
|
||||
* @param {boolean} isBaselineMode - Flag indicating whether the script is running in baseline mode.
|
||||
* @returns {Promise<void>} - A promise that resolves when the data migration is complete.
|
||||
*/
|
||||
import {spawn} from "child_process";
|
||||
import {MongoClient} from "mongodb";
|
||||
import * as fs from "node:fs";
|
||||
|
||||
let isBaselineMode = false;
|
||||
|
||||
// Don't use `localhost` here, it'll try to connect on IPv6, irrespective of whether you have it enabled or not.
|
||||
let mongoDbUrl;
|
||||
|
||||
let mongoDumpFile = null;
|
||||
const EXPORT_ROOT = "/appsmith-stacks/mongo-data";
|
||||
|
||||
for (let i = 2; i < process.argv.length; ++i) {
|
||||
const arg = process.argv[i];
|
||||
if (arg.startsWith("--mongodb-url=") && !mongoDbUrl) {
|
||||
mongoDbUrl = extractValueFromArg(arg);
|
||||
} else if (arg.startsWith("--mongodb-dump=") && !mongoDumpFile) {
|
||||
mongoDumpFile = extractValueFromArg(arg);
|
||||
} else if (arg === "--baseline") {
|
||||
isBaselineMode = true;
|
||||
console.warn("Running in baseline mode. If you're not an Appsmith team member, we sure hope you know what you're doing.")
|
||||
} else {
|
||||
console.error("Unknown/unexpected argument: " + arg);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
if (!mongoDbUrl && !mongoDumpFile) {
|
||||
console.error("No source specified");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
let mongoServer;
|
||||
if (mongoDumpFile) {
|
||||
fs.mkdirSync("/tmp/db-tmp", {recursive: true});
|
||||
|
||||
mongoServer = spawn("mongod", ["--bind_ip_all", "--dbpath", "/tmp/db-tmp", "--port", "27500"], {
|
||||
stdio: "inherit",
|
||||
});
|
||||
|
||||
mongoDbUrl = "mongodb://localhost/tmp";
|
||||
|
||||
// mongorestore 'mongodb://localhost/' --archive=mongodb-data.gz --gzip --nsFrom='appsmith.*' --nsTo='appsmith.*'
|
||||
spawn("mongorestore", [mongoDbUrl, "--archive=" + mongoDumpFile, "--gzip", "--noIndexRestore"]);
|
||||
}
|
||||
|
||||
const mongoClient = new MongoClient(mongoDbUrl);
|
||||
mongoClient.on("error", console.error);
|
||||
await mongoClient.connect();
|
||||
const mongoDb = mongoClient.db();
|
||||
|
||||
// Make sure EXPORT_ROOT directory is empty
|
||||
fs.rmSync(EXPORT_ROOT, { recursive: true, force: true });
|
||||
fs.mkdirSync(EXPORT_ROOT, { recursive: true });
|
||||
|
||||
const filters = {};
|
||||
|
||||
if (isBaselineMode) {
|
||||
filters.config = {
|
||||
// Remove the "appsmith_registered" value, since this is baseline static data, and we want new instances to do register.
|
||||
name: {$ne: "appsmith_registered"},
|
||||
};
|
||||
filters.plugin = {
|
||||
// Remove saas plugins so they can be fetched from CS again, as usual.
|
||||
packageName: {$ne: "saas-plugin"},
|
||||
};
|
||||
}
|
||||
|
||||
const collectionNames = await mongoDb.listCollections({}, { nameOnly: true }).toArray();
|
||||
const sortedCollectionNames = collectionNames.map(collection => collection.name).sort();
|
||||
|
||||
for await (const collectionName of sortedCollectionNames) {
|
||||
|
||||
console.log("Collection:", collectionName);
|
||||
if (isBaselineMode && collectionName.startsWith("mongock")) {
|
||||
continue;
|
||||
}
|
||||
let outFile = null;
|
||||
for await (const doc of mongoDb.collection(collectionName).find(filters[collectionName])) {
|
||||
|
||||
// Skip archived objects as they are not migrated during the Mongock migration which may end up failing for the
|
||||
// constraints in the Postgres DB.
|
||||
if (isArchivedObject(doc)) {
|
||||
continue;
|
||||
}
|
||||
transformFields(doc);
|
||||
if (doc.policyMap == null) {
|
||||
doc.policyMap = {};
|
||||
}
|
||||
|
||||
if (outFile == null) {
|
||||
// Don't create the file unless there's data to write.
|
||||
outFile = fs.openSync(EXPORT_ROOT + "/" + collectionName + ".jsonl", "w");
|
||||
}
|
||||
|
||||
fs.writeSync(outFile, toJsonSortedKeys(doc) + "\n");
|
||||
}
|
||||
|
||||
if (outFile != null) {
|
||||
fs.closeSync(outFile);
|
||||
}
|
||||
}
|
||||
|
||||
await mongoClient.close();
|
||||
mongoServer?.kill();
|
||||
|
||||
console.log("done");
|
||||
|
||||
// TODO(Shri): We shouldn't need this.
|
||||
process.exit(0);
|
||||
|
||||
function extractValueFromArg(arg) {
|
||||
return arg.replace(/^.*?=/, "");
|
||||
}
|
||||
|
||||
function isArchivedObject(doc) {
|
||||
return doc.deleted === true || doc.deletedAt != null;
|
||||
}
|
||||
|
||||
function toJsonSortedKeys(obj) {
|
||||
// We want the keys sorted in the serialized JSON string, so that everytime we run this script, we don't see diffs
|
||||
// that are just keys being reshuffled, which we don't care about, and don't need a diff for.
|
||||
return JSON.stringify(obj, replacer);
|
||||
}
|
||||
|
||||
function replacer(key, value) {
|
||||
// Ref: https://gist.github.com/davidfurlong/463a83a33b70a3b6618e97ec9679e490
|
||||
return value instanceof Object && !Array.isArray(value) ?
|
||||
Object.keys(value)
|
||||
.sort()
|
||||
.reduce((sorted, key) => {
|
||||
sorted[key] = value[key];
|
||||
return sorted
|
||||
}, {}) :
|
||||
value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Method to transform the data in the object to be compatible with Postgres.
|
||||
* Updates:
|
||||
* 1. Changes the _id field to id, and removes the _id field.
|
||||
* @param {Document} obj - The object to transform.
|
||||
* @returns {void} - No return value.
|
||||
*/
|
||||
function transformFields(obj) {
|
||||
for (const key in obj) {
|
||||
if (key === "_id") { // Change the _id field to id
|
||||
obj.id = obj._id.toString();
|
||||
delete obj._id;
|
||||
} else if (typeof obj[key] === "object") {
|
||||
transformFields(obj[key]);
|
||||
}
|
||||
}
|
||||
}
|
||||
67
scripts/regen-baseline.sh
Executable file
67
scripts/regen-baseline.sh
Executable file
|
|
@ -0,0 +1,67 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
set -o errexit
|
||||
set -o nounset
|
||||
set -o pipefail
|
||||
if [[ ${TRACE-0} == 1 ]]; then
|
||||
set -o xtrace
|
||||
fi
|
||||
|
||||
#
|
||||
# 1. Run a new Appsmith container.
|
||||
# 2. Wait for, and ensure backend server is up.
|
||||
# 3. Kill backend.
|
||||
# 4. Ensure connection to the embedded MongoDB.
|
||||
# 5. Run the export script against the embedded MongoDB.
|
||||
# 6. If needed, move the jsonl files to the right place.
|
||||
# 7. Remove the container.
|
||||
#
|
||||
# Unfortunately, everytime this script runs, there will be a diff in the jsonl files. Always. This is because the ObjectID
|
||||
# values, and the "createdAt" values would be different everytime a new Appsmith container is started up. This is...
|
||||
# _okay_ for now. Since once we passover to only writing migrations on Postgres, these files will effectively be sealed.
|
||||
# So we have to put up with that "problem" only until then. Which makes it not worth our time to solve.
|
||||
# That said, please carefully review the diff nevertheless, keeping in mind the implementation of the migrations that
|
||||
# reads these files.
|
||||
|
||||
container_name=appsmith-for-baseline
|
||||
project_root="$(git rev-parse --show-toplevel)"
|
||||
|
||||
edition=ce
|
||||
if [[ "$(git remote get-url origin)" == *appsmithorg/appsmith-ee.git ]]; then
|
||||
edition=ee
|
||||
fi
|
||||
|
||||
docker rm --force "$container_name"
|
||||
docker run \
|
||||
--detach \
|
||||
--name "$container_name" \
|
||||
--pull always "appsmith/appsmith-$edition":release
|
||||
|
||||
docker cp \
|
||||
"$project_root/deploy/docker/fs/opt/appsmith/utils/bin/move-to-postgres.mjs" \
|
||||
"$container_name":/opt/appsmith/utils/export.mjs
|
||||
|
||||
docker exec "$container_name" bash -c '
|
||||
set -o errexit
|
||||
set -o nounset
|
||||
for attempt in {1..99}; do
|
||||
if curl --silent --fail --fail-early 127.0.0.1:8080/api/v1/health; then
|
||||
break
|
||||
fi
|
||||
echo "Waiting for backend to come up..."
|
||||
sleep 3
|
||||
done
|
||||
if ! supervisorctl stop editor postgres rts backend redis; then
|
||||
echo "Warning: Some services may not have stopped correctly."
|
||||
fi
|
||||
source /appsmith-stacks/configuration/docker.env
|
||||
node utils/export.mjs --mongodb-url="$APPSMITH_DB_URL" --baseline
|
||||
'
|
||||
|
||||
baseline_dir="$project_root/deploy/docker/fs/opt/appsmith/baseline-$edition"
|
||||
rm -rf "$baseline_dir"
|
||||
docker cp "$container_name":/appsmith-stacks/mongo-data "$baseline_dir"
|
||||
docker rm -f "$container_name"
|
||||
echo Removed "$container_name" and copied the new baseline files.
|
||||
|
||||
echo Finish
|
||||
Loading…
Reference in New Issue
Block a user