PromucFlow_constructor/app/server/scripts/node/acl-migration.js

if (process.argv.length !== 4) {
	console.error("Takes two arguments, the API URL (like 'https://localhost/api/v1/')" +
		" and the MongoDB URL (like 'mongodb://localhost:27017/mobtools').");
	process.exit(1);
}

const [API_URL, MONGODB_URL] = process.argv.slice(2);

const SUPER_EMAIL = "superuser_acl@appsmith.com";
const SUPER_PASSWORD = "migration";


const axios = require("axios");
const https = require("https");
const tough = require("tough-cookie");
const axiosCookieJarSupport = require("axios-cookiejar-support").default;
const { MongoClient, ObjectID } = require("mongodb");


const mongoClient = new MongoClient(MONGODB_URL, {
	useNewUrlParser: true,
	useUnifiedTopology: true,
});

axiosCookieJarSupport(axios);


console.time("total time taken");
migrate()
	.then(() => console.log("Finished Successfully."))
	.catch(error => console.error(error))
	.finally(() => {
		mongoClient.close();
		console.timeEnd("total time taken");
		console.log();
	});


async function migrate() {
	const ax = axios.create({
		baseURL: API_URL,

		headers: {
			"content-type": "application/json",
			"origin": API_URL.match(/^\w+:\/\/[^/]+/)[0],  // Match just the protocol, host and port in the URL.
		},

		// This is for the Authorization header, for applying basic auth.
		auth: {
			username: "api_user",
			password: "8uA@;&mB:cnvN~{#",
		},

		// Keep session over multiple requests.
		withCredentials: true,
		jar: new tough.CookieJar(),

		// This is needed so Axios won't yell at self-signed certificates.
		httpsAgent: new https.Agent({
			rejectUnauthorized: false,
		}),
	});

	const con = await mongoClient.connect();
	const db = con.db();

	await run("Deleting super user from the database, if exists from a previous failed run.", purgeSuperUser, db, true);

	await run("Signing up using API.", signUpSuperUser, ax);

	await run("Logging in using API.", loginSuperUser, ax);

	await run("Inserting super permissions in the database.", addSuperPermissions, db);

	await run("Adding user own profile permissions in the database.", addSelfPermissions, db);

	const organizationEmailPairs =
		await run("Finding organization-user pairs.", findOrganizationUserPairs, db);

	await run("Inviting users to their organizations using API.", inviteUsers, ax, organizationEmailPairs);

	await run("Logging the super user out.", logoutSuperUser, ax);

	await run("Removing super user permissions in the database.", removeSuperPermissions, db);

	await run("Deleting super user from the database.", purgeSuperUser, db);

	await run("Running checks.", runChecks, db);

	await run("Finding orphans (documents without any policies).", findOrphans, db);
}

async function run(label, fn, ...args) {
	// Runs the given async function with the given args, inside a console group made of the given label.
	console.time("time");
	console.group(label);
	const result = await fn(...args);
	console.timeEnd("time");
	console.groupEnd();
	console.log();
	return result;
}

async function signUpSuperUser(ax) {
	const signUpResponse = await ax.post("users", {
		name: SUPER_EMAIL,
		email: SUPER_EMAIL,
		source: "FORM",
		state: "ACTIVATED",
		isEnabled: "true",
		password: SUPER_PASSWORD,
	});

	if (!signUpResponse.data.responseMeta.success) {
		console.log("Sign up failed", signUpResponse.data.responseMeta.error);
		throw new Error("Sign up failed: " + signUpResponse.data.responseMeta.error.message);
	}
}

async function loginSuperUser(ax) {
	const loginParams = new URLSearchParams();
	loginParams.append("username", SUPER_EMAIL);
	loginParams.append("password", SUPER_PASSWORD);

	const loginResponse = await ax.post("login", loginParams, {
		headers: {
			"content-type": "application/x-www-form-urlencoded",
		},
	});

	if (loginResponse.request.path !== "/") {
		console.log("Login failed", loginResponse);
		throw new Error("Login failed: " + loginResponse.data);
	}

	console.log("Logged in. Trying to get super user profile to verify session.");
	console.log("cookieJar.store", ax.defaults.jar.store);

	const profileResponse = await ax.get("users/me");
	if (!profileResponse.data.responseMeta.success) {
		console.log(profileResponse);
		throw new Error("Failure logging in as super user.");
	}
}

async function logoutSuperUser(ax) {
	await ax.post("logout");
}

function* computeSuperPermissions() {
	for (const collectionName of ["organization", "page", "action", "application", "datasource"]) {
		for (const scope of ["read", "manage"]) {
			const permission = scope + ":" + collectionName + "s";
			yield [collectionName, permission];
		}
	}
}

async function addSuperPermissions(db) {
	const promises = [];

	for (const [collectionName, permission] of computeSuperPermissions()) {
		promises.push(addPermission(db.collection(collectionName), permission));
	}

	await Promise.all(promises);

	async function addPermission(collection, permission) {
		console.log(`Adding policy for permission ${permission}.`);
		await collection.updateMany(
			{ "policies.permission": { $ne: permission } },
			{
				$push: {
					policies: {
						permission,
						users: [],
						groups: [],
					},
				},
			},
		);

		console.log(`Adding superuser to permission ${permission}.`);
		await collection.updateMany(
			{ "policies.permission": permission },
			{
				$addToSet: {
					"policies.$.users": SUPER_EMAIL,
				},
			},
		);
	}
}

async function removeSuperPermissions(db) {
	const promises = [];

	for (const [collectionName, permission] of computeSuperPermissions()) {
		promises.push(removePermission(db.collection(collectionName), permission));
	}

	await Promise.all(promises);

	async function removePermission(collection, permission) {
		console.log(`Removing superuser from permission '${permission}'.`);

		await collection.updateMany(
			{ "policies.permission": permission },
			{
				$pull: {
					"policies.$.users": SUPER_EMAIL,
				},
			},
		);

		console.log(`Cleaning up empty policies for permission '${permission}'.`)
		await collection.updateMany(
			{ policies: { $exists: true } },
			{
				$pullAll: {
					policies: [
						{ permission, users: [], groups: [] },
					],
				},
			}
		);
	}
}

async function addSelfPermissions(db) {
	const userCollection = db.collection("user");

	const cursor = userCollection
		.find({ email: { $exists: true } })
		.project({ email: 1, policies: 1 });

	const promises = [];

	while (await cursor.hasNext()) {
		const user = await cursor.next();
		if (user === null) {
			break;
		}

		const policies = user.policies || [];

		const permissionsToAdd = new Set([
			"read:users",
			"manage:users",
			"resetPassword:users",
			"read:userOrganization",
			"manage:userOrganization",
		]);

		for (const policy of policies) {
			if (permissionsToAdd.delete(policy.permission) && policy.users.indexOf(user.email) < 0) {
				policy.users.push(user.email);
			}
		}

		for (const permission of permissionsToAdd) {
			policies.push({
				permission,
				users: [
					user.email,
				],
				groups: [],
			})
		}

		promises.push(userCollection.updateOne({ _id: user._id }, { $set: { policies } }));
	}

	await Promise.all(promises);
}

async function findOrganizationUserPairs(db) {
	const pairs = [];
	return new Promise((resolve, reject) => {
		db.collection("user")
			.find({ organizationIds: { $exists: true }, email: { $exists: true, $not: { $eq: SUPER_EMAIL } } })
			.project({ email: 1, organizationIds: 1 })
			.forEach(
				doc => {
					for (const organizationId of doc.organizationIds) {
						pairs.push({ organizationId, email: doc.email });
					}
				},
				err => {
					if (err === null) {
						console.info(`Identified ${pairs.length} invitation(s) to be sent.`);
						resolve(pairs);
					} else {
						console.error(err);
						reject(err);
					}
				}
			);
	});
}

async function inviteUsers(ax, organizationEmailPairs) {
	for (const { organizationId, email } of organizationEmailPairs) {
		console.log(`Inviting '${email}' to organizationId: '${organizationId}'.`);

		const membersResponse = (await ax.get(`organizations/${organizationId}/members`)).data;
		if (membersResponse.responseMeta.success
				&& membersResponse.data.filter(({ username }) => username === email).length > 0) {
			console.log("User already has access to this organization. Not inviting.");
			continue;
		}

		const response = await ax.post("users/invite", {
			email,
			orgId: organizationId,
			roleName: "Administrator",
		});

		if (!response.data.responseMeta.success) {
			console.error("response.data", response.data);
		}
	}
}

async function purgeSuperUser(db, isSilent) {
	console.log("Deleting super user.");
	const deleteUsersResult = await db.collection("user").deleteOne({ email: SUPER_EMAIL });
	if (!isSilent && deleteUsersResult.deletedCount !== 1) {
		throw new Error("Unexpected deleted count when deleting super user: " + deleteUsersResult.deletedCount);
	}

	console.log("Deleting super user's personal organization.");
	const deleteOrgResult = await db.collection("organization").deleteOne({ name: SUPER_EMAIL + "'s Personal Organization" });
	if (!isSilent && deleteOrgResult.deletedCount !== 1) {
		throw new Error("Unexpected deleted count when deleting super user's organization: " + deleteOrgResult.deletedCount);
	}
}

async function runChecks(db) {
	for (const collection of await db.collections()) {
		if (await collection.countDocuments({ "policies.users": SUPER_EMAIL }) !== 0 ) {
			console.error(`Super user lives on in the '${collection.collectionName}' collection.`);
		}
	}
}

async function findOrphans(db) {
	const counts = new Map;
	for (const collectionName of ["organization", "page", "action", "application", "datasource"]) {
		counts.set(
			collectionName,
			await db.collection(collectionName).countDocuments({
				policies: { $exists: true, $size: 0 },
				deleted: false,
			})
		);
	}

	console.log("Documents with policies=[], and deleted=false:", counts);

	const organizationCollection = db.collection("organization");

	let missedOrganizations = 0;
	const organizationsInLimbo = new Set();
	for await (const organization of cursorIterator(
		organizationCollection.find({ policies: { $exists: true, $size: 0 }, deleted: false }))
		) {

		const count = await db.collection("user").countDocuments({
			organizationIds: organization._id.toString(),
			deleted: false,
		});

		if (count > 0) {
			++missedOrganizations;
			console.log("Missed organization", organization._id.toString());
		} else {
			organizationsInLimbo.add(organization._id.toString());
		}
	}

	let missedDatasources = 0;
	const datasourcesInLimbo = new Set();
	for await (const datasource of cursorIterator(db.collection("datasource")
		.find({
			policies: { $exists: true, $size: 0 },
			organizationId: { $exists: true },
			deleted: false,
		}))) {

		if (organizationsInLimbo.has(datasource.organizationId) || !ObjectID.isValid(datasource.organizationId)) {
			datasourcesInLimbo.add(datasource._id.toString());
			continue;
		}

		const count = await organizationCollection.countDocuments({
			_id: new ObjectID(datasource.organizationId),
			deleted: false,
		});

		if (count > 0) {
			++missedDatasources;
			console.log("Missed datasource", datasource._id.toString());
		} else {
			datasourcesInLimbo.add(datasource._id.toString());
		}
	}

	let missedApplications = 0;
	const applicationsInLimbo = new Set();
	for await (const application of cursorIterator(db.collection("application")
		.find({
			policies: { $exists: true, $size: 0 },
			organizationId: { $exists: true },
			deleted: false,
		}))) {

		if (organizationsInLimbo.has(application.organizationId) || !ObjectID.isValid(application.organizationId)) {
			applicationsInLimbo.add(application._id.toString());
			continue;
		}

		const count = await organizationCollection.countDocuments({
			_id: new ObjectID(application.organizationId),
			deleted: false,
		});

		if (count > 0) {
			++missedApplications;
			console.log("Missed application", application._id.toString());
		} else {
			applicationsInLimbo.add(application._id.toString());
		}
	}

	const pageCursor = db.collection("page").find({
		policies: { $exists: true, $size: 0 },
		applicationId: { $exists: true },
		deleted: false,
	});

	let missedPages = 0;
	const pagesInLimbo = new Set();

	for await (const page of cursorIterator(pageCursor)) {
		if (applicationsInLimbo.has(page.applicationId) || !ObjectID.isValid(page.applicationId)) {
			pagesInLimbo.add(page._id.toString());
			continue;
		}

		const count = await db.collection("application").countDocuments({
			_id: new ObjectID(page.applicationId),
			deleted: false,
		});

		if (count > 0) {
			++missedPages;
			console.log("Missed page", page._id.toString());
		} else {
			pagesInLimbo.add(page._id.toString());
		}
	}

	const actionCursor = db.collection("action").find({
		policies: { $exists: true, $size: 0 },
		pageId: { $exists: true },
		deleted: false,
	});

	let missedActions = 0;
	const actionsInLimbo = new Set();

	for await (const action of cursorIterator(actionCursor)) {
		if (pagesInLimbo.has(action.pageId) || !ObjectID.isValid(action.pageId)) {
			actionsInLimbo.add(action._id.toString());
			continue;
		}

		const count = await db.collection("page").countDocuments({
			_id: new ObjectID(action.pageId),
			deleted: false,
		});

		if (count > 0) {
			++missedActions;
		} else {
			actionsInLimbo.add(action._id.toString());
		}
	}

	console.log("These are missing policies, where they shouldn've had some policies:", {
		organizations: missedOrganizations,
		datasources: missedDatasources,
		applications: missedApplications,
		pages: missedPages,
		actions: missedActions,
	});

	if (missedOrganizations + missedDatasources + missedApplications + missedPages + missedActions === 0) {
		console.log("Database state looks good.");
	} else {
		console.error("There's some documents that are missing policies, but that should've had some. Please check.");
	}
}

async function* cursorIterator(cursor) {
	while (true) {
		const doc = await cursor.next();
		if (doc === null) {
			break;
		}
		yield doc;
	}
}