fix: cleanup stale postgres postmaster.pid (#35171)

## Description
issue of Postgres not coming up with error` Previous Postgres was not
shutdown cleanly. Please start and stop Postgres 14 properly with
'supervisorctl' only. `


https://www.notion.so/appsmith/Closed-Beta-Customer-issues-45a274a9eb8e4762a72cbff74cd3bad5?pvs=4#ecca04d205414f25a289884cebdc0f9b


Fixes
https://app.zenhub.com/workspaces/workflows-pod-652fff131a95920b9bf2bc7e/issues/zh/226_

## Automation

/ok-to-test tags="@tag.Sanity"

### 🔍 Cypress test results
<!-- This is an auto-generated comment: Cypress test results  -->
> [!TIP]
> 🟢 🟢 🟢 All cypress tests have passed! 🎉 🎉 🎉
> Workflow run:
<https://github.com/appsmithorg/appsmith/actions/runs/10107093349>
> Commit: 357bf5d2ff0c1f65b4cbf46b0f3ee823ac192614
> <a
href="https://internal.appsmith.com/app/cypress-dashboard/rundetails-65890b3c81d7400d08fa9ee5?branch=master&workflowId=10107093349&attempt=1"
target="_blank">Cypress dashboard</a>.
> Tags: `@tag.Sanity`
> Spec:
> <hr>Fri, 26 Jul 2024 07:15:19 UTC
<!-- end of auto-generated comment: Cypress test results  -->


## Communication
Should the DevRel and Marketing teams inform users about this change?
- [ ] Yes
- [ ] No


<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

## Summary by CodeRabbit

- **New Features**
- Enhanced PostgreSQL upgrade process with improved error handling and
robust management of old server instances.

- **Bug Fixes**
- Reinstituted logic for checking and managing the `postmaster.pid`
file, ensuring proper startup and shutdown of old PostgreSQL servers.

- **Refactor**
- Improved formatting and readability of shell scripts without altering
their functionality.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

deploy/docker/fs/opt/appsmith/entrypoint.sh

@@ -437,10 +437,19 @@ init_postgres() {
 
 }
 
-safe_init_postgres(){
-runEmbeddedPostgres=1
-# fail safe to prevent entrypoint from exiting, and prevent postgres from starting
-init_postgres || runEmbeddedPostgres=0
+safe_init_postgres() {
+  runEmbeddedPostgres=1
+  # fail safe to prevent entrypoint from exiting, and prevent postgres from starting
+  # when runEmbeddedPostgres=0 , postgres conf file for supervisord will not be copied
+  # so postgres will not be started by supervisor. Explicit message helps us to know upgrade script failed.
+
+  if init_postgres; then
+    tlog "init_postgres succeeded."
+  else
+    local exit_status=$?
+    tlog "init_postgres failed with exit status $exit_status."
+    runEmbeddedPostgres=0
+  fi
 }
 
 setup_caddy() {

deploy/docker/fs/opt/appsmith/pg-upgrade.sh

@@ -37,11 +37,6 @@ if [[ -z "$old_version" ]]; then
 	exit
 fi
 
-if [[ -f "$pg_data_dir/postmaster.pid" ]]; then
-	tlog "Previous Postgres was not shutdown cleanly. Please start and stop Postgres $old_version properly with 'supervisorctl' only." >&2
-	exit 1
-fi
-
 top_available_version="$(postgres --version | grep -o '[[:digit:]]\+' | head -1)"
 
 declare -a to_uninstall
@@ -55,14 +50,30 @@ if [[ "$old_version" == 13 && "$top_available_version" > "$old_version" ]]; then
 		to_uninstall+=("postgresql-$old_version")
 	fi
 
-  new_version="$((old_version + 1))"
-  new_data_dir="$pg_data_dir-$new_version"
+	if [[ -f "$pg_data_dir/postmaster.pid" ]]; then
+		# Start old PostgreSQL using pg_ctl
+		tlog "Stale postmaster.pid found. Starting old PostgreSQL $old_version using pg_ctl to cleanup."
+		su postgres -c "$postgres_path/$old_version/bin/pg_ctl start -D '$pg_data_dir' "
 
-  # `pg_upgrade` writes log to current folder. So change to a temp folder first.
-  rm -rf "$TMP/pg_upgrade" "$new_data_dir"
-  mkdir -p "$TMP/pg_upgrade" "$new_data_dir"
-  chown -R postgres "$TMP/pg_upgrade" "$new_data_dir"
-  cd "$TMP/pg_upgrade"
+		# Wait for old PostgreSQL to be ready
+		until su postgres -c "$postgres_path/$old_version/bin/pg_isready"; do
+			tlog "Waiting for PostgreSQL $old_version to start..."
+			sleep 1
+		done
+
+		# Shut down PostgreSQL gracefully using pg_ctl
+		su postgres -c "$postgres_path/$old_version/bin/pg_ctl stop -D '$pg_data_dir' -m smart"
+		tlog "PostgreSQL $old_version has been shut down."
+	fi
+
+	new_version="$((old_version + 1))"
+	new_data_dir="$pg_data_dir-$new_version"
+
+	# `pg_upgrade` writes log to current folder. So change to a temp folder first.
+	rm -rf "$TMP/pg_upgrade" "$new_data_dir"
+	mkdir -p "$TMP/pg_upgrade" "$new_data_dir"
+	chown -R postgres "$TMP/pg_upgrade" "$new_data_dir"
+	cd "$TMP/pg_upgrade"
 
 	# Required by the temporary Postgres server started by `pg_upgrade`.
 	chown postgres /etc/ssl/private/ssl-cert-snakeoil.key
@@ -88,7 +99,7 @@ if [[ "$old_version" == 13 && "$top_available_version" > "$old_version" ]]; then
 fi
 
 if [[ -n "${#to_uninstall[@]}" ]]; then
-	apt-get purge "${to_uninstall[@]}"
+	DEBIAN_FRONTEND=noninteractive apt-get purge --yes "${to_uninstall[@]}"
 	apt-get clean
 fi