chore: Local Caddy support (#31325)

Replaces NGINX for local dev with Caddy.

The advandage here is that the script that generates `Caddyfile` at
runtime on production deployments, is also used to generate the
`Caddyfile` at local development. This reduces the local--production
gap.

/ok-to-test tags="@tag.Sanity"

app/client/.gitignore

@@ -54,6 +54,7 @@ build-storybook.log
 TODO
 
 /nginx
+/caddy
 
 /public/static/wds/
 

app/client/public/index.html

@@ -22,6 +22,7 @@
         // could return either boolean or string based on value
         const parseConfig = (config) => {
           if (
+            (config.startsWith("{") && config.startsWith("}")) ||
             config.indexOf("__") === 0 ||
             config.indexOf("$") === 0 ||
             config.indexOf("%") === 0
@@ -37,9 +38,9 @@
 
           return result;
         };
-        const CLOUD_HOSTING = parseConfig("__APPSMITH_CLOUD_HOSTING__");
+        const CLOUD_HOSTING = parseConfig('{{env "APPSMITH_CLOUD_HOSTING"}}');
-        const ZIPY_KEY = parseConfig("__APPSMITH_ZIPY_SDK_KEY__");
+        const ZIPY_KEY = parseConfig('{{env "APPSMITH_ZIPY_SDK_KEY"}}');
-        const AIRGAPPED = parseConfig("__APPSMITH_AIRGAP_ENABLED__");
+        const AIRGAPPED = parseConfig('{{env "APPSMITH_AIRGAP_ENABLED"}}');
       </script>
     <script>
         window.__APPSMITH_CHUNKS_TO_PRELOAD =
@@ -159,13 +160,13 @@
     <script type="text/javascript">
       const LOG_LEVELS = ["debug", "error"];
       const CONFIG_LOG_LEVEL_INDEX = LOG_LEVELS.indexOf(
-        parseConfig("__APPSMITH_CLIENT_LOG_LEVEL__"),
+        parseConfig('{{env "APPSMITH_CLIENT_LOG_LEVEL"}}'),
       );
 
       const INTERCOM_APP_ID =
         parseConfig("%REACT_APP_INTERCOM_APP_ID%") ||
-        parseConfig("__APPSMITH_INTERCOM_APP_ID__");
+        parseConfig('{{env "APPSMITH_INTERCOM_APP_ID"}}');
-      const DISABLE_INTERCOM = parseConfig("__APPSMITH_DISABLE_INTERCOM__");
+      const DISABLE_INTERCOM = parseConfig('{{env "APPSMITH_DISABLE_INTERCOM"}}');
 
       // Initialize the Intercom library
       if (INTERCOM_APP_ID.length && !DISABLE_INTERCOM) {
@@ -204,40 +205,40 @@
         })();
       }
 
-      window.SENTRY_CONFIG = parseConfig("__APPSMITH_SENTRY_DSN__");
+      window.SENTRY_CONFIG = parseConfig('{{env "APPSMITH_SENTRY_DSN"}}');
       window.APPSMITH_FEATURE_CONFIGS = {
         sentry: {
-          dsn: parseConfig("__APPSMITH_SENTRY_DSN__"),
+          dsn: parseConfig('{{env "APPSMITH_SENTRY_DSN"}}'),
-          release: parseConfig("__APPSMITH_SENTRY_RELEASE__"),
+          release: parseConfig('{{env "APPSMITH_SENTRY_RELEASE"}}'),
-          environment: parseConfig("__APPSMITH_SENTRY_ENVIRONMENT__"),
+          environment: parseConfig('{{env "APPSMITH_SENTRY_ENVIRONMENT"}}'),
         },
         smartLook: {
-          id: parseConfig("__APPSMITH_SMART_LOOK_ID__"),
+          id: parseConfig('{{env "APPSMITH_SMART_LOOK_ID"}}'),
         },
         segment: {
-          apiKey: parseConfig("__APPSMITH_SEGMENT_KEY__"),
+          apiKey: parseConfig('{{env "APPSMITH_SEGMENT_KEY"}}'),
-          ceKey: parseConfig("__APPSMITH_SEGMENT_CE_KEY__"),
+          ceKey: parseConfig('{{env "APPSMITH_SEGMENT_CE_KEY"}}'),
         },
         newRelic:{
-          enableNewRelic: parseConfig("__APPSMITH_NEW_RELIC_ACCOUNT_ENABLE__"),
+          enableNewRelic: parseConfig('{{env "APPSMITH_NEW_RELIC_ACCOUNT_ENABLE"}}'),
-          accountId: parseConfig("__APPSMITH_NEW_RELIC_ACCOUNT_ID__"),
+          accountId: parseConfig('{{env "APPSMITH_NEW_RELIC_ACCOUNT_ID"}}'),
-          applicationId: parseConfig("__APPSMITH_NEW_RELIC_APPLICATION_ID__"),
+          applicationId: parseConfig('{{env "APPSMITH_NEW_RELIC_APPLICATION_ID"}}'),
-          browserAgentlicenseKey: parseConfig("__APPSMITH_NEW_RELIC_BROWSER_AGENT_LICENSE_KEY__"),
+          browserAgentlicenseKey: parseConfig('{{env "APPSMITH_NEW_RELIC_BROWSER_AGENT_LICENSE_KEY"}}'),
-          browserAgentEndpoint: parseConfig("__APPSMITH_NEW_RELIC_BROWSER_AGENT_ENDPOINT__"),
+          browserAgentEndpoint: parseConfig('{{env "APPSMITH_NEW_RELIC_BROWSER_AGENT_ENDPOINT"}}'),
-          otlpLicenseKey: parseConfig("__APPSMITH_NEW_RELIC_OTLP_LICENSE_KEY__"),
+          otlpLicenseKey: parseConfig('{{env "APPSMITH_NEW_RELIC_OTLP_LICENSE_KEY"}}'),
           //OTLP following the naming convention of Sdk initialisation
-          otlpServiceName: parseConfig("__APPSMITH_NEW_RELIC_OTEL_SERVICE_NAME__"),
+          otlpServiceName: parseConfig('{{env "APPSMITH_NEW_RELIC_OTEL_SERVICE_NAME"}}'),
-          otlpEndpoint:parseConfig("__APPSMITH_NEW_RELIC_OTEL_EXPORTER_OTLP_ENDPOINT__"),
+          otlpEndpoint:parseConfig('{{env "APPSMITH_NEW_RELIC_OTEL_EXPORTER_OTLP_ENDPOINT"}}'),
 
         },
         fusioncharts: {
-          licenseKey: parseConfig("__APPSMITH_FUSIONCHARTS_LICENSE_KEY__"),
+          licenseKey: parseConfig('{{env "APPSMITH_FUSIONCHARTS_LICENSE_KEY"}}'),
         },
-        enableMixpanel: parseConfig("__APPSMITH_SEGMENT_KEY__"),
+        enableMixpanel: parseConfig('{{env "APPSMITH_SEGMENT_KEY"}}'),
         algolia: {
-          apiId: parseConfig("__APPSMITH_ALGOLIA_API_ID__"),
+          apiId: parseConfig('{{env "APPSMITH_ALGOLIA_API_ID"}}'),
-          apiKey: parseConfig("__APPSMITH_ALGOLIA_API_KEY__"),
+          apiKey: parseConfig('{{env "APPSMITH_ALGOLIA_API_KEY"}}'),
-          indexName: parseConfig("__APPSMITH_ALGOLIA_SEARCH_INDEX_NAME__"),
+          indexName: parseConfig('{{env "APPSMITH_ALGOLIA_SEARCH_INDEX_NAME"}}'),
         },
         logLevel:
           CONFIG_LOG_LEVEL_INDEX > -1
@@ -245,25 +246,25 @@
             : LOG_LEVELS[1],
         cloudHosting: CLOUD_HOSTING,
         appVersion: {
-          id: parseConfig("__APPSMITH_VERSION_ID__"),
+          id: parseConfig('{{env "APPSMITH_VERSION_ID"}}'),
-          sha: parseConfig("__APPSMITH_VERSION_SHA__"),
+          sha: parseConfig('{{env "APPSMITH_VERSION_SHA"}}'),
-          releaseDate: parseConfig("__APPSMITH_VERSION_RELEASE_DATE__"),
+          releaseDate: parseConfig('{{env "APPSMITH_VERSION_RELEASE_DATE"}}'),
         },
         intercomAppID: INTERCOM_APP_ID,
-        mailEnabled: parseConfig("__APPSMITH_MAIL_ENABLED__"),
+        mailEnabled: parseConfig('{{env "APPSMITH_MAIL_ENABLED"}}'),
         cloudServicesBaseUrl:
-          parseConfig("__APPSMITH_CLOUD_SERVICES_BASE_URL__") ||
+          parseConfig('{{env "APPSMITH_CLOUD_SERVICES_BASE_URL"}}') ||
           "https://cs.appsmith.com",
-        googleRecaptchaSiteKey: parseConfig("__APPSMITH_RECAPTCHA_SITE_KEY__"),
+        googleRecaptchaSiteKey: parseConfig('{{env "APPSMITH_RECAPTCHA_SITE_KEY"}}'),
-        hideWatermark: parseConfig("__APPSMITH_HIDE_WATERMARK__"),
+        hideWatermark: parseConfig('{{env "APPSMITH_HIDE_WATERMARK"}}'),
         disableIframeWidgetSandbox: parseConfig(
-          "__APPSMITH_DISABLE_IFRAME_WIDGET_SANDBOX__",
+          '{{env "APPSMITH_DISABLE_IFRAME_WIDGET_SANDBOX"}}',
         ),
         customerPortalUrl:
-          parseConfig("__APPSMITH_CUSTOMER_PORTAL_URL__") ||
+          parseConfig('{{env "APPSMITH_CUSTOMER_PORTAL_URL"}}') ||
           "https://customer.appsmith.com",
         pricingUrl:
-          parseConfig("__APPSMITH_PRICING_URL__") ||
+          parseConfig('{{env "APPSMITH_PRICING_URL"}}') ||
           "https://www.appsmith.com/pricing",
       };
     </script>

app/client/start-caddy.sh

@@ -0,0 +1,165 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+if [[ -n ${TRACE-} ]]; then
+    set -o xtrace
+fi
+
+cd "$(dirname "$0")"
+
+if [[ ${1-} =~ ^-*h(elp)?$ ]]; then
+    echo 'Run '"$0"' [option...]
+ --http: Require start with http, instead of the default HTTPS.
+
+--https-port: Port to use for https. Default: 443.
+ --http-port: Port to use for http. Default: 80.
+
+        If neither of the above are set, then we use 443 for https, and 80 for http.
+
+--env-file: Specify an alternate env file. Defaults to ".env" at the root of the project.
+
+A single positional argument can be given to set the backend server proxy address. Example:
+
+'"$0"' https://localhost:8080
+'"$0"' https://host.docker.internal:8080
+'"$0"' https://release.app.appsmith.com
+'"$0"' release  # This is identical to the one above
+' >&2
+    exit
+fi
+
+use_https=1
+run_as=caddy
+
+while [[ $# -gt 0 ]]; do
+    case $1 in
+        --https)
+            shift
+            ;;
+        --http)
+            use_https=0
+            shift
+            ;;
+        --https-port)
+            https_listen_port=$2
+            shift 2
+            ;;
+        --http-port)
+            http_listen_port=$2
+            shift 2
+            ;;
+        --env-file)
+            env_file=$2
+            shift
+            shift
+            ;;
+        -*|--*)
+            echo "Unknown option $1" >&2
+            exit 1
+            ;;
+        *)
+            backend="$1"
+            shift
+            ;;
+    esac
+done
+
+if [[ ${backend-} == release ]]; then
+  # Special shortcut for release environment.
+  backend=https://release.app.appsmith.com
+fi
+
+working_dir="$PWD/caddy"
+mkdir -pv "$working_dir"
+
+domain=dev.appsmith.com
+
+upstream_host=localhost
+
+frontend_host=${frontend_host-$upstream_host}
+backend_host=${backend_host-$upstream_host}
+rts_host=${rts_host-$upstream_host}
+
+http_listen_port="${http_listen_port-80}"
+https_listen_port="${https_listen_port-443}"
+
+if [[ -n ${env_file-} && ! -f $env_file ]]; then
+    echo "I got --env-file as '$env_file', but I cannot access it." >&2
+    exit 1
+elif [[ -n ${env_file-} || -f ../../.env ]]; then
+    set -o allexport
+    # shellcheck disable=SC1090
+    source "${env_file-../../.env}"
+    set +o allexport
+else
+    echo "
+        Please populate the .env at the root of the project and run again
+        Or add the environment variables defined in .env.example to the environment
+        -- to enable features
+    " >&2
+fi
+
+caddy_dev_conf="$working_dir/Caddyfile"
+
+APPSMITH_CUSTOM_DOMAIN="$domain" \
+    TMP="$working_dir" \
+    node \
+    "$(git rev-parse --show-toplevel)/deploy/docker/fs/opt/appsmith/caddy-reconfigure.mjs" \
+    --no-finalize-index-html
+
+changed="$(awk '
+/ root / || /import file_server/ { next }
+/acme_ca_root/ { print "log {\n output file '"$working_dir/logs.txt"'\n}"; next }
+/output stdout/ { print "output file '"$working_dir/logs.txt"'"; next }
+/ handle {/ { skip = 1; print "handle {\n import reverse_proxy 3000\n templates {\n  mime \"text/html; charset=utf-8\"\n }\n}\n" }
+/ handle \/info {/ { skip = 0 }
+!skip { print }
+/https:\/\/'"$domain"' / { print "tls internal" }
+' "$caddy_dev_conf")"
+
+echo "$changed" | caddy fmt - > "$caddy_dev_conf"
+
+# Have to stop and start, instead of reload, so that any new env variables are picked up for Caddy's templating.
+caddy stop --config "$caddy_dev_conf" || true
+remaining_listeners="$(
+    lsof -nP "-iTCP:$http_listen_port" -sTCP:LISTEN || true
+    lsof -nP "-iTCP:$https_listen_port" -sTCP:LISTEN || true
+)"
+if [[ -n $remaining_listeners ]]; then
+    echo $'\nThe following processes are listening on the ports we want to use:\n'"$remaining_listeners"$'\n' >&2
+    answer=
+    for attempt in 1 2 3; do
+        echo -n 'Kill and proceed? [y/n] ' >&2
+        read -rn1 answer
+        if [[ $answer == y ]]; then
+            (lsof -t "-iTCP:$http_listen_port" -sTCP:LISTEN | xargs kill) || true
+            (lsof -t "-iTCP:$https_listen_port" -sTCP:LISTEN | xargs kill) || true
+            break
+        elif [[ $answer == n || $attempt == 3 ]]; then
+            echo $'\nAborting.' >&2
+            exit 1
+        fi
+    done
+fi
+
+caddy start --config "$caddy_dev_conf"
+stop_cmd="caddy --config \"$caddy_dev_conf\" stop"
+
+url_to_open=""
+if [[ $use_https == 1 ]]; then
+    url_to_open="https://$domain"
+    if [[ $https_listen_port != 443 ]]; then
+        url_to_open="$url_to_open:$https_listen_port"
+    fi
+else
+    url_to_open="http://localhost"
+    if [[ $http_listen_port != 80 ]]; then
+        url_to_open="$url_to_open:$http_listen_port"
+    fi
+fi
+
+echo '✅ Started Caddy'
+echo "ℹ️  Stop with: $stop_cmd"
+echo "🎉 $url_to_open"

app/client/start-https.sh

@@ -7,6 +7,14 @@ if [[ -n ${TRACE-} ]]; then
     set -o xtrace
 fi
 
+{
+  echo
+  echo "-----------------------------------"
+  echo " ⚠️ This script is deprecated. Please 'brew install caddy' and use start-caddy.sh instead."
+  echo "-----------------------------------"
+  echo
+} >&2
+
 cd "$(dirname "$0")"
 
 if [[ ${1-} =~ ^-*h(elp)?$ ]]; then
@@ -205,6 +213,12 @@ rm -rf "$nginx_dev_conf"
 
 worker_connections=1024
 
+substitutions="$(
+  grep -Eo '{{env "APPSMITH_\w+"}}' public/index.html \
+    | cut -d\" -f2 \
+    | awk '{print "sub_filter '\''{{env \"" $0 "\"}}'\'' '\''" ENVIRON[$0] "'\'';"}'
+)"
+
 echo "
 worker_processes  1;
 
@@ -275,25 +289,7 @@ $(if [[ $use_https == 1 ]]; then echo "
         sub_filter_once off;
         location / {
             proxy_pass $frontend;
-            sub_filter __APPSMITH_SENTRY_DSN__ '${APPSMITH_SENTRY_DSN-}';
+            $substitutions
-            sub_filter __APPSMITH_SMART_LOOK_ID__ '${APPSMITH_SMART_LOOK_ID-}';
-            sub_filter __APPSMITH_SEGMENT_KEY__ '${APPSMITH_SEGMENT_KEY-}';
-            sub_filter __APPSMITH_ALGOLIA_API_ID__ '${APPSMITH_ALGOLIA_API_ID-}';
-            sub_filter __APPSMITH_ALGOLIA_SEARCH_INDEX_NAME__ '${APPSMITH_ALGOLIA_SEARCH_INDEX_NAME-}';
-            sub_filter __APPSMITH_ALGOLIA_API_KEY__ '${APPSMITH_ALGOLIA_API_KEY-}';
-            sub_filter __APPSMITH_CLIENT_LOG_LEVEL__ '${APPSMITH_CLIENT_LOG_LEVEL-}';
-            sub_filter __APPSMITH_SENTRY_RELEASE__ '${APPSMITH_SENTRY_RELEASE-}';
-            sub_filter __APPSMITH_SENTRY_ENVIRONMENT__ '${APPSMITH_SENTRY_ENVIRONMENT-}';
-            sub_filter __APPSMITH_VERSION_ID__ '${APPSMITH_VERSION_ID-}';
-            sub_filter __APPSMITH_VERSION_RELEASE_DATE__ '${APPSMITH_VERSION_RELEASE_DATE-}';
-            sub_filter __APPSMITH_INTERCOM_APP_ID__ '${APPSMITH_INTERCOM_APP_ID-}';
-            sub_filter __APPSMITH_MAIL_ENABLED__ '${APPSMITH_MAIL_ENABLED-}';
-            sub_filter __APPSMITH_CLOUD_SERVICES_BASE_URL__ '${APPSMITH_CLOUD_SERVICES_BASE_URL-}';
-            sub_filter __APPSMITH_RECAPTCHA_SITE_KEY__ '${APPSMITH_RECAPTCHA_SITE_KEY-}';
-            sub_filter __APPSMITH_DISABLE_INTERCOM__ '${APPSMITH_DISABLE_INTERCOM-}';
-            sub_filter __APPSMITH_ZIPY_SDK_KEY__ '${APPSMITH_ZIPY_SDK_KEY-}';
-            sub_filter __APPSMITH_HIDE_WATERMARK__ '${APPSMITH_HIDE_WATERMARK-}';
-            sub_filter __APPSMITH_DISABLE_IFRAME_WIDGET_SANDBOX__ '${APPSMITH_DISABLE_IFRAME_WIDGET_SANDBOX-}';
         }
 
         location /api {

deploy/docker/fs/opt/appsmith/caddy-reconfigure.mjs

@@ -41,6 +41,7 @@ const parts = []
 
 parts.push(`
 {
+  debug
   admin 127.0.0.1:2019
   persist_config off
   acme_ca_root /etc/ssl/certs/ca-certificates.crt
@@ -167,7 +168,10 @@ if (CUSTOM_DOMAIN !== "") {
   `)
 }
 
-finalizeIndexHtml()
+if (!process.argv.includes("--no-finalize-index-html")) {
+  finalizeIndexHtml()
+}
+
 fs.mkdirSync(dirname(CaddyfilePath), { recursive: true })
 fs.writeFileSync(CaddyfilePath, parts.join("\n"))
 spawnSync("/opt/caddy/caddy", ["fmt", "--overwrite", CaddyfilePath])
@@ -179,7 +183,7 @@ function finalizeIndexHtml() {
     info = JSON.parse(fs.readFileSync("/opt/appsmith/info.json", "utf8"))
   } catch(e) {
     // info will be empty, that's okay.
-    console.error("Error reading info.json", e)
+    console.error("Error reading info.json", e.message)
   }
 
   const extraEnv = {
@@ -188,8 +192,8 @@ function finalizeIndexHtml() {
     APPSMITH_VERSION_RELEASE_DATE: info?.imageBuiltAt ?? "",
   }
 
-  const content = fs.readFileSync("/opt/appsmith/editor/index.html", "utf8").replace(
+  const content = fs.readFileSync("/opt/appsmith/editor/index.html", "utf8").replaceAll(
-    /\b__(APPSMITH_[A-Z0-9_]+)__\b/g,
+    /\{\{env\s+"(APPSMITH_[A-Z0-9_]+)"}}/g,
     (_, name) => (process.env[name] || extraEnv[name] || "")
   )