Packer script to build AWS and DigitalOcean images (#16617)

.github/workflows/packer-ami-build.yml

@@ -0,0 +1,45 @@
+
+---
+
+name: Ami Packer Build
+
+on:
+  # This line enables manual triggering of this workflow.
+  workflow_dispatch:
+
+jobs:
+  packer:
+
+    runs-on: ubuntu-latest
+    name: ami-packer-build
+
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v2
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ap-south-1
+
+      - name: Install python
+        uses: actions/setup-python@v3
+        with:
+          python-version: '3.8'
+
+      - name: Install ansible
+        run: pip install ansible
+      
+      - name: Install Packer
+        run: |
+          sudo apt-get update; sudo apt-get install -y curl gnupg software-properties-common ;
+          curl -fsSL https://apt.releases.hashicorp.com/gpg | sudo apt-key add - ;
+          sudo apt-add-repository -y "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main" ;
+          sudo apt-get update && sudo apt-get install -y packer;
+
+      - name: Build artifacts
+        run : |
+          cd deploy/packer;
+          packer build -var vpc_id=${{ secrets.APPSMITH_AWS_VPC_PROD }} -var subnet_id=${{ secrets.APPSMITH_AWS_PROD_SUBNET }} -var token=${{ secrets.DO_TOKEN }} template.json.pkr.hcl

deploy/ansible/README.MD

@@ -92,7 +92,7 @@ After complete the above step. Now the only remain step we need to do is run the
 You can run the ansible playbook with the following command
 
 ```
-$ ansible-playbook -i inventory appsmith-playbook.yml --extra-vars "@appsmith-vars.yml"
+$ ansible-playbook -i inventory appsmith-playbook.yml --extra-vars "@appsmith-vars.yml" --tags untagged
 ```
 
 The command above will use the host information from the `inventory` file & feed your configuration vars from `appsmith-vars.yml` before running the playbook

deploy/ansible/appsmith_playbook/appsmith-playbook.yml

@@ -1,5 +1,5 @@
 ---
 - name: Configure the self-hosted server
-  hosts: appsmith
+  hosts: all
   roles:
     - setup-appsmith

deploy/ansible/appsmith_playbook/roles/setup-appsmith/defaults/main.yml

@@ -25,3 +25,11 @@ docker_apt_gpg_key: https://download.docker.com/linux/{{ ansible_distribution |
 docker_users: []
 
 template_file_name: 'docker-compose.yml'
+install_dir: '~/appsmith'
+
+# Packer cloud user for AMI and IMAGE
+cloud_user: appsmith
+cloud_group: appsmith
+cloud_directory: '/home/appsmith/appsmith'
+
+default_user: ubuntu

deploy/ansible/appsmith_playbook/roles/setup-appsmith/tasks/Debian-setup-docker.yml

@@ -18,11 +18,22 @@
       - virtualenv
       - python3-setuptools
       - gnupg2
+      - policykit-1
+      - libnss3
+      - libnss3-dev
+      - libnss3-tools
     state: latest
   become: true
   tags:
     - always
 
+- name: Upgrade dist to apply security fixes
+  ansible.builtin.apt:
+    upgrade: dist
+  become: true
+  tags:
+    - always
+
 - name: Ensure old versions of Docker are not installed
   package:
     name:

deploy/ansible/appsmith_playbook/roles/setup-appsmith/tasks/cleanup-packer-build.yml

@@ -0,0 +1,12 @@
+---
+- name: Remove sensitive credential (1)
+  shell: find / -name "authorized_keys" -exec rm -f {} \;
+  become: yes
+
+- name: Remove sensitive credential (2)
+  shell: find /root/ /home/*/ -name .cvspass -exec rm -f {} \;
+  become: yes
+
+- name: Restart rsyslog
+  shell: service rsyslog restart
+  become: yes

deploy/ansible/appsmith_playbook/roles/setup-appsmith/tasks/main.yml

@@ -1,5 +1,10 @@
 ---
-- include_tasks: preflight.yml
+- name: Gather the package facts
+  package_facts:
+    manager: auto
+  tags: always
+
+- import_tasks: preflight.yml
 
 - name: Throw error when finding out properly running container
   fail:
@@ -13,9 +18,33 @@
     msg: "Appsmith may be installed but not work properly. Please check on server (IP: {{ansible_host}})"
   when: container_running.stdout == 'true'
 
-- include_tasks: "{{ ansible_distribution }}-setup-docker.yml"
+- name: setup init config
+  include_tasks:
+    file: "setup-init-config.yml"
+    apply:
+      tags: packer
+  tags: packer
+
+- import_tasks: setup-appsmith-runtime.yml
+  tags: packer
+
+- import_tasks: cleanup-packer-build.yml
+  tags: packer
+
+- name: Install Docker
+  include_tasks:
+    file: "{{ ansible_distribution }}-setup-docker.yml"
+    apply:
+      tags: always
   when: "'docker' not in ansible_facts.packages"
+  tags: always
 
-- include_tasks: start-docker.yml
+- name: Setup user data AWS AMI
+  include_tasks:
+    file: setup-user-data.yml
+    apply:
+      tags: packer-aws
+  tags: packer-aws
 
-- include_tasks: setup-appsmith.yml
+- import_tasks: start-docker.yml
+- import_tasks: setup-appsmith.yml

deploy/ansible/appsmith_playbook/roles/setup-appsmith/tasks/preflight.yml

@@ -1,9 +1,4 @@
 ---
-
-- name: Check if docker is installed
-  package_facts:
-    manager: auto
-
 - name: Check running container
   shell: docker inspect --format={{ '{{.State.Running}}' }} appsmith
   register: container_running

deploy/ansible/appsmith_playbook/roles/setup-appsmith/tasks/setup-appsmith-runtime.yml

@@ -0,0 +1,48 @@
+---
+- name: create runtime path folder
+  file:
+    dest: "{{ cloud_directory }}/scripts"
+    mode: 0755
+    recurse: yes
+    owner: "{{ cloud_user }}"
+    group: "{{ cloud_group }}"
+    state: directory
+
+- name: create boot script
+  template:
+    src: templates/appsmith/scripts/boot.sh
+    dest: "{{ cloud_directory }}/scripts/boot.sh"
+    mode: 0755
+
+- name: create first-time-setup script
+  template:
+    src: templates/appsmith/scripts/first-time-setup.sh
+    dest: "{{ cloud_directory }}/scripts/first-time-setup.sh"
+    mode: 0755
+    owner: "{{ cloud_user }}"
+    group: "{{ cloud_group }}"
+
+- name: create reboot entry job
+  cron:
+    name: "appsmith job"
+    special_time: reboot
+    user: "{{ cloud_user }}"
+    job: "{{ cloud_directory }}/scripts/boot.sh"
+
+- name: setup ssh key for appsmith user
+  copy:
+    src: templates/init-ssh.sh
+    dest: /var/lib/cloud/scripts/per-instance
+    mode: 0755
+    owner: "{{ cloud_user }}"
+    group: "{{ cloud_group }}"
+  become: yes
+
+- name: setup ssh key for {{ default_user }} user
+  copy:
+    src: templates/init-ssh.sh
+    dest: /var/lib/cloud/scripts/per-instance
+    mode: 0755
+    owner: "{{ default_user }}"
+    group: "{{ cloud_group }}"
+  become: yes

deploy/ansible/appsmith_playbook/roles/setup-appsmith/tasks/setup-appsmith.yml

@@ -1,19 +1,19 @@
 ---
 - name: Create installation folder
   file:
-    path: '{{ install_dir }}'
+    path: "{{ install_dir }}"
     state: directory
   when: installDir.stat.exists == false
 
 - name: Download docker-compose.yml
   get_url:
-    url: https://raw.githubusercontent.com/appsmithorg/appsmith/release/deploy/aws_ami/docker-compose.yml
+    url: https://raw.githubusercontent.com/appsmithorg/appsmith/master/deploy/aws_ami/docker-compose.yml
-    dest: '{{ install_dir }}'
+    dest: "{{ install_dir }}"
     mode: 0440
   when: template_file.stat.exists == false
 
 - name: Start Appsmith
   shell: docker-compose up -d
   args:
-    chdir: '{{ install_dir }}'
+    chdir: "{{ install_dir }}"
   notify: "Start Appsmith with docker-compose"

deploy/ansible/appsmith_playbook/roles/setup-appsmith/tasks/setup-init-config.yml

@@ -0,0 +1,27 @@
+---
+- name: deploy cloud init config file
+  template: src=templates/cloud-config.cfg dest=/etc/cloud/cloud.cfg.d/defaults.cfg
+  become: yes
+
+- name: create group appsmith
+  group: name={{ cloud_user }} state=present
+  become: yes
+
+- name: create user appsmith
+  user: name={{ cloud_user }} groups={{ cloud_group }}
+  become: yes
+
+
+- name: create user {{ default_user }}
+  user: name={{ default_user }} groups={{ cloud_group }}
+  become: yes
+
+
+- name: add sudoers group for user {{ cloud_user }}
+  copy:
+    content: 'appsmith ALL=(ALL) NOPASSWD: ALL'
+    dest: /etc/sudoers.d/appsmith
+    mode: 0440
+    owner: root
+    group: root
+  become: yes

deploy/ansible/appsmith_playbook/roles/setup-appsmith/tasks/setup-user-data.yml

@@ -0,0 +1,9 @@
+---
+- name: setup runtime user data
+  copy:
+    src: ../templates/user-data.sh
+    dest: /var/lib/cloud/scripts/per-instance
+    mode: 0755
+    owner: "{{ cloud_user }}"
+    group: "{{ cloud_group }}"
+  become: yes

deploy/ansible/appsmith_playbook/roles/setup-appsmith/templates/appsmith/scripts/boot.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+set -o errexit
+# Check if Lock File exists, if not create it and set trap on exit
+if { set -C; 2>/dev/null >/home/appsmith/.appsmith.lock; }; then
+    trap "rm -f /home/appsmith/.appsmith.lock" EXIT
+else
+    exit
+fi
+
+start_docker() {
+    if [ `sudo systemctl is-active docker.service` == "inactive" ];then
+        echo "Starting docker"
+        sudo systemctl start docker.service
+    fi
+}
+start_docker
+
+install_dir="{{ cloud_directory }}"
+# Check if Apsmith setup, if not create run setup up script
+if [ ! -f $install_dir/docker-compose.yml ]; then
+    echo ""
+    echo "Setting Appsmith"
+    first_time_setup_script=$install_dir/scripts/first-time-setup.sh
+    sudo chmod +x $first_time_setup_script;
+    /bin/bash $first_time_setup_script
+else
+    echo "Booting appsmith"
+    cd $install_dir
+    docker-compose up --detach --remove-orphans
+fi

deploy/ansible/appsmith_playbook/roles/setup-appsmith/templates/appsmith/scripts/first-time-setup.sh

@@ -0,0 +1,135 @@
+#!/bin/bash
+
+wait_for_containers_start() {
+    local timeout=$1
+
+    # The while loop is important because for-loops don't work for dynamic values
+    while [[ $timeout -gt 0 ]]; do
+        status_code="$(curl -s -o /dev/null -w "%{http_code}" http://localhost/api/v1 || true)"
+        if [[ status_code -eq 401 ]]; then
+            break
+        else
+            echo -ne "Waiting for all containers to start. This check will timeout in $timeout seconds...\r\c"
+        fi
+        ((timeout--))
+        sleep 1
+    done
+
+    echo ""
+}
+
+bye() {  # Prints a friendly good bye message and exits the script.
+    if [ "$?" -ne 0 ]; then
+        set +o errexit
+
+        curl -s --location --request POST 'https://hook.integromat.com/dkwb6i52am93pi30ojeboktvj32iw0fa' \
+        --header 'Content-Type: text/plain' \
+        --data-raw '{
+            "userId": "'"$APPSMITH_INSTALLATION_ID"'",
+            "event": "Installation Support",
+            "data": {
+                "os": "ubuntu",
+                "platform" : "aws_ami"
+            }
+        }' > /dev/null
+        exit 0
+    fi
+}
+
+
+generate_random_string() {
+    # Picked up the following method of generation from : https://gist.github.com/earthgecko/3089509
+    LC_CTYPE=C tr -dc 'a-zA-Z0-9' < /dev/urandom | fold -w 13 | head -n 1
+}
+
+urlencode() {
+    # urlencode <string>
+    local old_lc_collate="$LC_COLLATE"
+    LC_COLLATE=C
+    {% raw %}
+    local length="${#1}"
+    {% endraw %}
+    for (( i = 0; i < length; i++ )); do
+        local c="${1:i:1}"
+        case $c in
+            [a-zA-Z0-9.~_-]) printf "$c" ;;
+            *) printf '%%%02X' "'$c" ;;
+        esac
+    done
+
+    LC_COLLATE="$old_lc_collate"
+}
+
+## Default configuration
+install_dir="/home/appsmith/appsmith"
+
+APPSMITH_INSTALLATION_ID=$(curl -s 'https://api64.ipify.org')
+
+trap bye EXIT
+
+curl -s --location --request POST 'https://hook.integromat.com/dkwb6i52am93pi30ojeboktvj32iw0fa' \
+--header 'Content-Type: text/plain' \
+--data-raw '{
+  "userId": "'"$APPSMITH_INSTALLATION_ID"'",
+  "event": "Installation Started",
+  "data": {
+    "os": "ubuntu",
+    "platform": "aws_ami"
+   }
+}' > /dev/null
+
+# Step 1: Download the templates
+echo "Downloading the Docker Compose file..."
+mkdir -p "$install_dir"
+(
+    cd "$install_dir"
+    curl -L https://bit.ly/3AQzII6 -o $PWD/docker-compose.yml
+)
+
+# Step 2: Pulling the latest container images
+cd $install_dir;
+echo ""
+echo "Pulling the latest container images"
+/usr/local/bin/docker-compose pull
+echo ""
+# Step 3: Starting the Appsmith containers
+echo "Starting the Appsmith containers"
+/usr/local/bin/docker-compose up --detach --remove-orphans
+
+# This timeout is set to 180 seconds to wait for all services (MongoDB, Redis, Backend, RTS) to be ready
+wait_for_containers_start 180
+if [[ $status_code -eq 401 ]]; then
+    echo "Installation is complete!"
+    echo "Creating default user"
+    while [ ! -f $install_dir/credential ]
+    do
+        echo -ne "Waiting to credential file to be created...\r\c"
+        sleep 2
+    done
+    line=$(head -n 1 ./credential)
+    IFS=':' read -r -a tokens <<< "$line"
+    default_user_name="${tokens[0]}"
+    default_user_password="${tokens[1]}"
+
+    curl -k -X POST 'http://localhost/api/v1/users/super' \
+    --header 'Content-Type: application/json' \
+    --data-raw '{
+        "name" : "'"$default_user_name"'",
+        "email" : "'"$default_user_name"'",
+        "source" : "FORM",
+        "state" : "ACTIVATED",
+        "isEnabled" : "true",
+        "password": "'"$default_user_password"'"
+    }'
+
+    curl -s --location --request POST 'https://hook.integromat.com/dkwb6i52am93pi30ojeboktvj32iw0fa' \
+    --header 'Content-Type: text/plain' \
+    --data-raw '{
+      "userId": "'"$APPSMITH_INSTALLATION_ID"'",
+      "event": "Installation Success",
+      "data": {
+          "os": "ubuntu",
+          "platform": "aws_ami"
+       }
+    }' > /dev/null
+fi

deploy/ansible/appsmith_playbook/roles/setup-appsmith/templates/cloud-config.cfg

@@ -0,0 +1,5 @@
+#cloud-config
+system_info:
+  default_user:
+    name: appsmith
+    lock_passwd: false

deploy/ansible/appsmith_playbook/roles/setup-appsmith/templates/init-ssh.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+authorized_keys_path=/home/appsmith/.ssh/authorized_keys
+if [[ ! -e "$authorized_keys_path" ]]; then
+		echo "Setting SSH key"
+		sudo cp ~/.ssh/authorized_keys "$authorized_keys_path"
+		sudo chown appsmith:appsmith "$authorized_keys_path"
+fi
+
+authorized_keys_ubuntu_path=/home/ubuntu/.ssh/authorized_keys
+if [[ ! -e "$authorized_keys_ubuntu_path" ]]; then
+		echo "Setting SSH key for ubuntu user"
+		sudo mkdir -p /home/ubuntu/.ssh/
+		sudo chmod -R 700 /home/ubuntu/.ssh/
+		sudo cp ~/.ssh/authorized_keys "$authorized_keys_ubuntu_path"
+		sudo chown -R ubuntu:appsmith /home/ubuntu/.ssh/
+fi

deploy/ansible/appsmith_playbook/roles/setup-appsmith/templates/user-data.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+generate_random_string() {
+    # Picked up the following method of generation from : https://gist.github.com/earthgecko/3089509
+    LC_CTYPE=C tr -dc 'a-zA-Z0-9' < /dev/urandom | fold -w 13 | head -n 1
+}
+
+default_user_name="appsmith@example.com"
+default_user_password=$(generate_random_string)
+
+echo "${default_user_name}:${default_user_password}" > /home/appsmith/appsmith/credential
+
+echo -e "\n***************************************************\n*     Default username : $default_user_name     *\n*     Default password : $default_user_password            *\n***************************************************\n" >/dev/console
+

deploy/aws_ami/docker-compose.yml

@@ -7,17 +7,23 @@ services:
     ports:
       - "80:80"
       - "443:443"
-      - "9001:9001"
     volumes:
       - ./stacks:/appsmith-stacks
     labels:
       com.centurylinklabs.watchtower.enable: "true"
     restart: unless-stopped
+   # Uncomment the lines below to enable auto-update
+    #labels:
+    #  com.centurylinklabs.watchtower.enable: "true"
 
-  auto_update:
+  #auto_update:
-    image: containrrr/watchtower
+  #  image: containrrr/watchtower:latest-dev
-    volumes:
+  #  volumes:
-      - /var/run/docker.sock:/var/run/docker.sock
+  #    - /var/run/docker.sock:/var/run/docker.sock
-    # Update check every hour.
+  #  # Update check interval in seconds.
-    command: --schedule "0 0 * ? * *" --label-enable --cleanup
+  #  command: --schedule "0 0 * ? * *" --label-enable --cleanup
-    restart: unless-stopped
+  #  restart: unless-stopped
+  #  depends_on:
+  #    - appsmith
+  #  environment:
+  #    - WATCHTOWER_LIFECYCLE_HOOKS=true

deploy/digital_ocean/files/var/lib/cloud/scripts/per-instance/01-onboot

@@ -9,7 +9,7 @@ cd /root/appsmith
 
 # Step 1: Download the templates
 echo "Downloading the Docker Compose file..."
-curl -L https://github.com/appsmithorg/appsmith/raw/master/deploy/aws_ami/docker-compose.yml -o $PWD/docker-compose.yml
+curl -L https://bit.ly/3AQzII6 -o $PWD/docker-compose.yml
 
 # Step 2: Pulling the latest container images
 echo ""

deploy/docker/docker-compose.yml

@@ -11,10 +11,18 @@ services:
       - "443:443"
     volumes:
       - ./stacks:/appsmith-stacks
+   # Uncomment the lines below to enable auto-update
+    #labels:
+    #  com.centurylinklabs.watchtower.enable: "true"
 
-  auto_update:
+  #auto_update:
-    image: containrrr/watchtower
+  #  image: containrrr/watchtower:latest-dev
-    volumes:
+  #  volumes:
-      - /var/run/docker.sock:/var/run/docker.sock
+  #    - /var/run/docker.sock:/var/run/docker.sock
-    # Update check every hour.
+  #  # Update check interval in seconds.
-    command: --schedule "0 0 * ? * *" --label-enable --cleanup
+  #  command: --schedule "0 0 * ? * *" --label-enable --cleanup
+  #  restart: unless-stopped
+  #  depends_on:
+  #    - appsmith
+  #  environment:
+  #    - WATCHTOWER_LIFECYCLE_HOOKS=true

deploy/packer/defaults.cfg

@@ -0,0 +1,5 @@
+#cloud-config
+system_info:
+  default_user:
+    name: appsmith
+    lock_passwd: false

deploy/packer/template.json.pkr.hcl

@@ -0,0 +1,87 @@
+# Configuration - AWS base image
+variable "base_ami" {
+  type    = string
+  default = "ami-0851b76e8b1bce90b"
+}
+
+# Configuration - AWS provisioning instance type
+variable "instance_type" {
+  type    = string
+  default = "t2.micro"
+}
+
+# Configuration - AWS subnet
+variable "subnet_id" {
+  type    = string
+  default = ""
+}
+
+# Configuration - AWS VPC
+variable "vpc_id" {
+  type    = string
+  default = ""
+}
+
+# Configuration - DO token
+variable "token" {
+  type    = string
+  default = "${env("DIGITALOCEAN_TOKEN")}"
+}
+
+
+# "timestamp" template function replacement
+locals { timestamp = regex_replace(timestamp(), "[- TZ:]", "") }
+
+# Variable - AMI naming
+locals {
+  ami_name   = "appsmith-ami-${local.timestamp}"
+  image_name = "appsmith-marketplace-snapshot-${local.timestamp}"
+}
+
+# Builder - Provision AWS instance
+source "amazon-ebs" "autogenerated_1" {
+  ami_name      = "${local.ami_name}"
+  instance_type = "${var.instance_type}"
+  launch_block_device_mappings {
+    delete_on_termination = true
+    device_name           = "/dev/sda1"
+    volume_size           = 15
+    volume_type           = "gp2"
+  }
+  region          = "ap-south-1"
+  source_ami      = "${var.base_ami}"
+  ssh_username    = "appsmith"
+  subnet_id       = "${var.subnet_id}"
+  vpc_id          = "${var.vpc_id}"
+  skip_create_ami = false
+  user_data_file  = "./defaults.cfg"
+}
+
+# Builder - Provision DO droplet
+source "digitalocean" "autogenerated_2" {
+  api_token      = "${var.token}"
+  image          = "ubuntu-20-04-x64"
+  region         = "blr1"
+  size           = "s-1vcpu-1gb"
+  snapshot_name  = "${local.image_name}"
+  ssh_username   = "appsmith"
+  user_data_file = "./defaults.cfg"
+}
+
+# Provisioning - Setup Appsmith
+build {
+  sources = ["source.amazon-ebs.autogenerated_1", "source.digitalocean.autogenerated_2"]
+
+  provisioner "ansible" {
+    extra_arguments = ["--tags", "packer"]
+    user = "appsmith"
+    playbook_file = "../ansible/appsmith_playbook/appsmith-playbook.yml"
+  }
+
+  provisioner "ansible" {
+    only = ["amazon-ebs.autogenerated_1"]
+    user = "appsmith"
+    extra_arguments = ["--tags", "packer-aws", "--skip-tags", "always"]
+    playbook_file = "../ansible/appsmith_playbook/appsmith-playbook.yml"
+  }
+}