chore: Added scout & trivy scan to github workflow (#37022)

## Description
Run trivy and scout scanner with image name


Fixes #`37036`  

## Automation

/ok-to-test tags="@tag.IDE"

### 🔍 Cypress test results
<!-- This is an auto-generated comment: Cypress test results  -->
> [!TIP]
> 🟢 🟢 🟢 All cypress tests have passed! 🎉 🎉 🎉
> Workflow run:
<https://github.com/appsmithorg/appsmith/actions/runs/11480586298>
> Commit: 5ebbcd37ec177c781d8b0be38a83ce695d211c9d
> <a
href="https://internal.appsmith.com/app/cypress-dashboard/rundetails-65890b3c81d7400d08fa9ee5?branch=master&workflowId=11480586298&attempt=1"
target="_blank">Cypress dashboard</a>.
> Tags: `@tag.IDE`
> Spec:
> <hr>Wed, 23 Oct 2024 13:36:44 UTC
<!-- end of auto-generated comment: Cypress test results  -->


## Communication
Should the DevRel and Marketing teams inform users about this change?
- [ ] Yes
- [x] No


<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

- **New Features**
- Introduced two new scripts for automated vulnerability scanning of
Docker images: `scout_vulnerabilities_data.sh` and
`trivy_vulnerabilities_data.sh`.
- Added a GitHub Actions workflow to automate vulnerability scanning and
update pull requests with results.

- **Bug Fixes**
- Improved error handling for missing environment variables in the new
scripts.

- **Documentation**
	- Added details on the new workflow and its steps for user reference.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

.github/workflows/test-vulnerabilities-data.yml

@@ -0,0 +1,143 @@
+name: Run Vulnerability Data Script with Parameters and Update PR
+
+on:
+  workflow_dispatch:
+    inputs:
+      image_name:
+        description: 'Docker image name to scan'
+        required: true
+        default: 'appsmith/appsmith-ce:release'
+
+jobs:
+  run-and-update-pr:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: '20'
+      
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}    
+
+      - name: Install pg
+        run: npm install pg
+
+      - name: Fetch vulnerability data
+        id: vulnerability_data
+        env:
+          DB_HOST: ${{ secrets.CYPRESS_DB_HOST }}
+          DB_NAME: ${{ secrets.CYPRESS_DB_NAME }}
+          DB_USER: ${{ secrets.CYPRESS_DB_USER }}
+          DB_PWD: ${{ secrets.CYPRESS_DB_PWD }}
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const { Pool } = require("pg");
+            const fs = require('fs');
+            const path = require('path');
+            const { DB_HOST, DB_NAME, DB_USER, DB_PWD } = process.env;
+
+            const pool = new Pool({
+              user: DB_USER,
+              host: DB_HOST,
+              database: DB_NAME,
+              password: DB_PWD,
+              port: 5432,
+              connectionTimeoutMillis: 60000,
+            });
+
+            (async () => {
+              const client = await pool.connect();
+              try {
+                // Fetch vurn_id, product, scanner_tool, and priority from the database
+                const result = await client.query(`SELECT vurn_id, product, scanner_tool, priority FROM vulnerability_tracking`);
+                console.log('Vulnerability Data:', result.rows);
+
+                // Extract relevant fields from the result
+                const extractedData = result.rows.map(({ vurn_id, product, scanner_tool, priority }) => ({
+                  vurn_id,
+                  product,
+                  scanner_tool,
+                  priority
+                }));
+                console.log('Extracted Vulnerability Data:', extractedData);
+
+                // Prepare CSV content
+                const csvContent = [
+                  ['vurn_id', 'product', 'scanner_tool', 'priority'],  // Add priority column header
+                  ...extractedData.map(row => [row.vurn_id, row.product, row.scanner_tool, row.priority])
+                ]
+                .map(e => e.join(',')) // Join columns
+                .join('\n'); // Join rows
+
+                // Write to CSV file in workspace
+                const csvFilePath = path.join(process.env.GITHUB_WORKSPACE, 'vulnerability_base_data.csv');
+                fs.writeFileSync(csvFilePath, csvContent);
+                console.log(`Data successfully written to ${csvFilePath}`);
+
+                // Prepare TXT content
+                const txtContent = extractedData
+                  .map(row => `vurn_id: ${row.vurn_id}, product: ${row.product}, scanner_tool: ${row.scanner_tool}, priority: ${row.priority}`)
+                  .join('\n'); // Join rows
+
+                // Write to TXT file in workspace
+                const txtFilePath = path.join(process.env.GITHUB_WORKSPACE, 'vulnerability_base_data.txt');
+                fs.writeFileSync(txtFilePath, txtContent);
+                console.log(`Data successfully written to ${txtFilePath}`);
+
+                client.release();
+                return extractedData; // Return the extracted data
+              } catch (err) {
+                console.error('Error fetching vulnerability data:', err);
+                client.release();
+              }
+            })();
+              
+      - name: Upload Vulnerability Data
+        uses: actions/upload-artifact@v3
+        with:
+          name: vulnerability-data
+          path: |
+            vulnerability_base_data.csv
+            vulnerability_base_data.txt      
+
+      # Run Scout vulnerability data script
+      - name: Run Scout vulnerability data script
+        if: always()
+        env:
+          DB_HOST: ${{ secrets.CYPRESS_DB_HOST }}
+          DB_NAME: ${{ secrets.CYPRESS_DB_NAME }}
+          DB_USER: ${{ secrets.CYPRESS_DB_USER }}
+          DB_PWD: ${{ secrets.CYPRESS_DB_PWD }}
+        run: |
+          chmod +x scripts/scout_vulnerabilities_data.sh
+          ./scripts/scout_vulnerabilities_data.sh \
+            "${{ inputs.image_name }}" \
+            "${{ github.event.pull_request.number }}" \
+            "${{ github.event.pull_request.html_url }}" \
+            "${{ github.run_id }}"
+
+      - name: Run Trivy vulnerability data script
+        if: always()
+        env:
+          DB_HOST: ${{ secrets.CYPRESS_DB_HOST }}
+          DB_NAME: ${{ secrets.CYPRESS_DB_NAME }}
+          DB_USER: ${{ secrets.CYPRESS_DB_USER }}
+          DB_PWD: ${{ secrets.CYPRESS_DB_PWD }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin
+          chmod +x scripts/trivy_vulnerabilities_data.sh
+          ./scripts/trivy_vulnerabilities_data.sh \
+            "${{ inputs.image_name }}" \
+            "${{ github.event.pull_request.number }}" \
+            "${{ github.event.pull_request.html_url }}" \
+            "${{ github.run_id }}"
+            

scripts/scout_vulnerabilities_data.sh

@@ -0,0 +1,173 @@
+#!/bin/bash
+
+#Check required environment variables
+required_vars=("DB_HOST" "DB_NAME" "DB_USER" "DB_PWD")
+for var in "${required_vars[@]}"; do
+  if [ -z "${!var}" ] || [[ "${!var}" == "your_${var,,}" ]]; then
+    echo "Error: Required environment variable $var is missing or not set correctly."
+    exit 1
+  fi
+done
+
+DB_HOST="${DB_HOST}"
+DB_NAME="${DB_NAME}"
+DB_USER="${DB_USER}"
+DB_PWD="${DB_PWD}"
+
+# Assign the parameters from the workflow
+IMAGE="$1"
+GITHUB_PR_ID="$2"
+GITHUB_PR_LINK="$3"
+GITHUB_RUN_ID="$4"
+OLD_VULN_FILE="${5:-vulnerability_base_data.csv}"
+
+# Function to install Docker Scout
+install_docker_scout() {
+    echo "Installing Docker Scout..."
+    local attempts=0
+    while [ $attempts -lt 3 ]; do
+        echo "Attempt $((attempts + 1))..."
+        curl -fsSL https://raw.githubusercontent.com/docker/scout-cli/main/install.sh -o install-scout.sh
+        sh install-scout.sh &> install_scout_log.txt
+        if [ $? -eq 0 ]; then
+            echo "Docker Scout installed successfully."
+            return 0
+        fi
+        echo "Attempt $((attempts + 1)) failed. Check install_scout_log.txt for details."
+        ((attempts++))
+        sleep 2
+    done
+    echo "Error: Docker Scout installation failed after $attempts attempts."
+    exit 1
+}
+
+# Check if Docker is installed
+if ! command -v docker &> /dev/null; then
+    echo "Error: Docker is not installed. Please install Docker and try again."
+    exit 1
+fi
+
+# Ensure Docker is running
+if ! systemctl is-active --quiet docker; then
+    echo "Starting Docker..."
+    sudo systemctl start docker
+fi
+
+# Check if Docker Scout is installed
+if ! command -v scout &> /dev/null; then
+    install_docker_scout
+fi
+
+# Prepare the output CSV file
+CSV_OUTPUT_FILE="scout_vulnerabilities.csv"
+rm -f "$CSV_OUTPUT_FILE"
+
+# Extract the product name from the image name
+case "$IMAGE" in
+    *appsmith/appsmith-ce:*) product_name="CE" ;;
+    *appsmith/appsmith-ee:*) product_name="EE" ;;
+    *appsmith/cloud-services:*) product_name="CLOUD" ;;
+    *) product_name="UNKNOWN" ;;
+esac
+
+# Fetch vulnerabilities and format the output correctly
+docker scout cves "$IMAGE" | grep -E "✗ |CVE-" | awk -v product_name="$product_name" -F' ' '
+{
+    # Check for valid vulnerability data and format it correctly
+    if ($2 != "" && $3 ~ /^CVE-/) {
+        # Extract severity level, CVE ID and format output correctly
+        print $3","product_name",""SCOUT"","$2
+    }
+}' | sort -u > "$CSV_OUTPUT_FILE"
+
+# Check if the CSV output file is empty
+[ -s "$CSV_OUTPUT_FILE" ] || echo "No vulnerabilities found for image: $IMAGE" > "$CSV_OUTPUT_FILE"
+
+# Compare new vulnerabilities against old vulnerabilities
+echo "Comparing new vulnerabilities with existing vulnerabilities in $OLD_VULN_FILE..."
+if [ -s "$OLD_VULN_FILE" ]; then
+    awk -F, 'NR==FNR {seen[$1","$2","$3","$4]; next} !($1","$2","$3","$4 in seen)' "$OLD_VULN_FILE" "$CSV_OUTPUT_FILE" > "scout_vulnerabilities_diff.csv"
+else
+    echo "$OLD_VULN_FILE is empty. All new vulnerabilities will be included."
+    cp "$CSV_OUTPUT_FILE" "scout_vulnerabilities_diff.csv"
+fi
+
+# Output for verification
+echo "Fetching passed data..."
+cat "$OLD_VULN_FILE"
+echo ""
+echo "Fetching new data..."
+cat "$CSV_OUTPUT_FILE"
+echo ""
+echo "Fetching diff..."
+cat "scout_vulnerabilities_diff.csv"
+echo ""
+
+# Insert new vulnerabilities into the PostgreSQL database using psql
+insert_vulns_into_db() {
+  local count=0
+  local query_file="insert_vulns.sql"
+  echo "BEGIN;" > "$query_file"  # Start the transaction
+
+  # Create an associative array to hold existing entries from the database
+  declare -A existing_entries
+
+  # Fetch existing vulnerabilities from the database to avoid duplicates
+  psql -t -c "SELECT vurn_id, product, scanner_tool, priority FROM vulnerability_tracking WHERE scanner_tool = 'SCOUT'" "postgresql://$DB_USER:$DB_PWD@$DB_HOST/$DB_NAME" | while IFS='|' read -r db_vurn_id db_product db_scanner_tool db_priority; do
+    existing_entries["$db_product,$db_scanner_tool,$db_vurn_id"]="$db_priority"
+  done
+
+  while IFS=, read -r vurn_id product scanner_tool priority; do
+    # Skip empty lines
+    if [[ -z "$vurn_id" || -z "$priority" || -z "$product" || -z "$scanner_tool" ]]; then
+      echo "Skipping empty vulnerability entry"
+      continue
+    fi
+
+    # Check if the entry already exists
+    if [[ -n "${existing_entries["$product,$scanner_tool,$vurn_id"]}" ]]; then
+      echo "Entry for $vurn_id already exists in the database. Skipping."
+      continue
+    fi
+
+    local pr_id="$GITHUB_PR_ID"
+    local pr_link="$GITHUB_PR_LINK"
+    local created_date=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+    local update_date="$created_date"
+    local comments="Initial vulnerability report"
+    local owner="John Doe"
+    local pod="Security"
+
+    # Escape single quotes in vulnerability ID, product, and priority
+    vurn_id=$(echo "$vurn_id" | sed "s/'/''/g")
+    priority=$(echo "$priority" | sed "s/'/''/g")
+    product=$(echo "$product" | sed "s/'/''/g")
+    scanner_tool=$(echo "$scanner_tool" | sed "s/'/''/g")
+
+    # Write each insert query to the SQL file
+    echo "INSERT INTO vulnerability_tracking (product, scanner_tool, vurn_id, priority, pr_id, pr_link, github_run_id, created_date, update_date, comments, owner, pod) VALUES ('$product', '$scanner_tool', '$vurn_id', '$priority', '$pr_id', '$pr_link', '$GITHUB_RUN_ID', '$created_date', '$update_date', '$comments', '$owner', '$pod');" >> "$query_file"
+    
+    ((count++))
+  done < "scout_vulnerabilities_diff.csv"
+
+  echo "COMMIT;" >> "$query_file"  # End the transaction
+  echo "Queries written to $query_file."
+
+  # Execute the SQL file
+  psql -e "postgresql://$DB_USER:$DB_PWD@$DB_HOST/$DB_NAME" -f "$query_file"
+
+  # Check if the execution was successful
+  if [ $? -eq 0 ]; then
+    echo "Vulnerabilities successfully inserted into the database."
+  else
+    echo "Error: Failed to insert vulnerabilities. Please check the database connection or query."
+    exit 1
+  fi
+}
+
+# Call the function to generate the insert queries and execute them
+if [ -s "scout_vulnerabilities_diff.csv" ]; then
+  insert_vulns_into_db
+else
+  echo "No new vulnerabilities to insert."
+fi

scripts/trivy_vulnerabilities_data.sh

@@ -0,0 +1,218 @@
+#!/bin/bash
+
+#Check required environment variables
+required_vars=("DB_HOST" "DB_NAME" "DB_USER" "DB_PWD")
+for var in "${required_vars[@]}"; do
+  if [ -z "${!var}" ] || [[ "${!var}" == "your_${var,,}" ]]; then
+    echo "Error: Required environment variable $var is missing or not set correctly."
+    exit 1
+  fi
+done
+
+DB_HOST="${DB_HOST}"
+DB_NAME="${DB_NAME}"
+DB_USER="${DB_USER}"
+DB_PWD="${DB_PWD}"
+
+# Assign the parameters from the workflow
+IMAGE="$1"
+GITHUB_PR_ID="$2"
+GITHUB_PR_LINK="$3"
+GITHUB_RUN_ID="$4"
+OLD_VULN_FILE="${5:-vulnerability_base_data.csv}"
+
+
+# Define the maximum number of retries
+MAX_RETRIES=3
+
+# Function to install Trivy with retry logic
+install_trivy_with_retry() {
+    local count=0
+    local success=false
+
+    while [[ $count -lt $MAX_RETRIES ]]; do
+        echo "Attempting to install Trivy (attempt $((count + 1)))..."
+        
+        # Fetch the latest release dynamically instead of hardcoding
+        TRIVY_VERSION=$(curl -s https://api.github.com/repos/aquasecurity/trivy/releases/latest | grep '"tag_name"' | sed -E 's/.*"v([^"]+)".*/\1/')
+        TRIVY_URL="https://github.com/aquasecurity/trivy/releases/download/v$TRIVY_VERSION/trivy_"$TRIVY_VERSION"_Linux-64bit.tar.gz"
+        echo "Attempting to install $TRIVY_VERSION from $TRIVY_URL"
+        # Download and extract Trivy
+        curl -sfL "$TRIVY_URL" | tar -xzf - trivy
+        
+        # Check if extraction was successful
+        if [[ $? -eq 0 ]]; then
+            # Create a local bin directory if it doesn't exist
+            mkdir -p "$HOME/bin"
+            # Move Trivy to the local bin directory
+            mv trivy "$HOME/bin/"
+            # Manually add the bin directory to PATH for this session
+            export PATH="$HOME/bin:$PATH"
+
+            # Check if Trivy is successfully installed
+            if command -v trivy &> /dev/null; then
+                success=true
+                break
+            fi
+        fi
+        
+        echo "Trivy installation failed. Retrying..."
+        count=$((count + 1))
+    done
+
+    if [[ $success = false ]]; then
+        echo "Error: Trivy installation failed after $MAX_RETRIES attempts."
+        exit 1
+    fi
+
+    echo "Trivy installed successfully."
+}
+
+# Check if Trivy is installed, if not, install it with retry logic
+if ! command -v trivy &> /dev/null; then
+    install_trivy_with_retry
+fi
+
+NEW_VULN_FILE="trivy_vulnerabilities_new.csv"
+DIFF_OUTPUT_FILE="trivy_vulnerabilities_diff.csv"
+
+rm -f "$NEW_VULN_FILE" "$DIFF_OUTPUT_FILE"
+touch "$OLD_VULN_FILE"
+
+# Extract the product name from the image name
+case "$IMAGE" in
+    *appsmith/appsmith-ce:*) product_name="CE" ;;
+    *appsmith/appsmith-ee:*) product_name="EE" ;;
+    *appsmith/cloud-services:*) product_name="CLOUD" ;;
+    *) product_name="UNKNOWN" ;;
+esac
+
+# Function to run Trivy scan
+run_trivy_scan() {
+    echo "Cleaning up Trivy data..."
+    trivy clean --all
+
+    echo "Running Trivy scan for image: $IMAGE..."
+    if ! trivy image \
+        --db-repository public.ecr.aws/aquasecurity/trivy-db \
+        --java-db-repository public.ecr.aws/aquasecurity/trivy-java-db \
+        --insecure \
+        --format json \
+        "$IMAGE" > "trivy_vulnerabilities.json"; then
+        echo "Error: Trivy is not available or the image does not exist."
+        exit 1
+    fi
+}
+
+# Call the function to run the scan
+run_trivy_scan
+
+# Process vulnerabilities and generate the desired CSV format
+if jq -e '.Results | length > 0' "trivy_vulnerabilities.json" > /dev/null; then
+    jq -r --arg product "$product_name" '.Results[].Vulnerabilities[] | "\(.VulnerabilityID),\($product),TRIVY,\(.Severity)"' "trivy_vulnerabilities.json" | sed 's/^\s*//;s/\s*$//' | sort -u > "$NEW_VULN_FILE"
+    echo "Vulnerabilities saved to $NEW_VULN_FILE"
+else
+    echo "No vulnerabilities found for image: $IMAGE"
+    echo "No vulnerabilities found." > "$NEW_VULN_FILE"
+fi
+
+# Compare new vulnerabilities with the old file
+if [ -s "$NEW_VULN_FILE" ]; then
+    sort "$OLD_VULN_FILE" -o "$OLD_VULN_FILE"  # Sort the old vulnerabilities file
+    sort "$NEW_VULN_FILE" -o "$NEW_VULN_FILE"  # Sort the new vulnerabilities file
+    
+    # Get the difference between new and old vulnerabilities
+    comm -13 "$OLD_VULN_FILE" "$NEW_VULN_FILE" > "$DIFF_OUTPUT_FILE"
+
+    if [ -s "$DIFF_OUTPUT_FILE" ]; then
+        echo "New vulnerabilities found and recorded in $DIFF_OUTPUT_FILE."
+    else
+        echo "No new vulnerabilities found for image: $IMAGE."
+    fi
+else
+    echo "No new vulnerabilities found for image: $IMAGE."
+fi
+
+
+# Cleanup JSON file
+rm -f "trivy_vulnerabilities.json"
+
+# Output for verification
+echo "Fetching passed data..."
+cat "$OLD_VULN_FILE"
+echo ""
+echo "Fetching new data..."
+cat "$NEW_VULN_FILE"
+echo ""
+echo "Fetching diff..."
+cat $DIFF_OUTPUT_FILE
+echo ""
+
+# Insert new vulnerabilities into the PostgreSQL database using psql
+insert_vulns_into_db() {
+  local count=0
+  local query_file="insert_vulns.sql"
+  echo "BEGIN;" > "$query_file"  # Start the transaction
+
+  # Create an associative array to hold existing entries from the database
+  declare -A existing_entries
+
+  # Fetch existing vulnerabilities from the database to avoid duplicates
+  psql -t -c "SELECT vurn_id, product, scanner_tool, priority FROM vulnerability_tracking WHERE scanner_tool = 'TRIVY'" "postgresql://$DB_USER:$DB_PWD@$DB_HOST/$DB_NAME" | while IFS='|' read -r db_vurn_id db_product db_scanner_tool db_priority; do
+    existing_entries["$db_product,$db_scanner_tool,$db_vurn_id"]="$db_priority"
+  done
+
+  while IFS=, read -r vurn_id product scanner_tool priority; do
+    # Skip empty lines
+    if [[ -z "$vurn_id" || -z "$priority" || -z "$product" || -z "$scanner_tool" ]]; then
+      echo "Skipping empty vulnerability entry"
+      continue
+    fi
+
+    # Check if the entry already exists
+    if [[ -n "${existing_entries["$product,$scanner_tool,$vurn_id"]}" ]]; then
+      echo "Entry for $vurn_id already exists in the database. Skipping."
+      continue
+    fi
+
+    local pr_id="$GITHUB_PR_ID"
+    local pr_link="$GITHUB_PR_LINK"
+    local created_date=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+    local update_date="$created_date"
+    local comments="Initial vulnerability report"
+    local owner="John Doe"
+    local pod="Security"
+
+    # Escape single quotes in vulnerability ID, product, and priority
+    vurn_id=$(echo "$vurn_id" | sed "s/'/''/g")
+    priority=$(echo "$priority" | sed "s/'/''/g")
+    product=$(echo "$product" | sed "s/'/''/g")
+    scanner_tool=$(echo "$scanner_tool" | sed "s/'/''/g")
+
+    # Write each insert query to the SQL file
+    echo "INSERT INTO vulnerability_tracking (product, scanner_tool, vurn_id, priority, pr_id, pr_link, github_run_id, created_date, update_date, comments, owner, pod) VALUES ('$product', '$scanner_tool', '$vurn_id', '$priority', '$pr_id', '$pr_link', '$GITHUB_RUN_ID', '$created_date', '$update_date', '$comments', '$owner', '$pod');" >> "$query_file"
+    
+    ((count++))
+  done < $DIFF_OUTPUT_FILE
+
+  echo "COMMIT;" >> "$query_file"  # End the transaction
+  echo "Queries written to $query_file."
+
+  # Execute the SQL file
+  psql -e "postgresql://$DB_USER:$DB_PWD@$DB_HOST/$DB_NAME" -f "$query_file"
+
+  # Check if the execution was successful
+  if [ $? -eq 0 ]; then
+    echo "Vulnerabilities successfully inserted into the database."
+  else
+    echo "Error: Failed to insert vulnerabilities. Please check the database connection or query."
+    exit 1
+  fi
+}
+
+# Call the function to generate the insert queries and execute them
+if [ -s $DIFF_OUTPUT_FILE ]; then
+  insert_vulns_into_db
+else
+  echo "No new vulnerabilities to insert."
+fi