Protei/PromucFlow_constructor
2
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

chore: new update for failing job on new vulnerabilities (#37273)

Browse Source 
## Description
Failing job if new vurn found


Fixes #`37036`  

## Automation

/ok-to-test tags=""

### 🔍 Cypress test results
<!-- This is an auto-generated comment: Cypress test results  -->
> [!CAUTION]  
> If you modify the content in this section, you are likely to disrupt
the CI result for your PR.

<!-- end of auto-generated comment: Cypress test results  -->


## Communication
Should the DevRel and Marketing teams inform users about this change?
- [ ] Yes
- [x] No
This commit is contained in:
Sagar Khalasi 2024-11-07 13:29:35 +05:30 committed by GitHub
parent 83e75583e3
commit d00c15d2e7
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 69 additions and 153 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
.github/workflows/test-vulnerabilities-data.yml vendored
View File

@ -62,3 +62,19 @@ jobs:
"${{ github.event.pull_request.html_url }}" \ "${{ github.event.pull_request.html_url }}" \
"${{ github.run_id }}" "${{ github.run_id }}"
- name: Check for new vulnerabilities in Scout and Trivy files
if: always()
run: |
# Check if Scout vulnerabilities file is not empty
if [ -s "scout_new_vulnerabilities.csv" ]; then
echo "Scout vulnerabilities detected."
cat scout_new_vulnerabilities.csv
exit 1 # Fail the job if data exists
fi
# Check if Trivy vulnerabilities file is not empty
if [ -s "trivy_new_vulnerabilities.csv" ]; then
echo "Trivy vulnerabilities detected."
cat trivy_new_vulnerabilities.csv
exit 1 # Fail the job if data exists
fi

137
scripts/scout_vulnerabilities_data.sh
View File

@ -21,72 +21,11 @@ GITHUB_PR_LINK="$3"
GITHUB_RUN_ID="$4" GITHUB_RUN_ID="$4"
OLD_VULN_FILE="${5:-vulnerability_base_data.csv}" OLD_VULN_FILE="${5:-vulnerability_base_data.csv}"
# Function to install Docker Scout # Compare each vulnerability with the database and store new ones in a CSV file
install_docker_scout() { compare_and_store_vulns() {
echo "Installing Docker Scout..." local new_vulns_file="scout_new_vulnerabilities.csv"
local attempts=0
while [ $attempts -lt 5 ]; do
echo "Attempt $((attempts + 1))..."
curl -fsSL https://raw.githubusercontent.com/docker/scout-cli/main/install.sh -o install-scout.sh
sh install-scout.sh &> install_scout_log.txt
if [ $? -eq 0 ]; then
echo "Docker Scout installed successfully."
return 0
fi
echo "Attempt $((attempts + 1)) failed. Check install_scout_log.txt for details."
((attempts++))
sleep 2
done
echo "Error: Docker Scout installation failed after $attempts attempts."
exit 1
}
# Check if Docker is installed echo "vurn_id,product,scanner_tool,priority" > "$new_vulns_file" # CSV header
if ! command -v docker &> /dev/null; then
echo "Error: Docker is not installed. Please install Docker and try again."
exit 1
fi
# Ensure Docker is running
if ! systemctl is-active --quiet docker; then
echo "Starting Docker..."
sudo systemctl start docker
fi
# Check if Docker Scout is installed
if ! command -v scout &> /dev/null; then
install_docker_scout
fi
# Prepare the output CSV file
CSV_OUTPUT_FILE="scout_vulnerabilities.csv"
rm -f "$CSV_OUTPUT_FILE"
# Extract the product name from the image name
case "$IMAGE" in
*appsmith/appsmith-ce:*) product_name="CE" ;;
*appsmith/appsmith-ee:*) product_name="EE" ;;
*appsmith/cloud-services:*) product_name="CLOUD" ;;
*) product_name="UNKNOWN" ;;
esac
# Fetch vulnerabilities and format the output correctly
docker scout cves "$IMAGE" | grep -E "✗ |CVE-" | awk -v product_name="$product_name" -F' ' '
{
# Check for valid vulnerability data and format it correctly
if ($2 != "" && $3 ~ /^CVE-/) {
# Extract severity level, CVE ID, and format output correctly
print $3","product_name",""SCOUT"","$2
}
}' | sort -u > "$CSV_OUTPUT_FILE"
# Check if the CSV output file is empty
[ -s "$CSV_OUTPUT_FILE" ] || echo "No vulnerabilities found for image: $IMAGE" > "$CSV_OUTPUT_FILE"
# Insert new vulnerabilities into the PostgreSQL database using psql
insert_vulns_into_db() {
local query_file="insert_vulns.sql"
echo "BEGIN;" > "$query_file"
while IFS=, read -r vurn_id product scanner_tool priority; do while IFS=, read -r vurn_id product scanner_tool priority; do
if [[ -z "$vurn_id" || -z "$priority" || -z "$product" || -z "$scanner_tool" ]]; then if [[ -z "$vurn_id" || -z "$priority" || -z "$product" || -z "$scanner_tool" ]]; then
@ -94,65 +33,39 @@ insert_vulns_into_db() {
continue continue
fi fi
local pr_id="${GITHUB_PR_ID:-}" # Clean up and trim spaces from input values
local pr_link="${GITHUB_PR_LINK:-}" vurn_id=$(echo "$vurn_id" | sed "s/'/''/g" | sed 's/^[ \t]*//;s/[ \t]*$//')
local created_date=$(date -u +"%Y-%m-%dT%H:%M:%SZ") priority=$(echo "$priority" | sed "s/'/''/g" | sed 's/^[ \t]*//;s/[ \t]*$//')
local comments="Initial vulnerability report" product=$(echo "$product" | sed "s/'/''/g" | sed 's/^[ \t]*//;s/[ \t]*$//' | tr -d '[:space:]')
local owner="John Doe" scanner_tool=$(echo "$scanner_tool" | sed "s/'/''/g" | sed 's/^[ \t]*//;s/[ \t]*$//' | tr -d '[:space:]')
local pod="Security"
# Clean up input values # Check if vurn_id exists in the database
vurn_id=$(echo "$vurn_id" | sed "s/'/''/g") existing_entry=$(psql -t -c "SELECT vurn_id FROM vulnerability_tracking WHERE vurn_id = '$vurn_id'" "postgresql://$DB_USER:$DB_PWD@$DB_HOST/$DB_NAME" 2>/dev/null)
priority=$(echo "$priority" | sed "s/'/''/g")
product=$(echo "$product" | sed "s/'/''/g" | tr -d '[:space:]' | sed 's/[|]//g' | sed 's/,$//')
scanner_tool=$(echo "$scanner_tool" | sed "s/'/''/g" | tr -d '[:space:]' | sed 's/[|]//g' | sed 's/,$//')
# Fetch existing values for this vulnerability ID
existing_entry=$(psql -t -c "SELECT product, scanner_tool FROM vulnerability_tracking WHERE vurn_id = '$vurn_id'" "postgresql://$DB_USER:$DB_PWD@$DB_HOST/$DB_NAME" 2>/dev/null)
# Process fetched data
if [[ -z "$existing_entry" ]]; then if [[ -z "$existing_entry" ]]; then
combined_products="$product" # If vurn_id doesn't exist, store data in CSV file
combined_scanner_tools="$scanner_tool" echo "$vurn_id,$product,$scanner_tool,$priority" >> "$new_vulns_file"
echo "New vulnerability detected: $vurn_id"
else else
IFS='|' read -r existing_product existing_scanner_tool <<< "$existing_entry" echo "Skipping existing vulnerability: $vurn_id"
combined_products=$(echo "$existing_product,$product" | tr ',' '\n' | sed '/^$/d' | sort -u | tr '\n' ',' | sed 's/^,//; s/,$//')
combined_scanner_tools=$(echo "$existing_scanner_tool,$scanner_tool" | tr ',' '\n' | sed '/^$/d' | sort -u | tr '\n' ',' | sed 's/^,//; s/,$//')
fi fi
# Write the insert query to the SQL file
echo "INSERT INTO vulnerability_tracking (product, scanner_tool, vurn_id, priority, pr_id, pr_link, github_run_id, created_date, update_date, comments, owner, pod)
VALUES ('$combined_products', '$combined_scanner_tools', '$vurn_id', '$priority', '$pr_id', '$pr_link', '$GITHUB_RUN_ID', '$created_date', '$created_date', '$comments', '$owner', '$pod')
ON CONFLICT (vurn_id)
DO UPDATE SET
product = '$combined_products',
scanner_tool = '$combined_scanner_tools',
priority = EXCLUDED.priority,
pr_id = EXCLUDED.pr_id,
pr_link = EXCLUDED.pr_link,
github_run_id = EXCLUDED.github_run_id,
update_date = EXCLUDED.update_date,
comments = EXCLUDED.comments,
owner = EXCLUDED.owner,
pod = EXCLUDED.pod;" >> "$query_file"
done < "$CSV_OUTPUT_FILE" done < "$CSV_OUTPUT_FILE"
echo "COMMIT;" >> "$query_file" # Print the contents of new vulnerabilities
echo "Queries written to $query_file." if [ -s "$new_vulns_file" ]; then
echo "****************************************************************"
# Execute the SQL file and rollback on failure echo "New vulnerabilities stored in $new_vulns_file:"
if psql -e "postgresql://$DB_USER:$DB_PWD@$DB_HOST/$DB_NAME" -f "$query_file"; then cat "$new_vulns_file"
echo "Vulnerabilities successfully inserted into the database." echo "****************************************************************"
else else
echo "Error: Failed to insert vulnerabilities. Performing rollback." echo "No new vulnerabilities to store."
echo "ROLLBACK;" | psql "postgresql://$DB_USER:$DB_PWD@$DB_HOST/$DB_NAME"
exit 1
fi fi
} }
# Check if there are vulnerabilities to process
if [ -s "$CSV_OUTPUT_FILE" ]; then if [ -s "$CSV_OUTPUT_FILE" ]; then
insert_vulns_into_db compare_and_store_vulns
else else
echo "No new vulnerabilities to insert." echo "No vulnerabilities to process."
fi fi

61
scripts/trivy_vulnerabilities_data.sh
View File

@ -89,30 +89,24 @@ else
fi fi
# Insert new vulnerabilities into PostgreSQL # Compare each vulnerability with the database and store new ones in a CSV file
insert_vulns_into_db() { compare_and_store_vulns() {
local query_file="insert_vulns.sql" local new_vulns_file="trivy_new_vulnerabilities.csv"
echo "BEGIN;" > "$query_file"
echo "vurn_id,product,scanner_tool,priority" > "$new_vulns_file" # CSV header
while IFS=, read -r vurn_id product scanner_tool priority; do while IFS=, read -r vurn_id product scanner_tool priority; do
if [[ -z "$vurn_id" || -z "$priority" || -z "$product" || -z "$scanner_tool" ]]; then if [[ -z "$vurn_id" || -z "$priority" || -z "$product" || -z "$scanner_tool" ]]; then
continue continue
fi fi
local pr_id="${GITHUB_PR_ID:-}"
local pr_link="${GITHUB_PR_LINK:-}"
local created_date=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
local comments="Initial vulnerability report"
local owner="John Doe"
local pod="Security"
# Remove spaces and redundant commas, and escape single quotes for SQL # Remove spaces and redundant commas, and escape single quotes for SQL
vurn_id=$(echo "$vurn_id" | sed "s/'/''/g") vurn_id=$(echo "$vurn_id" | sed "s/'/''/g")
priority=$(echo "$priority" | sed "s/'/''/g") priority=$(echo "$priority" | sed "s/'/''/g")
product=$(echo "$product" | sed "s/'/''/g" | tr -d ' ' | sed 's/,*$//') product=$(echo "$product" | sed "s/'/''/g" | tr -d ' ' | sed 's/,*$//')
scanner_tool=$(echo "$scanner_tool" | sed "s/'/''/g" | tr -d ' ' | sed 's/,*$//') scanner_tool=$(echo "$scanner_tool" | sed "s/'/''/g" | tr -d ' ' | sed 's/,*$//')
# Fetch existing product and scanner_tool values for the vulnerability # Check if vurn_id exists in the database
existing_entry=$(psql -t -c "SELECT product, scanner_tool FROM vulnerability_tracking WHERE vurn_id = '$vurn_id'" "postgresql://$DB_USER:$DB_PWD@$DB_HOST/$DB_NAME" 2>/dev/null) existing_entry=$(psql -t -c "SELECT product, scanner_tool FROM vulnerability_tracking WHERE vurn_id = '$vurn_id'" "postgresql://$DB_USER:$DB_PWD@$DB_HOST/$DB_NAME" 2>/dev/null)
if [ -n "$existing_entry" ]; then if [ -n "$existing_entry" ]; then
@ -131,39 +125,32 @@ insert_vulns_into_db() {
unique_scanner_tools="$scanner_tool" unique_scanner_tools="$scanner_tool"
fi fi
# Write the insert query to the SQL file # If the vulnerability is new, store it in the CSV file
echo "INSERT INTO vulnerability_tracking (product, scanner_tool, vurn_id, priority, pr_id, pr_link, github_run_id, created_date, update_date, comments, owner, pod) if [[ -z "$existing_entry" ]]; then
VALUES ('$unique_products', '$unique_scanner_tools', '$vurn_id', '$priority', '$pr_id', '$pr_link', '$GITHUB_RUN_ID', '$created_date', '$created_date', '$comments', '$owner', '$pod') echo "$vurn_id,$unique_products,$unique_scanner_tools,$priority" >> "$new_vulns_file"
ON CONFLICT (vurn_id) echo "New vulnerability detected: $vurn_id"
DO UPDATE SET else
product = '$unique_products', echo "Skipping existing vulnerability: $vurn_id"
scanner_tool = '$unique_scanner_tools', fi
priority = EXCLUDED.priority,
pr_id = EXCLUDED.pr_id,
pr_link = EXCLUDED.pr_link,
github_run_id = EXCLUDED.github_run_id,
update_date = EXCLUDED.update_date,
comments = EXCLUDED.comments,
owner = EXCLUDED.owner,
pod = EXCLUDED.pod;" >> "$query_file"
done < "$NEW_VULN_FILE" done < "$NEW_VULN_FILE"
echo "COMMIT;" >> "$query_file" # Print the contents of new vulnerabilities
if [ -s "$new_vulns_file" ]; then
# Execute the SQL file echo "****************************************************************"
if psql -e "postgresql://$DB_USER:$DB_PWD@$DB_HOST/$DB_NAME" -f "$query_file"; then echo "New vulnerabilities stored in $new_vulns_file:"
echo "Vulnerabilities successfully inserted into the database." cat "$new_vulns_file"
echo "****************************************************************"
else else
echo "Error: Failed to insert vulnerabilities. Check logs for details." echo "No new vulnerabilities to store."
exit 1
fi fi
} }
# Run insertion if vulnerabilities are found # Run comparison and storage if vulnerabilities are found
if [ -s "$NEW_VULN_FILE" ]; then if [ -s "$NEW_VULN_FILE" ]; then
insert_vulns_into_db compare_and_store_vulns
else else
echo "No vulnerabilities to insert." echo "No vulnerabilities to process."
fi fi
# Cleanup # Cleanup