diff --git a/.github/workflows/ci-test-custom-script.yml b/.github/workflows/ci-test-custom-script.yml index f7e1ccef83..4bc883b0ae 100644 --- a/.github/workflows/ci-test-custom-script.yml +++ b/.github/workflows/ci-test-custom-script.yml @@ -44,6 +44,12 @@ jobs: - name: Set up Depot CLI uses: depot/setup-action@v1 + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + # Check out merge commit - name: Fork based /ok-to-test checkout if: inputs.pr != 0 @@ -104,11 +110,16 @@ jobs: working-directory: app/client run: | failed_spec_env="" - while IFS= read -r line - do - spec_name=$(echo $line | awk -F'/' '{print $NF}') - failed_spec=$(find . -name $spec_name | sed 's|./||') - failed_spec_env="$failed_spec_env,$failed_spec" + while IFS= read -r line || [ -n "$line" ]; do + file_path=$(echo "$line" | awk -F'/' '{print $(NF-1)"/"$NF}') + spec_name=$(echo "$file_path" | awk -F'/' '{print $NF}') + failed_spec=$(find . -name "$spec_name" | sed 's|./||') + for file in $failed_spec; do + new_file_path=$(echo "$file" | awk -F'/' '{print $(NF-1)"/"$NF}') + if [ "$new_file_path" == "$file_path" ]; then + failed_spec_env="$failed_spec_env,$file" + fi + done done < ~/failed_spec_ci/failed_spec_ci-${{ matrix.job }} failed_spec_env=${failed_spec_env#,} echo "failed_spec_env=$failed_spec_env" >> $GITHUB_ENV @@ -148,6 +159,33 @@ jobs: docker run --name test-event-driver -d -p 22:22 -p 5001:5001 -p 3306:3306 \ -p 5432:5432 -p 28017:27017 -p 25:25 -p 5000:5000 -p 3001:3000 -p 6001:6001 --privileged --pid=host --ipc=host --volume /:/host -v ~/git-server/keys:/git-server/keys \ -v ~/git-server/repos:/git-server/repos appsmith/test-event-driver:latest + docker run --name cloud-services -d -p 8000:80 -p 8090:8090 \ + --privileged --pid=host --ipc=host --add-host=host.docker.internal:host-gateway\ + -e APPSMITH_CLOUD_SERVICES_MONGODB_URI=mongodb://host.docker.internal:27017 \ + -e APPSMITH_CLOUD_SERVICES_MONGODB_DATABASE=cs \ + -e APPSMITH_CLOUD_SERVICES_MONGODB_AUTH_DATABASE=admin \ + -e APPSMITH_REDIS_URL=redis://host.docker.internal:6379/ \ + -e APPSMITH_APPS_API_KEY=dummy-api-key \ + -e APPSMITH_REMOTE_API_KEY=dummy-api-key \ + -e APPSMITH_GITHUB_API_KEY=dummy-appsmith-gh-api-key \ + -e APPSMITH_JWT_SECRET=appsmith-cloud-services-jwt-secret-dummy-key \ + -e APPSMITH_ENCRYPTION_SALT=encryption-salt \ + -e APPSMITH_ENCRYPTION_PASSWORD=encryption-password \ + -e APPSMITH_CLOUD_SERVICES_URL=https://cs-dev.appsmith.com \ + -e APPSMITH_CUSTOMER_PORTAL_URL=https://dev.appsmith.com \ + -e APPSMITH_CLOUD_SERVICES_BASE_URL=https://cs-dev.appsmith.com \ + -e APPSMITH_CLOUD_SERVER_BASE_URL=https://release.app.appsmith.com \ + -e AUTH0_ISSUER_URL=https://login.release-customer.appsmith.com/ \ + -e AUTH0_CLIENT_ID=dummy-client-id \ + -e AUTH0_CLIENT_SECRET=dummy-secret-id \ + -e AUTH0_AUDIENCE_URL=https://login.local-customer.appsmith.com/ \ + -e CLOUDSERVICES_URL=cs-dev.appsmith.com \ + -e CUSTOMER_URL=dev.appsmith.com \ + -e ENTERPRISE_USER_NAME=ent-user@appsmith.com \ + -e ENTERPRISE_USER_PASSWORD=ent_user_password \ + -e ENTERPRISE_ADMIN_NAME=ent-admin@appsmith.com \ + -e ENTERPRISE_ADMIN_PASSWORD=ent_admin_password \ + appsmith/cloud-services:release cd cicontainerlocal docker run -d --name appsmith -p 80:80 -p 9001:9001 \ -v "$PWD/stacks:/appsmith-stacks" \ @@ -272,7 +310,11 @@ jobs: if [[ ${{ inputs.pr }} -ne 0 ]]; then echo "COMMIT_INFO_MESSAGE=${{ env.COMMIT_INFO_MESSAGE }}" >> $GITHUB_ENV else - echo "COMMIT_INFO_MESSAGE=$(echo \"${{ env.EVENT_COMMITS }}\" | awk -F '\\\\n' '{print $1}' | sed 's/^\"//')" >> $GITHUB_ENV + if [[ '${{env.EVENT_COMMITS}}' == 'null' ]]; then + echo "COMMIT_INFO_MESSAGE=${{ github.event_name }} by ${{ env.COMMIT_INFO_AUTHOR }}" >> $GITHUB_ENV + else + echo "COMMIT_INFO_MESSAGE=$(echo \"${{ env.EVENT_COMMITS }}\" | awk -F '\\\\n' '{print $1}' | sed 's/^\"//')" >> $GITHUB_ENV + fi fi - name: Run the cypress test @@ -328,6 +370,7 @@ jobs: CYPRESS_CONFIG_FILE: cypress_ci_custom.config.ts CYPRESS_HEADLESS: "true" CYPRESS_BROWSER: ${{ env.BROWSER_PATH }} + RUNID: ${{ github.run_id }} ATTEMPT_NUMBER: ${{ github.run_attempt }} REPOSITORY: ${{ github.repository }} COMMITTER: ${{ env.COMMIT_INFO_AUTHOR }} @@ -394,6 +437,7 @@ jobs: CYPRESS_CONFIG_FILE: cypress_ci.config.ts CYPRESS_HEADLESS: "true" CYPRESS_BROWSER: ${{ env.BROWSER_PATH }} + RUNID: ${{ github.run_id }} ATTEMPT_NUMBER: ${{ github.run_attempt }} REPOSITORY: ${{ github.repository }} COMMITTER: ${{ env.COMMIT_INFO_AUTHOR }} diff --git a/.github/workflows/ci-test-limited.yml b/.github/workflows/ci-test-limited.yml index 7ed390537e..4a46662543 100644 --- a/.github/workflows/ci-test-limited.yml +++ b/.github/workflows/ci-test-limited.yml @@ -71,6 +71,12 @@ jobs: - name: Set up Depot CLI uses: depot/setup-action@v1 + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + # Check out merge commit - name: Fork based /ok-to-test checkout if: inputs.pr != 0 @@ -183,6 +189,33 @@ jobs: docker run --name test-event-driver -d -p 22:22 -p 5001:5001 -p 3306:3306 \ -p 5432:5432 -p 28017:27017 -p 25:25 -p 5000:5000 -p 3001:3000 -p 6001:6001 --privileged --pid=host --ipc=host --volume /:/host -v ~/git-server/keys:/git-server/keys \ -v ~/git-server/repos:/git-server/repos appsmith/test-event-driver:latest + docker run --name cloud-services -d -p 8000:80 -p 8090:8090 \ + --privileged --pid=host --ipc=host --add-host=host.docker.internal:host-gateway\ + -e APPSMITH_CLOUD_SERVICES_MONGODB_URI=mongodb://host.docker.internal:27017 \ + -e APPSMITH_CLOUD_SERVICES_MONGODB_DATABASE=cs \ + -e APPSMITH_CLOUD_SERVICES_MONGODB_AUTH_DATABASE=admin \ + -e APPSMITH_REDIS_URL=redis://host.docker.internal:6379/ \ + -e APPSMITH_APPS_API_KEY=dummy-api-key \ + -e APPSMITH_REMOTE_API_KEY=dummy-api-key \ + -e APPSMITH_GITHUB_API_KEY=dummy-appsmith-gh-api-key \ + -e APPSMITH_JWT_SECRET=appsmith-cloud-services-jwt-secret-dummy-key \ + -e APPSMITH_ENCRYPTION_SALT=encryption-salt \ + -e APPSMITH_ENCRYPTION_PASSWORD=encryption-password \ + -e APPSMITH_CLOUD_SERVICES_URL=https://cs-dev.appsmith.com \ + -e APPSMITH_CUSTOMER_PORTAL_URL=https://dev.appsmith.com \ + -e APPSMITH_CLOUD_SERVICES_BASE_URL=https://cs-dev.appsmith.com \ + -e APPSMITH_CLOUD_SERVER_BASE_URL=https://release.app.appsmith.com \ + -e AUTH0_ISSUER_URL=https://login.release-customer.appsmith.com/ \ + -e AUTH0_CLIENT_ID=dummy-client-id \ + -e AUTH0_CLIENT_SECRET=dummy-secret-id \ + -e AUTH0_AUDIENCE_URL=https://login.local-customer.appsmith.com/ \ + -e CLOUDSERVICES_URL=cs-dev.appsmith.com \ + -e CUSTOMER_URL=dev.appsmith.com \ + -e ENTERPRISE_USER_NAME=ent-user@appsmith.com \ + -e ENTERPRISE_USER_PASSWORD=ent_user_password \ + -e ENTERPRISE_ADMIN_NAME=ent-admin@appsmith.com \ + -e ENTERPRISE_ADMIN_PASSWORD=ent_admin_password \ + appsmith/cloud-services:release cd cicontainerlocal docker run -d --name appsmith -p 80:80 -p 9001:9001 \ -v "$PWD/stacks:/appsmith-stacks" \ @@ -313,7 +346,11 @@ jobs: elif [[ ${{ inputs.pr }} -ne 0 && ${{github.event_name}} == 'workflow_dispatch' ]]; then echo "COMMIT_INFO_MESSAGE=Workflow run on PR# ${{ inputs.pr }}" >> $GITHUB_ENV else - echo "COMMIT_INFO_MESSAGE=$(echo \"${{ env.EVENT_COMMITS }}\" | awk -F '\\\\n' '{print $1}' | sed 's/^\"//')" >> $GITHUB_ENV + if [[ '${{env.EVENT_COMMITS}}' == 'null' ]]; then + echo "COMMIT_INFO_MESSAGE=${{ github.event_name }} by ${{ env.COMMIT_INFO_AUTHOR }}" >> $GITHUB_ENV + else + echo "COMMIT_INFO_MESSAGE=$(echo \"${{ env.EVENT_COMMITS }}\" | awk -F '\\\\n' '{print $1}' | sed 's/^\"//')" >> $GITHUB_ENV + fi fi - name: Run the cypress test @@ -364,6 +401,7 @@ jobs: APPSMITH_GOOGLE_MAPS_API_KEY: ${{ secrets.APPSMITH_GOOGLE_MAPS_API_KEY }} COMMIT_INFO_MESSAGE: ${{ env.COMMIT_INFO_MESSAGE }} CYPRESS_VERIFY_TIMEOUT: 100000 + RUNID: ${{ github.run_id }} ATTEMPT_NUMBER: ${{ github.run_attempt }} REPOSITORY: ${{ github.repository }} COMMITTER: ${{ env.COMMIT_INFO_AUTHOR }} @@ -438,6 +476,7 @@ jobs: APPSMITH_GOOGLE_MAPS_API_KEY: ${{ secrets.APPSMITH_GOOGLE_MAPS_API_KEY }} COMMIT_INFO_MESSAGE: ${{ env.COMMIT_INFO_MESSAGE }} CYPRESS_VERIFY_TIMEOUT: 100000 + RUNID: ${{ github.run_id }} ATTEMPT_NUMBER: ${{ github.run_attempt }} REPOSITORY: ${{ github.repository }} COMMITTER: ${{ env.COMMIT_INFO_AUTHOR }} diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 1581711a62..6424167385 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -312,7 +312,11 @@ jobs: if [[ ${{ inputs.pr }} -ne 0 ]]; then echo "COMMIT_INFO_MESSAGE=${{ env.COMMIT_INFO_MESSAGE }}" >> $GITHUB_ENV else - echo "COMMIT_INFO_MESSAGE=$(echo \"${{ env.EVENT_COMMITS }}\" | awk -F '\\\\n' '{print $1}' | sed 's/^\"//')" >> $GITHUB_ENV + if [[ '${{env.EVENT_COMMITS}}' == 'null' ]]; then + echo "COMMIT_INFO_MESSAGE=${{ github.event_name }} by ${{ env.COMMIT_INFO_AUTHOR }}" >> $GITHUB_ENV + else + echo "COMMIT_INFO_MESSAGE=$(echo \"${{ env.EVENT_COMMITS }}\" | awk -F '\\\\n' '{print $1}' | sed 's/^\"//')" >> $GITHUB_ENV + fi fi - name: Run the cypress test @@ -363,6 +367,7 @@ jobs: APPSMITH_GOOGLE_MAPS_API_KEY: ${{ secrets.APPSMITH_GOOGLE_MAPS_API_KEY }} COMMIT_INFO_MESSAGE: ${{ env.COMMIT_INFO_MESSAGE }} CYPRESS_VERIFY_TIMEOUT: 100000 + RUNID: ${{ github.run_id }} ATTEMPT_NUMBER: ${{ github.run_attempt }} REPOSITORY: ${{ github.repository }} COMMITTER: ${{ env.COMMIT_INFO_AUTHOR }} @@ -454,6 +459,7 @@ jobs: APPSMITH_DISABLE_TELEMETRY: true APPSMITH_GOOGLE_MAPS_API_KEY: ${{ secrets.APPSMITH_GOOGLE_MAPS_API_KEY }} COMMIT_INFO_MESSAGE: ${{ env.COMMIT_INFO_MESSAGE }} + RUNID: ${{ github.run_id }} ATTEMPT_NUMBER: ${{ github.run_attempt }} REPOSITORY: ${{ github.repository }} COMMITTER: ${{ env.COMMIT_INFO_AUTHOR }} diff --git a/app/client/cypress/scripts/cypress-hooks.js b/app/client/cypress/scripts/cypress-hooks.js index 117c548cb2..372db2e27a 100644 --- a/app/client/cypress/scripts/cypress-hooks.js +++ b/app/client/cypress/scripts/cypress-hooks.js @@ -1,4 +1,4 @@ -const { Client } = require("pg"); +const { Pool } = require("pg"); const os = require("os"); const AWS = require("aws-sdk"); const fs = require("fs"); @@ -25,12 +25,12 @@ function configureDbClient() { database: getEnvValue("CYPRESS_DB_NAME", { required: true }), password: getEnvValue("CYPRESS_DB_PWD", { required: true }), port: 5432, + connectionTimeoutMillis: 60000, ssl: true, keepalives: 0, }; - const dbClient = new Client(dbConfig); - dbClient.connect(); + const dbClient = new Pool(dbConfig); return dbClient; } @@ -61,7 +61,6 @@ function uploadToS3(s3Client, filePath, key) { async function cypressHooks(on, config) { const s3 = configureS3(); - const dbClient = configureDbClient(); const runData = { commitMsg: getEnvValue("COMMIT_INFO_MESSAGE", { required: false }), workflowId: getEnvValue("RUNID", { required: true }), @@ -81,6 +80,7 @@ async function cypressHooks(on, config) { await on("before:run", async (runDetails) => { runData.browser = runDetails.browser.name; + const dbClient = await configureDbClient().connect(); try { const runResponse = await dbClient.query( `INSERT INTO public.attempt ("workflowId", "attempt", "browser", "os", "repo", "committer", "type", "commitMsg", "branch") @@ -125,12 +125,15 @@ async function cypressHooks(on, config) { matrix.id = matrixResponse.rows[0].id; // Save the inserted matrix ID for later updates } catch (err) { console.log(err); + } finally { + await dbClient.release(); } }); await on("before:spec", async (spec) => { specData.name = spec.relative; specData.matrixId = matrix.id; + const dbClient = await configureDbClient().connect(); try { const specResponse = await dbClient.query( 'INSERT INTO public.specs ("name", "matrixId") VALUES ($1, $2) RETURNING id', @@ -139,6 +142,8 @@ async function cypressHooks(on, config) { specData.specId = specResponse.rows[0].id; // Save the inserted spec ID for later updates } catch (err) { console.log(err); + } finally { + await dbClient.release(); } }); @@ -150,6 +155,7 @@ async function cypressHooks(on, config) { specData.skipped = results.stats.skipped; specData.status = results.stats.failures > 0 ? "fail" : "pass"; + const dbClient = await configureDbClient().connect(); try { await dbClient.query( 'UPDATE public.specs SET "testCount" = $1, "passes" = $2, "failed" = $3, "skipped" = $4, "pending" = $5, "status" = $6 WHERE id = $7', @@ -207,10 +213,13 @@ async function cypressHooks(on, config) { } } catch (err) { console.log(err); + } finally { + await dbClient.release(); } }); on("after:run", async (runDetails) => { + const dbClient = await configureDbClient().connect(); try { await dbClient.query( `UPDATE public.matrix SET "status" = $1 WHERE id = $2`, @@ -220,9 +229,10 @@ async function cypressHooks(on, config) { `UPDATE public.attempt SET "endTime" = $1 WHERE "id" = $2`, [new Date(), runData.attemptId], ); - await dbClient.end(); } catch (err) { console.log(err); + } finally { + await dbClient.end(); } }); }