From 2e376a79f280994d079038663949289f04950735 Mon Sep 17 00:00:00 2001 From: Saroj <43822041+sarojsarab@users.noreply.github.com> Date: Tue, 24 Jan 2023 13:25:58 +0530 Subject: [PATCH] test: Upgrade Appsmith [POC] (#20006) ## Description - Appsmith Upgrade Test ## Type of change - Cypress ## How Has This Been Tested? - Cypress ## Checklist: ### QA activity: - [ ] Test plan has been approved by relevant developers - [ ] Test plan has been peer reviewed by QA - [ ] Cypress test cases have been added and approved by either SDET or manual QA - [ ] Organized project review call with relevant stakeholders after Round 1/2 of QA - [ ] Added Test Plan Approved label after reveiwing all Cypress test Co-authored-by: Aishwarya UR --- .github/workflows/upgrade-appsmith-tests.yml | 464 ++++++++++++++++++ .../UpgradeAppsmith/UpgradeAppsimth_spec.js | 10 +- 2 files changed, 469 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/upgrade-appsmith-tests.yml diff --git a/.github/workflows/upgrade-appsmith-tests.yml b/.github/workflows/upgrade-appsmith-tests.yml new file mode 100644 index 0000000000..ebf1811c63 --- /dev/null +++ b/.github/workflows/upgrade-appsmith-tests.yml @@ -0,0 +1,464 @@ +name: Upgrde Appsmith Test Workflow + +on: + # This workflow is only triggered by the ok to test command dispatch + repository_dispatch: + types: [ ok-to-test-command ] + +jobs: + notify: + runs-on: ubuntu-latest + steps: + # This step creates a comment on the PR with a link to this workflow run. + - name: Add a comment on the PR with link to workflow run + uses: peter-evans/create-or-update-comment@v2 + with: + issue-number: ${{ github.event.client_payload.pull_request.number }} + body: | + Tests running at: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}>. + Workflow: `${{ github.workflow }}`. + Commit: `${{ github.event.client_payload.slash_command.args.named.sha }}`. + PR: ${{ github.event.client_payload.pull_request.number }}. + Perf tests will be available at + + server-build: + name: server-build + uses: ./.github/workflows/server-build.yml + secrets: inherit + with: + pr: ${{ github.event.client_payload.pull_request.number }} + + client-build: + name: client-build + uses: ./.github/workflows/client-build.yml + secrets: inherit + with: + pr: ${{ github.event.client_payload.pull_request.number }} + + rts-build: + name: rts-build + uses: ./.github/workflows/rts-build.yml + secrets: inherit + with: + pr: ${{ github.event.client_payload.pull_request.number }} + + test-shared-modules: + name: shared-modules + uses: ./.github/workflows/shared-modules.yml + secrets: inherit + with: + pr: ${{ github.event.client_payload.pull_request.number }} + + test-appsmithctl: + name: appsmithctl + uses: ./.github/workflows/appsmithctl.yml + secrets: inherit + with: + pr: ${{ github.event.client_payload.pull_request.number }} + + fat-container-test: + needs: [ client-build, server-build, rts-build ] + # Only run if the build step is successful + if: success() + runs-on: ubuntu-latest + defaults: + run: + shell: bash + strategy: + fail-fast: false + matrix: + job: [ 0, 1 ] + + # Service containers to run with this job. Required for running tests + services: + # Label used to access the service container + redis: + # Docker Hub image for Redis + image: redis + ports: + # Opens tcp port 6379 on the host and service container + - 6379:6379 + mongo: + image: mongo + ports: + - 27017:27017 + + steps: + - name: Set up Depot CLI + uses: depot/setup-action@v1 + + # Check out merge commit + - name: Fork based /ok-to-test checkout + uses: actions/checkout@v3 + with: + ref: "refs/pull/${{ github.event.client_payload.pull_request.number }}/merge" + + # Timestamp will be used to create cache key + - id: timestamp + run: echo "timestamp=$(date +'%Y-%m-%dT%H:%M:%S')" >> $GITHUB_OUTPUT + + # In case this is second attempt try restoring status of the prior attempt from cache + - name: Restore the previous run result + id: cache-appsmith + uses: martijnhols/actions-cache@v3.0.2 + with: + path: | + ~/run_result + key: ${{ github.run_id }}-${{ github.job }}-${{ matrix.job }} + restore-keys: | + ${{ github.run_id }}-${{ github.job }}-${{ matrix.job }} + + - name: Get the previous run result + if: steps.cache-appsmith.outputs.cache-hit == 'true' + id: run_result + run: | + run_result_env=$(cat ~/run_result) + echo "run_result=$run_result_env" >> $GITHUB_OUTPUT + + - name: Dump steps context + env: + STEPS_CONTEXT: ${{ toJson(steps) }} + run: echo "$STEPS_CONTEXT" + + # In case this is second attempt try restoring failed tests + - name: Restore the previous failed combine result + if: steps.run_result.outputs.run_result == 'failedtest' + uses: martijnhols/actions-cache/restore@v3 + with: + path: | + ~/combined_failed_spec_fat + key: ${{ github.run_id }}-"ui-test-result" + restore-keys: | + ${{ github.run_id }}-${{ github.job }} + + # failed_spec_env will contain list of all failed specs + # We are using environment variable instead of regular to support multiline + - name: Get failed_spec + if: steps.run_result.outputs.run_result == 'failedtest' + run: | + failed_spec_env=$(cat ~/combined_failed_spec_fat) + echo "failed_spec_env<> $GITHUB_ENV + echo "$failed_spec_env" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + + - if: steps.run_result.outputs.run_result != 'success' && steps.run_result.outputs.run_result != 'failedtest' + run: echo "Starting full run" && exit 0 + + - if: steps.run_result.outputs.run_result == 'failedtest' + run: echo "Rerunning failed tests" && exit 0 + + - name: cat run_result + run: echo ${{ steps.run_result.outputs.run_result }} + + # Setup Java + - name: Set up JDK 17 + if: steps.run_result.outputs.run_result != 'success' + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: '17' + + - name: Download the client build artifact + uses: actions/download-artifact@v3 + with: + name: build + path: app/client/build + + - name: Download the server build artifact + uses: actions/download-artifact@v3 + with: + name: build + path: app/server/dist/ + + - name: Download the rts build artifact + uses: actions/download-artifact@v3 + with: + name: rts-dist + path: app/rts/dist + + - name: Un-tar the rts folder + run: | + tar -xvf app/rts/dist/rts-dist.tar -C app/rts/ + echo "Cleaning up the tar files" + rm app/rts/dist/rts-dist.tar + + # We don't use Depot Docker builds because it's faster for local Docker images to be built locally. + # It's slower and more expensive to build these Docker images on Depot and download it back to the CI node. + - name: Build docker image + if: steps.run_result.outputs.run_result != 'success' + working-directory: "." + run: | + docker build -t fatcontainer . + + - name: Create folder + if: steps.run_result.outputs.run_result != 'success' + env: + APPSMITH_LICENSE_KEY: ${{ secrets.APPSMITH_LICENSE_KEY }} + working-directory: "." + run: | + mkdir -p fatcontainerlocal/stacks/configuration/ + mkdir -p fatcontainerlocal/oldstack + + - name: Download S3 image + uses: keithweaver/aws-s3-github-action@v1.0.0 + with: + command: cp + source: s3://ci-assets--appsmith/ + destination: /home/runner/work/appsmith/appsmith/fatcontainerlocal/oldstack + aws_access_key_id: ${{ secrets.S3_CI_ASSETS_ACCESS_KEY_ID }} + aws_secret_access_key: ${{ secrets.S3_CI_ASSETS_SECRET_ACCESS_KEY }} + aws_region: ap-south-1 + flags: --recursive + + - name: Load docker image + if: steps.run_result.outputs.run_result != 'success' + env: + APPSMITH_LICENSE_KEY: ${{ secrets.APPSMITH_LICENSE_KEY }} + working-directory: "." + run: | + mkdir -p ~/git-server/keys + mkdir -p ~/git-server/repos + docker run --name test-event-driver -d -p 2222:22 -p 5001:5001 -p 3306:3306 \ + -p 5432:5432 -p 28017:27017 -p 25:25 --privileged --pid=host --ipc=host --volume /:/host -v ~/git-server/keys:/git-server/keys \ + -v ~/git-server/repos:/git-server/repos appsmith/test-event-driver:latest + cd fatcontainerlocal + docker run -d --name appsmith -p 80:80 -p 9001:9001 \ + -v "$PWD/stacks:/appsmith-stacks" -e APPSMITH_LICENSE_KEY=$APPSMITH_LICENSE_KEY \ + -e APPSMITH_AUDITLOG_ENABLED=true \ + -e APPSMITH_CLOUD_SERVICES_BASE_URL=http://host.docker.internal:5001 \ + --add-host=host.docker.internal:host-gateway \ + fatcontainer + + - name: Use Node.js 16.14.0 + if: steps.run_result.outputs.run_result != 'success' + uses: actions/setup-node@v3 + with: + node-version: "16.14.0" + + # Install all the dependencies + - name: Install dependencies + if: steps.run_result.outputs.run_result != 'success' + run: | + cd app/client + yarn install + + - name: Setting up the cypress tests + if: steps.run_result.outputs.run_result != 'success' + shell: bash + env: + APPSMITH_SSL_CERTIFICATE: ${{ secrets.APPSMITH_SSL_CERTIFICATE }} + APPSMITH_SSL_KEY: ${{ secrets.APPSMITH_SSL_KEY }} + CYPRESS_URL: ${{ secrets.CYPRESS_URL }} + CYPRESS_USERNAME: ${{ secrets.CYPRESS_USERNAME }} + CYPRESS_PASSWORD: ${{ secrets.CYPRESS_PASSWORD }} + CYPRESS_TESTUSERNAME1: ${{ secrets.CYPRESS_TESTUSERNAME1 }} + CYPRESS_TESTPASSWORD1: ${{ secrets.CYPRESS_TESTPASSWORD1 }} + CYPRESS_TESTUSERNAME2: ${{ secrets.CYPRESS_TESTUSERNAME2 }} + CYPRESS_TESTPASSWORD2: ${{ secrets.CYPRESS_TESTPASSWORD1 }} + CYPRESS_TESTUSERNAME3: ${{ secrets.CYPRESS_TESTUSERNAME3 }} + CYPRESS_TESTPASSWORD3: ${{ secrets.CYPRESS_TESTPASSWORD3 }} + CYPRESS_TESTUSERNAME4: ${{ secrets.CYPRESS_TESTUSERNAME4 }} + CYPRESS_TESTPASSWORD4: ${{ secrets.CYPRESS_TESTPASSWORD4 }} + CYPRESS_S3_ACCESS_KEY: ${{ secrets.CYPRESS_S3_ACCESS_KEY }} + CYPRESS_S3_SECRET_KEY: ${{ secrets.CYPRESS_S3_SECRET_KEY }} + CYPRESS_GITHUB_PERSONAL_ACCESS_TOKEN: ${{ secrets.CYPRESS_GITHUB_PERSONAL_ACCESS_TOKEN }} + CYPRESS_TEST_GITHUB_USER_NAME: ${{ secrets.CYPRESS_TEST_GITHUB_USER_NAME }} + CYPRESS_APPSMITH_OAUTH2_GOOGLE_CLIENT_ID: ${{ secrets.CYPRESS_APPSMITH_OAUTH2_GOOGLE_CLIENT_ID }} + CYPRESS_APPSMITH_OAUTH2_GOOGLE_CLIENT_SECRET: ${{ secrets.CYPRESS_APPSMITH_OAUTH2_GOOGLE_CLIENT_SECRET }} + CYPRESS_APPSMITH_OAUTH2_GITHUB_CLIENT_ID: ${{ secrets.CYPRESS_APPSMITH_OAUTH2_GITHUB_CLIENT_ID }} + CYPRESS_APPSMITH_OAUTH2_GITHUB_CLIENT_SECRET: ${{ secrets.CYPRESS_APPSMITH_OAUTH2_GITHUB_CLIENT_SECRET }} + APPSMITH_DISABLE_TELEMETRY: true + APPSMITH_GOOGLE_MAPS_API_KEY: ${{ secrets.APPSMITH_GOOGLE_MAPS_API_KEY }} + POSTGRES_PASSWORD: postgres + run: | + cd app/client + chmod a+x ./cypress/setup-test-fat.sh + ./cypress/setup-test-fat.sh + + - uses: browser-actions/setup-chrome@latest + with: + chrome-version: stable + - run: | + echo "BROWSER_PATH=$(which chrome)" >> $GITHUB_ENV + + - name: Run the cypress test + if: steps.run_result.outputs.run_result != 'success' && steps.run_result.outputs.run_result != 'failedtest' + uses: cypress-io/github-action@v2 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }} + CYPRESS_PROJECT_ID: ${{ secrets.CYPRESS_PROJECT_ID }} + CYPRESS_USERNAME: ${{ secrets.CYPRESS_USERNAME }} + CYPRESS_PASSWORD: ${{ secrets.CYPRESS_PASSWORD }} + CYPRESS_TESTUSERNAME1: ${{ secrets.CYPRESS_TESTUSERNAME1 }} + CYPRESS_TESTPASSWORD1: ${{ secrets.CYPRESS_TESTPASSWORD1 }} + CYPRESS_TESTUSERNAME2: ${{ secrets.CYPRESS_TESTUSERNAME2 }} + CYPRESS_TESTPASSWORD2: ${{ secrets.CYPRESS_TESTPASSWORD1 }} + CYPRESS_TESTUSERNAME3: ${{ secrets.CYPRESS_TESTUSERNAME3 }} + CYPRESS_TESTPASSWORD3: ${{ secrets.CYPRESS_TESTPASSWORD3 }} + CYPRESS_TESTUSERNAME4: ${{ secrets.CYPRESS_TESTUSERNAME4 }} + CYPRESS_TESTPASSWORD4: ${{ secrets.CYPRESS_TESTPASSWORD4 }} + CYPRESS_S3_ACCESS_KEY: ${{ secrets.CYPRESS_S3_ACCESS_KEY }} + CYPRESS_S3_SECRET_KEY: ${{ secrets.CYPRESS_S3_SECRET_KEY }} + CYPRESS_GITHUB_PERSONAL_ACCESS_TOKEN: ${{ secrets.CYPRESS_GITHUB_PERSONAL_ACCESS_TOKEN }} + CYPRESS_TEST_GITHUB_USER_NAME: ${{ secrets.CYPRESS_TEST_GITHUB_USER_NAME }} + CYPRESS_APPSMITH_OAUTH2_GOOGLE_CLIENT_ID: ${{ secrets.CYPRESS_APPSMITH_OAUTH2_GOOGLE_CLIENT_ID }} + CYPRESS_APPSMITH_OAUTH2_GOOGLE_CLIENT_SECRET: ${{ secrets.CYPRESS_APPSMITH_OAUTH2_GOOGLE_CLIENT_SECRET }} + CYPRESS_APPSMITH_OAUTH2_GITHUB_CLIENT_ID: ${{ secrets.CYPRESS_APPSMITH_OAUTH2_GITHUB_CLIENT_ID }} + CYPRESS_APPSMITH_OAUTH2_GITHUB_CLIENT_SECRET: ${{ secrets.CYPRESS_APPSMITH_OAUTH2_GITHUB_CLIENT_SECRET }} + APPSMITH_DISABLE_TELEMETRY: true + APPSMITH_GOOGLE_MAPS_API_KEY: ${{ secrets.APPSMITH_GOOGLE_MAPS_API_KEY }} + COMMIT_INFO_MESSAGE: ${{ github.event.pull_request.title }} + with: + browser: ${{ env.BROWSER_PATH }} + headless: true + record: true + install: false + parallel: true + config-file: cypress_fat.json + group: "Electrons on Github Action Fat Container" + spec: "cypress/UpgradeAppsmith/UpgradeAppsimth_spec.js" + working-directory: app/client + # tag will be either "push" or "pull_request" + tag: ${{ github.event_name }} + env: "NODE_ENV=development" + + # In case of second attempt only run failed specs + - name: Run the cypress test with failed tests + if: steps.run_result.outputs.run_result == 'failedtest' + uses: cypress-io/github-action@v2 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }} + CYPRESS_PROJECT_ID: ${{ secrets.CYPRESS_PROJECT_ID }} + CYPRESS_USERNAME: ${{ secrets.CYPRESS_USERNAME }} + CYPRESS_PASSWORD: ${{ secrets.CYPRESS_PASSWORD }} + CYPRESS_TESTUSERNAME1: ${{ secrets.CYPRESS_TESTUSERNAME1 }} + CYPRESS_TESTPASSWORD1: ${{ secrets.CYPRESS_TESTPASSWORD1 }} + CYPRESS_TESTUSERNAME2: ${{ secrets.CYPRESS_TESTUSERNAME2 }} + CYPRESS_TESTPASSWORD2: ${{ secrets.CYPRESS_TESTPASSWORD1 }} + CYPRESS_TESTUSERNAME3: ${{ secrets.CYPRESS_TESTUSERNAME3 }} + CYPRESS_TESTPASSWORD3: ${{ secrets.CYPRESS_TESTPASSWORD3 }} + CYPRESS_TESTUSERNAME4: ${{ secrets.CYPRESS_TESTUSERNAME4 }} + CYPRESS_TESTPASSWORD4: ${{ secrets.CYPRESS_TESTPASSWORD4 }} + CYPRESS_S3_ACCESS_KEY: ${{ secrets.CYPRESS_S3_ACCESS_KEY }} + CYPRESS_S3_SECRET_KEY: ${{ secrets.CYPRESS_S3_SECRET_KEY }} + CYPRESS_GITHUB_PERSONAL_ACCESS_TOKEN: ${{ secrets.CYPRESS_GITHUB_PERSONAL_ACCESS_TOKEN }} + CYPRESS_TEST_GITHUB_USER_NAME: ${{ secrets.CYPRESS_TEST_GITHUB_USER_NAME }} + CYPRESS_APPSMITH_OAUTH2_GOOGLE_CLIENT_ID: ${{ secrets.CYPRESS_APPSMITH_OAUTH2_GOOGLE_CLIENT_ID }} + CYPRESS_APPSMITH_OAUTH2_GOOGLE_CLIENT_SECRET: ${{ secrets.CYPRESS_APPSMITH_OAUTH2_GOOGLE_CLIENT_SECRET }} + CYPRESS_APPSMITH_OAUTH2_GITHUB_CLIENT_ID: ${{ secrets.CYPRESS_APPSMITH_OAUTH2_GITHUB_CLIENT_ID }} + CYPRESS_APPSMITH_OAUTH2_GITHUB_CLIENT_SECRET: ${{ secrets.CYPRESS_APPSMITH_OAUTH2_GITHUB_CLIENT_SECRET }} + APPSMITH_DISABLE_TELEMETRY: true + APPSMITH_GOOGLE_MAPS_API_KEY: ${{ secrets.APPSMITH_GOOGLE_MAPS_API_KEY }} + COMMIT_INFO_MESSAGE: ${{ github.event.pull_request.title }} + with: + browser: ${{ env.BROWSER_PATH }} + headless: true + record: true + install: false + parallel: true + config-file: cypress_fat.json + group: "Electrons on Github Action" + spec: "cypress/integration/Smoke_TestSuite_Fat/**/*" + working-directory: app/client + # tag will be either "push" or "pull_request" + tag: ${{ github.event_name }} + env: "NODE_ENV=development" + + - name: Collect fat container logs + if: always() + working-directory: "." + run: | + docker logs appsmith 2>&1 > ~/dockerlogs.txt + # Upload docker logs + - name: Upload failed test list artifact + if: always() + uses: actions/upload-artifact@v3 + with: + name: dockerlogs + path: ~/dockerlogs.txt + + # Set status = failedtest + - name: Set fail if there are test failures + if: failure() + run: echo "failedtest" >> $GITHUB_OUTPUT > ~/run_result + + # Create a directory ~/failed_spec_fat and add a dummy file + # This will ensure upload and download steps are successful + - name: Create directory for failed tests + if: always() + run: | + mkdir -p ~/failed_spec_fat + echo "empty" >> ~/failed_spec_fat/dummy-${{ matrix.job }} + + # add list failed tests to a file + - name: In case of test failures copy them to a file + if: failure() + run: | + cd ${{ github.workspace }}/app/client/cypress/ + find screenshots -type d|grep -i spec |sed 's/screenshots/cypress\/integration/g' > ~/failed_spec_fat/failed_spec_fat-${{ matrix.job }} + + # Upload failed test list using common path for all matrix job + - name: Upload failed test list artifact + if: always() + uses: actions/upload-artifact@v3 + with: + name: failed-spec-fat + path: ~/failed_spec_fat + + # Force store previous run result to cache + - name: Store the previous run result + if: failure() + uses: martijnhols/actions-cache/save@v3 + with: + path: | + ~/run_result + key: ${{ github.run_id }}-${{ github.job }}-${{ matrix.job }} + restore-keys: | + ${{ github.run_id }}-${{ github.job }}-${{ matrix.job }} + + # Force store previous failed test list to cache + - name: Store the previous failed test result + if: failure() + uses: martijnhols/actions-cache/save@v3 + with: + path: | + ~/failed_spec_fat + key: ${{ github.run_id }}-${{ github.job }}-${{ matrix.job }} + restore-keys: | + ${{ github.run_id }}-${{ github.job }}-${{ matrix.job }} + + # Upload the screenshots as artifacts if there's a failure + - uses: actions/upload-artifact@v3 + if: failure() + with: + name: cypress-screenshots-${{ matrix.job }} + path: app/client/cypress/screenshots/ + + - name: Restore the previous bundle + uses: actions/cache@v3 + with: + path: | + app/client/cypress/snapshots/ + key: ${{ github.run_id }}-${{ github.job }}-${{ matrix.job }} + restore-keys: | + ${{ github.run_id }}-${{ github.job }}-${{ matrix.job }} + + # Upload the snapshots as artifacts for layout validation + - uses: actions/upload-artifact@v3 + with: + name: cypress-snapshots-visualRegression + path: app/client/cypress/snapshots/ + + # Upload the log artifact so that it can be used by the test & deploy job in the workflow + - name: Upload server logs bundle on failure + uses: actions/upload-artifact@v3 + if: failure() + with: + name: server-logs-${{ matrix.job }} + path: app/server/server-logs.log + + # Set status = success + - name: Save the status of the run + run: echo "run_result=success" >> $GITHUB_OUTPUT > ~/run_result \ No newline at end of file diff --git a/app/client/cypress/manual_TestSuite/CommentedScriptFiles/UpgradeAppsmith/UpgradeAppsimth_spec.js b/app/client/cypress/manual_TestSuite/CommentedScriptFiles/UpgradeAppsmith/UpgradeAppsimth_spec.js index cb23a81704..1324106fe7 100644 --- a/app/client/cypress/manual_TestSuite/CommentedScriptFiles/UpgradeAppsmith/UpgradeAppsimth_spec.js +++ b/app/client/cypress/manual_TestSuite/CommentedScriptFiles/UpgradeAppsmith/UpgradeAppsimth_spec.js @@ -3,7 +3,7 @@ const testdata = require("../../../fixtures/testdata.json"); const testUrl = "http://localhost:5001/v1/parent/cmd"; describe("Upgrade appsmith version", () => { - it.skip("Upgrade Appsmith version and verify the Applications", () => { + it("Upgrade Appsmith version and verify the Applications", () => { const uuid = () => Cypress._.random(0, 10000); const name = uuid(); cy.wait(2000); @@ -29,8 +29,8 @@ describe("Upgrade appsmith version", () => { cy.log("Start old stack container"); cy.CreateAContainer( testUrl, - path + "/oldstack/160", - "latest", + path + "/oldstack", + "fatcontainer", `appsmith-160_${name}_updated`, ); cy.wait(45000); @@ -84,8 +84,8 @@ describe("Upgrade appsmith version", () => { cy.log("Start old stack container"); cy.CreateEEContainer( testUrl, - path + "/oldstack/ce", - "latest", + path + "/oldstack", + "fatcontainer", `appsmith-160-ce-${name}-enterprise`, ); cy.wait(45000);