From ceb00c6199da153729d1d52e206a12560bff7bc2 Mon Sep 17 00:00:00 2001 From: yasser khan Date: Fri, 18 Jul 2025 19:34:45 +0530 Subject: [PATCH] E2E: Run E2E tests with Server Init (#3452) * Run mm server via Docker * Run mm server via Docker * test separate server setup for e2e jobs * Revert "test separate server setup for e2e jobs" This reverts commit efd23823ac65de8945d635ce9f699c7a9e4fbd77. * Fix docker on windows * test run * test run * fix * try a script to trigger Server spinup * try a script to trigger Server spinup * test * remove the trigger workflow * Change Label * test-run with disabled E2E/Run label * test-run * test-run * Fixes desktop e2e * Fixes desktop e2e * Fixes desktop e2e * Fixes desktop e2e --------- Co-authored-by: Mattermost Build --- .github/PULL_REQUEST_TEMPLATE.md | 2 +- .github/workflows/e2e-functional-template.yml | 20 +- .github/workflows/e2e-functional.yml | 182 ++++++------------ e2e/modules/environment.js | 18 +- 4 files changed, 70 insertions(+), 152 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index a7c65983..42c701f0 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -28,7 +28,7 @@ Place an '[x]' (no spaces) in all applicable fields. Please remove unrelated fie - [ ] read and understood our [Contributing Guidelines](https://github.com/mattermost/desktop/blob/master/CONTRIBUTING.md) - [ ] completed [Mattermost Contributor Agreement](https://mattermost.com/contribute/) - [ ] executed `npm run lint:js` for proper code formatting -- [ ] Run E2E tests by adding label `Run Desktop E2E Tests` +- [ ] Run E2E tests by adding label `E2E/Run` #### Device Information This PR was tested on: diff --git a/.github/workflows/e2e-functional-template.yml b/.github/workflows/e2e-functional-template.yml index 99027365..bb8d9eaf 100644 --- a/.github/workflows/e2e-functional-template.yml +++ b/.github/workflows/e2e-functional-template.yml @@ -24,7 +24,7 @@ on: type: string description: "The E2E tests underlying OS" required: true - default: "ubuntu-22.04" + default: "ubuntu-24.04" nightly: type: boolean description: "True if this is nightly build" @@ -43,7 +43,7 @@ on: MM_SERVER_VERSION: type: string required: false - default: "9.9.1" + default: "10.5.0" outputs: NEW_FAILURES_LINUX: description: "The output to comment" @@ -96,7 +96,7 @@ on: type: string description: "The E2E tests underlying OS" required: true - default: "ubuntu-22.04" + default: "ubuntu-24.04" nightly: type: boolean description: "True if this is nightly build" @@ -115,14 +115,14 @@ on: MM_SERVER_VERSION: type: string required: false - default: "9.9.1" + default: "10.10.0" env: AWS_S3_BUCKET: "mattermost-cypress-report" BRANCH: ${{ github.head_ref || github.ref_name }} BUILD_TAG: ${{ github.event.pull_request.head.sha || github.sha }} JIRA_PROJECT_KEY: "MM" - MM_TEST_SERVER_URL: ${{ inputs.MM_TEST_SERVER_URL || secrets.MM_DESKTOP_E2E_SERVER_URL }} + MM_TEST_SERVER_URL: ${{ inputs.MM_TEST_SERVER_URL }} MM_TEST_USER_NAME: ${{ inputs.MM_TEST_USER_NAME || secrets.MM_DESKTOP_E2E_USER_NAME }} MM_TEST_PASSWORD: ${{ inputs.MM_TEST_PASSWORD || secrets.MM_DESKTOP_E2E_USER_CREDENTIALS }} PULL_REQUEST: "https://github.com/mattermost/desktop/pull/${{ github.event.number }}" @@ -136,7 +136,7 @@ env: ZEPHYR_API_KEY: ${{ secrets.MM_DESKTOP_E2E_ZEPHYR_API_KEY }} REPORT_LINK: "none" NODE_VERSION: 18 - SERVER_VERSION: ${{ inputs.MM_SERVER_VERSION || '9.9.1' }} + SERVER_VERSION: ${{ inputs.MM_SERVER_VERSION }} DESKTOP_VERSION: ${{ inputs.DESKTOP_VERSION }} PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 @@ -249,12 +249,11 @@ jobs: cd e2e && npm ci npx electron-rebuild --platform=${{ env.RUNNER_OS }} -f -t prod,optional,dev -w robotjs --module-dir ../ - - name: e2e/run-playright-tests-linux + - name: e2e/run-playwright-tests-linux if: runner.os == 'Linux' run: | eval "$(dbus-launch --sh-syntax)" export NO_AT_BRIDGE=1 - npm run build-test cd e2e export PW_CHROMIUM_ARGS="--disable-gpu --no-sandbox --disable-dev-shm-usage" @@ -264,7 +263,7 @@ jobs: SERVER_VERSION: ${{ inputs.MM_SERVER_VERSION }} DESKTOP_VERSION: ${{ inputs.DESKTOP_VERSION }} - - name: e2e/run-playright-tests-macos + - name: e2e/run-playwright-tests-macos if: runner.os == 'macOS' run: | sudo spctl --master-disable @@ -277,7 +276,7 @@ jobs: SERVER_VERSION: ${{ inputs.MM_SERVER_VERSION }} DESKTOP_VERSION: ${{ inputs.DESKTOP_VERSION }} - - name: e2e/run-playright-tests-windows + - name: e2e/run-playwright-tests-windows if: runner.os == 'Windows' run: | npm run build-test @@ -299,7 +298,6 @@ jobs: process.chdir('./e2e'); const { analyzeFlakyTests } = require('./utils/analyze-flaky-test.js'); const { newFailedTests, os } = analyzeFlakyTests(); - switch (os) { case 'linux': core.setOutput('NEW_FAILURES_LINUX', newFailedTests.length); diff --git a/.github/workflows/e2e-functional.yml b/.github/workflows/e2e-functional.yml index 37dcfee3..13b640bc 100644 --- a/.github/workflows/e2e-functional.yml +++ b/.github/workflows/e2e-functional.yml @@ -9,156 +9,92 @@ on: inputs: version_name: type: string - description: "Desktop Version name eg: 5.6" + description: "Desktop Version name eg: v5.11.0" required: true - platform: - type: choice - description: "Choose platform to run tests" - required: true - default: "all" - options: [linux, macos, windows, all] + instance_details: + type: string + description: "JSON array of platform details" + required: false + MM_TEST_USER_NAME: + description: "The admin username of the test instance" + required: false + type: string + MM_TEST_PASSWORD: + description: "The admin password of the test instance" + required: false + type: string + MM_SERVER_VERSION: + type: string + description: "The server version to test against" + required: false + default: "10.5.0" jobs: - update-initial-status-linux: - runs-on: ubuntu-22.04 - if: contains(github.event.label.name, 'Run Desktop E2E Tests') - steps: - - uses: mattermost/actions/delivery/update-commit-status@main - env: - GITHUB_TOKEN: ${{ github.token }} - with: - repository_full_name: ${{ github.repository }} - commit_sha: ${{ github.event.pull_request.head.sha }} - context: e2e/linux - description: E2E tests for Mattermost desktop app on Linux have started... - status: pending - - update-initial-status-macos: - runs-on: ubuntu-22.04 - if: contains(github.event.label.name, 'Run Desktop E2E Tests') - steps: - - uses: mattermost/actions/delivery/update-commit-status@main - env: - GITHUB_TOKEN: ${{ github.token }} - with: - repository_full_name: ${{ github.repository }} - commit_sha: ${{ github.event.pull_request.head.sha }} - context: e2e/macOS - description: E2E tests for Mattermost desktop app on macOS have started... - status: pending - - update-initial-status-windows: - runs-on: ubuntu-22.04 - if: contains(github.event.label.name, 'Run Desktop E2E Tests') - steps: - - uses: mattermost/actions/delivery/update-commit-status@main - env: - GITHUB_TOKEN: ${{ github.token }} - with: - repository_full_name: ${{ github.repository }} - commit_sha: ${{ github.event.pull_request.head.sha }} - context: e2e/windows - description: E2E tests for Mattermost desktop app on Windows have started... - status: pending - determine-platforms: - if: contains(github.event.label.name, 'Run Desktop E2E Tests') - needs: [update-initial-status-linux, update-initial-status-macos, update-initial-status-windows] runs-on: ubuntu-latest outputs: - platforms: ${{ steps.set-matrix.outputs.platforms }} + matrix: ${{ steps.set-matrix.outputs.matrix }} steps: - id: set-matrix run: | if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then - if [[ "${{ inputs.platform }}" == "all" ]]; then - echo "platforms=['ubuntu-24.04', 'macos-13', 'windows-2022']" >> $GITHUB_OUTPUT + if [[ -n "${{ inputs.instance_details }}" ]]; then + MATRIX=$(echo '${{ inputs.instance_details }}' | jq -c '[.[] | {os: .runner, platform: .platform, url: .url, installation_id: .installation_id}]') + echo "matrix=$MATRIX" >> $GITHUB_OUTPUT else - case "${{ inputs.platform }}" in - linux) echo "platforms=['ubuntu-24.04']" ;; - macos) echo "platforms=['macos-13']" ;; - windows) echo "platforms=['windows-2022']" - esac >> $GITHUB_OUTPUT + MATRIX='[{"os":"ubuntu-24.04","platform":"linux","url":"${{ secrets.MM_TEST_SERVER_URL }}","installation_id":"unknown"}, + {"os":"macos-13","platform":"macos","url":"${{ secrets.MM_TEST_SERVER_URL }}","installation_id":"unknown"}, + {"os":"windows-2022","platform":"windows","url":"${{ secrets.MM_TEST_SERVER_URL }}","installation_id":"unknown"}]' + echo "matrix=$MATRIX" >> $GITHUB_OUTPUT fi - elif [[ "${{ github.event_name }}" == "push" ]] || [[ "${{ contains(github.event.pull_request.labels.*.name, 'Run Desktop E2E Tests') }}" == "true" ]]; then - echo "platforms=['ubuntu-24.04', 'macos-13', 'windows-2022']" >> $GITHUB_OUTPUT + elif [[ "${{ github.event_name }}" == "push" ]] || [[ "${{ contains(github.event.pull_request.labels.*.name, 'E2E/Run') }}" == "true" ]]; then + MATRIX='[{"os":"ubuntu-24.04","platform":"linux","url":"${{ secrets.MM_TEST_SERVER_URL }}","installation_id":"unknown"}, + {"os":"macos-13","platform":"macos","url":"${{ secrets.MM_TEST_SERVER_URL }}","installation_id":"unknown"}, + {"os":"windows-2022","platform":"windows","url":"${{ secrets.MM_TEST_SERVER_URL }}","installation_id":"unknown"}]' + echo "matrix=$MATRIX" >> $GITHUB_OUTPUT else - echo "platforms=[]" >> $GITHUB_OUTPUT + echo "matrix=[]" >> $GITHUB_OUTPUT fi e2e-tests: needs: determine-platforms strategy: matrix: - os: ${{ fromJson(needs.determine-platforms.outputs.platforms) }} + platform: ${{ fromJson(needs.determine-platforms.outputs.matrix) }} fail-fast: false - uses: ./.github/workflows/e2e-functional-template.yml - secrets: inherit - with: - runs-on: ${{ matrix.os }} - DESKTOP_VERSION: ${{ inputs.version_name || github.head_ref || github.ref }} - - update-final-status-linux: - runs-on: ubuntu-22.04 - needs: - - e2e-tests + runs-on: ${{ matrix.platform.os }} steps: - - uses: mattermost/actions/delivery/update-commit-status@main + - name: Update initial commit status + uses: mattermost/actions/delivery/update-commit-status@main env: GITHUB_TOKEN: ${{ github.token }} with: repository_full_name: ${{ github.repository }} - commit_sha: ${{ github.event.pull_request.head.sha }} - context: e2e/linux - description: Completed with ${{ needs.e2e-tests.outputs.NEW_FAILURES_LINUX }} failures - status: ${{ needs.e2e-tests.outputs.STATUS_LINUX }} - target_url: ${{ needs.e2e-tests.outputs.REPORT_LINK_LINUX }} + commit_sha: ${{ github.event.pull_request.head.sha || github.sha }} + context: e2e/${{ matrix.platform.platform }} + description: E2E tests for Mattermost desktop app on ${{ matrix.platform.platform }} have started... + status: pending - update-final-status-macos: - runs-on: ubuntu-22.04 - needs: - - e2e-tests - steps: - - uses: mattermost/actions/delivery/update-commit-status@main + - name: Run E2E tests + id: run-tests + uses: ./.github/workflows/e2e-functional-template.yml + with: + runs-on: ${{ matrix.platform.os }} + DESKTOP_VERSION: ${{ inputs.version_name || github.head_ref || github.ref }} + MM_TEST_SERVER_URL: ${{ matrix.platform.url }} + MM_TEST_USER_NAME: ${{ inputs.MM_TEST_USER_NAME || secrets.MM_DESKTOP_E2E_USER_NAME }} + MM_TEST_PASSWORD: ${{ inputs.MM_TEST_PASSWORD || secrets.MM_DESKTOP_E2E_USER_CREDENTIALS }} + MM_SERVER_VERSION: ${{ inputs.MM_SERVER_VERSION || '10.5.0' }} + + - name: Update final commit status + if: always() + uses: mattermost/actions/delivery/update-commit-status@main env: GITHUB_TOKEN: ${{ github.token }} with: repository_full_name: ${{ github.repository }} - commit_sha: ${{ github.event.pull_request.head.sha }} - context: e2e/macOS - description: Completed with ${{ needs.e2e-tests.outputs.NEW_FAILURES_MACOS }} failures - status: ${{ needs.e2e-tests.outputs.STATUS_MACOS }} - target_url: ${{ needs.e2e-tests.outputs.REPORT_LINK_MACOS }} - - update-final-status-windows: - runs-on: ubuntu-22.04 - needs: - - e2e-tests - steps: - - uses: mattermost/actions/delivery/update-commit-status@main - env: - GITHUB_TOKEN: ${{ github.token }} - with: - repository_full_name: ${{ github.repository }} - commit_sha: ${{ github.event.pull_request.head.sha }} - context: e2e/windows - description: Completed with ${{ needs.e2e-tests.outputs.NEW_FAILURES_WINDOWS }} failures - status: ${{ needs.e2e-tests.outputs.STATUS_WINDOWS }} - target_url: ${{ needs.e2e-tests.outputs.REPORT_LINK_WINDOWS }} - - e2e-remove-label: - if: ${{ always() && contains(github.event.pull_request.labels.*.name, 'Run Desktop E2E Tests') }} - needs: e2e-tests - runs-on: ubuntu-22.04 - steps: - - name: Remove E2E test label - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 - continue-on-error: true # Label might have been removed manually - with: - script: | - github.rest.issues.removeLabel({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - name: 'Run Desktop E2E Tests', - }); + commit_sha: ${{ github.event.pull_request.head.sha || github.sha }} + context: e2e/${{ matrix.platform.platform }} + description: Completed with ${{ steps.run-tests.outputs.NEW_FAILURES }} failures + status: ${{ steps.run-tests.outputs.STATUS }} + target_url: ${{ steps.run-tests.outputs.REPORT_LINK }} diff --git a/e2e/modules/environment.js b/e2e/modules/environment.js index 1455ec79..d235aee4 100644 --- a/e2e/modules/environment.js +++ b/e2e/modules/environment.js @@ -259,23 +259,7 @@ module.exports = { await window.waitForSelector('#input_password-input'); await window.waitForSelector('#saveSetting'); - let username = process.env.MM_TEST_USERNAME; - switch (process.platform) { - case 'darwin': - username = 'success+sysadmin+macos@simulator.amazonses.com'; - break; - case 'linux': - username = 'success+sysadmin+linux@simulator.amazonses.com'; - break; - case 'win32': - username = 'success+sysadmin+windows@simulator.amazonses.com'; - break; - default: - throw new Error('Unsupported platform'); - } - - await window.type('#input_loginId', username); - + await window.type('#input_loginId', process.env.MM_TEST_USER_NAME); await window.type('#input_password-input', process.env.MM_TEST_PASSWORD); await window.click('#saveSetting'); },