diff --git a/.github/actions/install-os-dependencies/action.yaml b/.github/actions/install-os-dependencies/action.yaml new file mode 100644 index 00000000..d24b240f --- /dev/null +++ b/.github/actions/install-os-dependencies/action.yaml @@ -0,0 +1,56 @@ +name: 'Install OS Dependencies' +description: 'Installs OS-specific dependencies for E2E tests' + +inputs: + os: + description: 'Operating system (Linux, macOS, Windows)' + required: true + +runs: + using: "composite" + steps: + - name: Install Linux Dependencies + if: inputs.os == 'Linux' + shell: bash + run: | + wget -qO - https://download.opensuse.org/repositories/Emulators:/Wine:/Debian/xUbuntu_22.04/Release.key | sudo apt-key add - + wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/v4.20.1/yq_linux_amd64 && chmod a+x /usr/local/bin/yq + sudo apt-get update || true + sudo apt-get install -y \ + ca-certificates \ + libxtst-dev \ + libpng++-dev \ + gcc-aarch64-linux-gnu \ + g++-aarch64-linux-gnu \ + jq \ + icnsutils \ + graphicsmagick \ + tzdata \ + xsel \ + xvfb \ + libgtk-3-0 \ + libnss3 \ + libxss1 \ + libasound2 + + - name: Install macOS Dependencies + if: inputs.os == 'macOS' + shell: bash + run: | + brew install nss + jq '.mac.target=["zip"]' electron-builder.json | \ + jq '.mac.gatekeeperAssess=false' > /tmp/electron-builder.json && \ + cp /tmp/electron-builder.json . + + - name: Install Windows Dependencies + if: inputs.os == 'Windows' + shell: bash + run: | + choco install yq --version 4.15.1 -y + choco install vcredist-all -y + npm i -g node-gyp + ELECTRON_VERSION=$(jq -r .devDependencies.electron package.json) + node-gyp install + node-gyp install --devdir="C:\Users\runneradmin\.electron-gyp" --target=$ELECTRON_VERSION --dist-url="https://electronjs.org/headers" + node-gyp install --devdir="C:\Users\runneradmin\.electron-gyp" --target=$ELECTRON_VERSION --dist-url="https://electronjs.org/headers" --arch arm64 + npm install mochawesome-report-generator diff --git a/.github/workflows/e2e-functional-template.yml b/.github/workflows/e2e-functional-template.yml index 0bf2f6bf..b8de2b71 100644 --- a/.github/workflows/e2e-functional-template.yml +++ b/.github/workflows/e2e-functional-template.yml @@ -35,14 +35,87 @@ on: description: "True if this is Compatibility Matrix Testing" required: false default: false + TYPE: + type: boolean + description: "True if this is Triggered from PR" + required: false + default: false MM_SERVER_VERSION: type: string required: false default: "9.9.1" outputs: - COMMENT_BODY: + NEW_FAILURES_LINUX: description: "The output to comment" - value: ${{ jobs.e2e.outputs.COMMENT_BODY }} + value: ${{ jobs.e2e.outputs.NEW_FAILURES_LINUX }} + NEW_FAILURES_MACOS: + description: "The output to comment" + value: ${{ jobs.e2e.outputs.NEW_FAILURES_MACOS }} + NEW_FAILURES_WINDOWS: + description: "The output to comment" + value: ${{ jobs.e2e.outputs.NEW_FAILURES_WINDOWS }} + REPORT_LINK_LINUX: + description: "Link to Linux report" + value: ${{ jobs.e2e.outputs.REPORT_LINK_LINUX }} + REPORT_LINK_MACOS: + description: "Link to MacOS report" + value: ${{ jobs.e2e.outputs.REPORT_LINK_MACOS }} + REPORT_LINK_WINDOWS: + description: "Link to Windows report" + value: ${{ jobs.e2e.outputs.REPORT_LINK_WINDOWS }} + STATUS_LINUX: + description: "The status of the linux test" + value: ${{ jobs.e2e.outputs.STATUS_LINUX }} + STATUS_MACOS: + description: "The status of the macOS test" + value: ${{ jobs.e2e.outputs.STATUS_MACOS }} + STATUS_WINDOWS: + description: "The status of the windows test" + value: ${{ jobs.e2e.outputs.STATUS_WINDOWS }} + + workflow_dispatch: + inputs: + MM_TEST_SERVER_URL: + description: "The test server URL" + required: false + type: string + MM_TEST_USER_NAME: + description: "The admin username of the test instance" + required: false + type: string + MM_TEST_PASSWORD: + description: "The admin password of the test instance" + required: false + type: string + DESKTOP_VERSION: + description: "The desktop version to test" + required: false + default: "master" + type: string + runs-on: + type: string + description: "The E2E tests underlying OS" + required: true + default: "ubuntu-22.04" + nightly: + type: boolean + description: "True if this is nightly build" + required: false + default: false + cmt: + type: boolean + description: "True if this is Compatibility Matrix Testing" + required: false + default: false + TYPE: + type: boolean + description: "True if this is Triggered from PR" + required: false + default: false + MM_SERVER_VERSION: + type: string + required: false + default: "9.9.1" env: AWS_S3_BUCKET: "mattermost-cypress-report" @@ -62,49 +135,73 @@ env: WEBHOOK_URL: ${{ secrets.MM_DESKTOP_E2E_WEBHOOK_URL }} ZEPHYR_API_KEY: ${{ secrets.MM_DESKTOP_E2E_ZEPHYR_API_KEY }} REPORT_LINK: "none" + NODE_VERSION: 18 + SERVER_VERSION: ${{ inputs.MM_SERVER_VERSION || '9.9.1' }} + DESKTOP_VERSION: ${{ inputs.DESKTOP_VERSION }} jobs: e2e: + name: e2e-on-${{ inputs.runs-on }} runs-on: ${{ inputs.runs-on }} defaults: run: shell: bash outputs: - COMMENT_BODY: ${{ steps.analyze-flaky-tests.outputs.COMMENT_BODY }} + NEW_FAILURES_LINUX: ${{ steps.analyze-flaky-tests.outputs.NEW_FAILURES_LINUX }} + NEW_FAILURES_MACOS: ${{ steps.analyze-flaky-tests.outputs.NEW_FAILURES_MACOS }} + NEW_FAILURES_WINDOWS: ${{ steps.analyze-flaky-tests.outputs.NEW_FAILURES_WINDOWS }} + REPORT_LINK_LINUX: ${{ steps.analyze-flaky-tests.outputs.REPORT_LINK_LINUX }} + REPORT_LINK_MACOS: ${{ steps.analyze-flaky-tests.outputs.REPORT_LINK_MACOS }} + REPORT_LINK_WINDOWS: ${{ steps.analyze-flaky-tests.outputs.REPORT_LINK_WINDOWS }} + STATUS_LINUX: ${{ steps.analyze-flaky-tests.outputs.STATUS_LINUX }} + STATUS_WINDOWS: ${{ steps.analyze-flaky-tests.outputs.STATUS_WINDOWS }} + STATUS_MACOS: ${{ steps.analyze-flaky-tests.outputs.STATUS_MACOS }} steps: - name: e2e/set-required-variables id: variables run: | RUNNER_OS=$(echo "${{ runner.os }}" | tr '[:upper:]' '[:lower:]') - - if [ "${{ github.event_name }}" == "pull_request" ]; then - echo "BUILD_SUFFIX=desktop-pr-${RUNNER_OS}" >> $GITHUB_OUTPUT - echo "TYPE=PR" >> $GITHUB_ENV - - elif [ "${{ github.event_name }}" == "workflow_dispatch" && "${{ inputs.cmt }}" ]; then - echo "BUILD_SUFFIX=desktop-release-${RUNNER_OS}" >> $GITHUB_OUTPUT - echo "TYPE=CMT" >> $GITHUB_ENV - echo "ZEPHYR_ENABLE=true" >> $GITHUB_ENV - echo "ZEPHYR_FOLDER_LINUX_REPORT=12358649" >> $GITHUB_ENV - echo "ZEPHYR_FOLDER_MACOS_REPORT=12358650" >> $GITHUB_ENV - echo "ZEPHYR_FOLDER_WIN_REPORT=12358651" >> $GITHUB_ENV - - elif [ "${{ github.event_name }}" == "workflow_dispatch" ] && ! ${{ inputs.nightly }}; then - echo "BUILD_SUFFIX=desktop-manual-trigger-${RUNNER_OS}" >> $GITHUB_OUTPUT - echo "TYPE=MANUAL" >> $GITHUB_ENV - - elif [ "${{ github.event_name }}" == "push" ] && [ "${{ github.ref }}" == "refs/heads/master" ]; then - echo "BUILD_SUFFIX=desktop-master-push-${RUNNER_OS}" >> $GITHUB_OUTPUT - echo "TYPE=MASTER" >> $GITHUB_ENV - echo "ZEPHYR_ENABLE=true" >> $GITHUB_ENV - - elif ${{ inputs.nightly }}; then + echo "RUNNER_OS=${RUNNER_OS}" >> $GITHUB_ENV + + # Define build type and suffix + case "${{ github.event_name }}" in + "pull_request") + echo "BUILD_SUFFIX=desktop-pr-${RUNNER_OS}" >> $GITHUB_OUTPUT + echo "TYPE=PR" >> $GITHUB_ENV + ;; + "workflow_dispatch") + if ${{ inputs.cmt }}; then + echo "BUILD_SUFFIX=desktop-release-${RUNNER_OS}" >> $GITHUB_OUTPUT + echo "TYPE=CMT" >> $GITHUB_ENV + echo "ZEPHYR_ENABLE=true" >> $GITHUB_ENV + echo "ZEPHYR_FOLDER_${RUNNER_OS^^}_REPORT=${{ + runner.os == 'Linux' && '12358649' || + runner.os == 'macOS' && '12358650' || + '12358651' + }}" >> $GITHUB_ENV + else + echo "BUILD_SUFFIX=desktop-manual-trigger-${RUNNER_OS}" >> $GITHUB_OUTPUT + echo "TYPE=MANUAL" >> $GITHUB_ENV + fi + ;; + "push") + if [[ "${{ github.ref }}" == "refs/heads/master" ]]; then + echo "BUILD_SUFFIX=desktop-master-push-${RUNNER_OS}" >> $GITHUB_OUTPUT + echo "TYPE=MASTER" >> $GITHUB_ENV + echo "ZEPHYR_ENABLE=true" >> $GITHUB_ENV + fi + ;; + esac + + if ${{ inputs.nightly }}; then echo "BUILD_SUFFIX=desktop-nightly-${RUNNER_OS}" >> $GITHUB_OUTPUT echo "TYPE=NIGHTLY" >> $GITHUB_ENV echo "ZEPHYR_ENABLE=true" >> $GITHUB_ENV - echo "ZEPHYR_FOLDER_LINUX_REPORT=12363689" >> $GITHUB_ENV - echo "ZEPHYR_FOLDER_MACOS_REPORT=12363687" >> $GITHUB_ENV - echo "ZEPHYR_FOLDER_WIN_REPORT=12363690" >> $GITHUB_ENV + echo "ZEPHYR_FOLDER_${RUNNER_OS^^}_REPORT=${{ + runner.os == 'Linux' && '12363689' || + runner.os == 'macOS' && '12363687' || + '12363690' + }}" >> $GITHUB_ENV fi - name: e2e/set-build-id @@ -140,52 +237,16 @@ jobs: with: python-version: "3.10" - ## Linux Dependencies - - name: e2e/install-dependencies-linux - if: runner.os == 'Linux' - env: - PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 0 - run: | - wget -qO - https://download.opensuse.org/repositories/Emulators:/Wine:/Debian/xUbuntu_22.04/Release.key | sudo apt-key add - - wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/v4.20.1/yq_linux_amd64 && chmod a+x /usr/local/bin/yq - sudo apt-get update || true && sudo apt-get install -y ca-certificates libxtst-dev libpng++-dev gcc-aarch64-linux-gnu g++-aarch64-linux-gnu jq icnsutils graphicsmagick tzdata xsel libgtk-3-0 libnss3 libxss1 libasound2 - sudo apt-get install -y xvfb - export DISPLAY=:99 - Xvfb :99 -screen 0 1024x768x24 & - npm ci - cd e2e - npm ci - npx electron-rebuild --platform=linux -f -t prod,optional,dev -w robotjs --module-dir ../ + - name: e2e/install-os-dependencies + uses: ./.github/actions/install-os-dependencies + with: + os: ${{ runner.os }} - - name: e2e/install-dependencies-macos - if: runner.os == 'macOS' - env: - PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 + - name: e2e/install-node-dependencies run: | - brew install nss - jq '.mac.target=["zip"]' electron-builder.json | jq '.mac.gatekeeperAssess=false' > /tmp/electron-builder.json && cp /tmp/electron-builder.json . npm ci - cd e2e - npm ci - npx electron-rebuild --platform=darwin -f -t prod,optional,dev -w robotjs --module-dir ../ - - ## Windows Dependencies - - name: e2e/install-dependencies-windows - if: runner.os == 'windows' - env: - PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 - run: | - choco install yq --version 4.15.1 -y - choco install vcredist-all -y - npm i -g node-gyp - node-gyp install - node-gyp install --devdir="C:\Users\runneradmin\.electron-gyp" --target=$(jq -r .devDependencies.electron package.json) --dist-url="https://electronjs.org/headers" - node-gyp install --devdir="C:\Users\runneradmin\.electron-gyp" --target=$(jq -r .devDependencies.electron package.json) --dist-url="https://electronjs.org/headers" --arch arm64 - npm ci --openssl_fips='' - cd e2e - npm ci - npx electron-rebuild --platform=win32 -f -t prod,optional,dev -w robotjs --module-dir ../ - npm install mochawesome-report-generator + cd e2e && npm ci + npx electron-rebuild --platform=${{ env.RUNNER_OS }} -f -t prod,optional,dev -w robotjs --module-dir ../ - name: e2e/run-playright-tests-linux if: runner.os == 'Linux' @@ -224,6 +285,9 @@ jobs: SERVER_VERSION: ${{ inputs.MM_SERVER_VERSION }} DESKTOP_VERSION: ${{ inputs.DESKTOP_VERSION }} + - name: Capture Report Link + run: echo "REPORT_LINK=$(cat e2e/report-link.txt)" >> $GITHUB_ENV + - name: e2e/analyze-flaky-tests id: analyze-flaky-tests uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 @@ -231,8 +295,32 @@ jobs: script: | process.chdir('./e2e'); const { analyzeFlakyTests } = require('./utils/analyze-flaky-test.js'); - const { commentBody, newFailedTests } = analyzeFlakyTests(); - core.setOutput('COMMENT_BODY', commentBody); - if (newFailedTests.length > 0) { - core.setFailed('E2E tests failed.'); + const { newFailedTests, os } = analyzeFlakyTests(); + + switch (os) { + case 'linux': + core.setOutput('NEW_FAILURES_LINUX', newFailedTests.length); + core.setOutput('REPORT_LINK_LINUX', "${{ env.REPORT_LINK}}"); + newFailedTests.length > 0 ? core.setOutput('STATUS_LINUX', 'failure') : core.setOutput('STATUS_LINUX', 'success'); + break; + case 'darwin': + core.setOutput('NEW_FAILURES_MACOS', newFailedTests.length); + core.setOutput('REPORT_LINK_MACOS', "${{ env.REPORT_LINK}}"); + newFailedTests.length > 0 ? core.setOutput('STATUS_MACOS', 'failure') : core.setOutput('STATUS_MACOS', 'success'); + break; + case 'win32': + core.setOutput('NEW_FAILURES_WINDOWS', newFailedTests.length); + core.setOutput('REPORT_LINK_WINDOWS', "${{ env.REPORT_LINK}}"); + newFailedTests.length > 0 ? core.setOutput('STATUS_WINDOWS', 'failure') : core.setOutput('STATUS_WINDOWS', 'success'); + break; + default: + throw new Error(`Unsupported OS: ${os}`); } + + - name: Upload test results + if: always() + uses: actions/upload-artifact@v4 + with: + name: test-results-${{ runner.os }} + path: e2e/mochawesome-report + retention-days: 7 diff --git a/.github/workflows/e2e-functional.yml b/.github/workflows/e2e-functional.yml index 364a4410..850e5687 100644 --- a/.github/workflows/e2e-functional.yml +++ b/.github/workflows/e2e-functional.yml @@ -2,126 +2,159 @@ name: Electron Playwright Tests on: push: - branches: - - master + branches: [master] pull_request: - types: - - labeled + types: [labeled] workflow_dispatch: inputs: version_name: type: string description: "Desktop Version name eg: 5.6" required: true - job_name: + platform: type: choice - description: "Job name" + description: "Choose platform to run tests" required: true - default: "All" - options: - - "e2e-linux" - - "e2e-macos" - - "e2e-windows" - - "All" + default: "all" + options: [linux, macos, windows, all] jobs: - e2e-linux: - if: ${{ - ( - (inputs.job_name == 'e2e-linux' || inputs.job_name == 'All') - && - github.event_name == 'workflow_dispatch' - ) || - ( - github.event_name == 'push' - ) || - ( - github.event_name == 'pull_request' && - github.event.pull_request.labels && - contains(github.event.pull_request.labels.*.name, 'Run Desktop E2E Tests') - ) - }} + update-initial-status-linux: + runs-on: ubuntu-22.04 + if: contains(github.event.label.name, 'Run Desktop E2E Tests') + steps: + - uses: mattermost/actions/delivery/update-commit-status@main + env: + GITHUB_TOKEN: ${{ github.token }} + with: + repository_full_name: ${{ github.repository }} + commit_sha: ${{ github.event.pull_request.head.sha }} + context: e2e/linux + description: E2E tests for Mattermost desktop app on Linux have started... + status: pending + + update-initial-status-macos: + runs-on: ubuntu-22.04 + if: contains(github.event.label.name, 'Run Desktop E2E Tests') + steps: + - uses: mattermost/actions/delivery/update-commit-status@main + env: + GITHUB_TOKEN: ${{ github.token }} + with: + repository_full_name: ${{ github.repository }} + commit_sha: ${{ github.event.pull_request.head.sha }} + context: e2e/macOS + description: E2E tests for Mattermost desktop app on macOS have started... + status: pending + + # TODO: uncomment when https://mattermost.atlassian.net/browse/MM-63397 is fixed + # update-initial-status-windows: + # runs-on: ubuntu-22.04 + # if: contains(github.event.label.name, 'Run Desktop E2E Tests') + # steps: + # - uses: mattermost/actions/delivery/update-commit-status@main + # env: + # GITHUB_TOKEN: ${{ github.token }} + # with: + # repository_full_name: ${{ github.repository }} + # commit_sha: ${{ github.event.pull_request.head.sha }} + # context: e2e/windows + # description: E2E tests for Mattermost desktop app on Windows have started... + # status: pending + + determine-platforms: + if: contains(github.event.label.name, 'Run Desktop E2E Tests') + needs: [update-initial-status-linux, update-initial-status-macos] #, update-initial-status-windows] + runs-on: ubuntu-latest + outputs: + platforms: ${{ steps.set-matrix.outputs.platforms }} + steps: + - id: set-matrix + run: | + if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then + if [[ "${{ inputs.platform }}" == "all" ]]; then + echo "platforms=['ubuntu-22.04', 'macos-13', 'windows-2022']" >> $GITHUB_OUTPUT + else + case "${{ inputs.platform }}" in + linux) echo "platforms=['ubuntu-22.04']" ;; + macos) echo "platforms=['macos-13']" ;; + # windows) echo "platforms=['windows-2022']" ;; TODO: uncomment when https://mattermost.atlassian.net/browse/MM-63397 is fixed + esac >> $GITHUB_OUTPUT + fi + elif [[ "${{ github.event_name }}" == "push" ]] || [[ "${{ contains(github.event.pull_request.labels.*.name, 'Run Desktop E2E Tests') }}" == "true" ]]; then + echo "platforms=['ubuntu-22.04', 'macos-13']" >> $GITHUB_OUTPUT + # echo "platforms=['ubuntu-22.04', 'macos-13', 'windows-2022']" >> $GITHUB_OUTPUT + else + echo "platforms=[]" >> $GITHUB_OUTPUT + fi + + e2e-tests: + needs: determine-platforms + strategy: + matrix: + os: ${{ fromJson(needs.determine-platforms.outputs.platforms) }} + fail-fast: false uses: ./.github/workflows/e2e-functional-template.yml secrets: inherit with: - runs-on: ubuntu-22.04 + runs-on: ${{ matrix.os }} DESKTOP_VERSION: ${{ inputs.version_name || github.head_ref || github.ref }} - e2e-macos: - if: ${{ - ( - (inputs.job_name == 'e2e-macos' || inputs.job_name == 'All') - && - github.event_name == 'workflow_dispatch' - ) || - ( - github.event_name == 'push' - ) || - ( - github.event_name == 'pull_request' && - github.event.pull_request.labels && - contains(github.event.pull_request.labels.*.name, 'Run Desktop E2E Tests') - ) - }} - uses: ./.github/workflows/e2e-functional-template.yml - secrets: inherit - with: - runs-on: macos-13 - DESKTOP_VERSION: ${{ inputs.version_name || github.head_ref || github.ref }} + update-final-status-linux: + runs-on: ubuntu-22.04 + needs: + - e2e-tests + steps: + - uses: mattermost/actions/delivery/update-commit-status@main + env: + GITHUB_TOKEN: ${{ github.token }} + with: + repository_full_name: ${{ github.repository }} + commit_sha: ${{ github.event.pull_request.head.sha }} + context: e2e/linux + description: Completed with ${{ needs.e2e-tests.outputs.NEW_FAILURES_LINUX }} failures + status: ${{ needs.e2e-tests.outputs.STATUS_LINUX }} + target_url: ${{ needs.e2e-tests.outputs.REPORT_LINK_LINUX }} - # TODO: Uncomment when https://mattermost.atlassian.net/browse/MM-63397 is resolved - # e2e-windows: - # if: ${{ - # ( - # (inputs.job_name == 'e2e-windows' || inputs.job_name == 'All') - # && - # github.event_name == 'workflow_dispatch' - # ) || - # ( - # github.event_name == 'push' - # ) || - # ( - # github.event_name == 'pull_request' && - # github.event.pull_request.labels && - # contains(github.event.pull_request.labels.*.name, 'Run Desktop E2E Tests') - # ) - # }} - # uses: ./.github/workflows/e2e-functional-template.yml - # secrets: inherit - # with: - # runs-on: windows-2022 - # DESKTOP_VERSION: ${{ inputs.version_name || github.head_ref || github.ref }} + update-final-status-macos: + runs-on: ubuntu-22.04 + needs: + - e2e-tests + steps: + - uses: mattermost/actions/delivery/update-commit-status@main + env: + GITHUB_TOKEN: ${{ github.token }} + with: + repository_full_name: ${{ github.repository }} + commit_sha: ${{ github.event.pull_request.head.sha }} + context: e2e/macOS + description: Completed with ${{ needs.e2e-tests.outputs.NEW_FAILURES_MACOS }} failures + status: ${{ needs.e2e-tests.outputs.STATUS_MACOS }} + target_url: ${{ needs.e2e-tests.outputs.REPORT_LINK_MACOS }} + + # TODO: uncomment when https://mattermost.atlassian.net/browse/MM-63397 is fixed + # update-final-status-windows: + # runs-on: ubuntu-22.04 + # needs: + # - e2e-tests + # steps: + # - uses: mattermost/actions/delivery/update-commit-status@main + # env: + # GITHUB_TOKEN: ${{ github.token }} + # with: + # repository_full_name: ${{ github.repository }} + # commit_sha: ${{ github.event.pull_request.head.sha }} + # context: e2e/windows + # description: Completed with ${{ needs.e2e-tests.outputs.NEW_FAILURES_WINDOWS }} failures + # status: ${{ needs.e2e-tests.outputs.STATUS_WINDOWS }} + # target_url: ${{ needs.e2e-tests.outputs.REPORT_LINK_WINDOWS }} e2e-remove-label: if: ${{ always() && contains(github.event.pull_request.labels.*.name, 'Run Desktop E2E Tests') }} - needs: [e2e-linux, e2e-macos] #, e2e-windows] TODO: Uncomment when https://mattermost.atlassian.net/browse/MM-63397 is resolved + needs: e2e-tests runs-on: ubuntu-22.04 - env: - COMMENT_BODY_LINUX: "${{ needs.e2e-linux.outputs.COMMENT_BODY }}" - COMMENT_BODY_MACOS: "${{ needs.e2e-macos.outputs.COMMENT_BODY }}" - # COMMENT_BODY_WINDOWS: "${{ needs.e2e-windows.outputs.COMMENT_BODY }}" steps: - - name: e2e/unify-comments-in-single-comment - run: | - echo "PR_COMMENT<> "${GITHUB_ENV}" - echo "Here are the test results below:" >> "${GITHUB_ENV}" - echo "$COMMENT_BODY_LINUX" >> "${GITHUB_ENV}" - echo "$COMMENT_BODY_MACOS" >> "${GITHUB_ENV}" - echo "$COMMENT_BODY_WINDOWS" >> "${GITHUB_ENV}" - echo "EOF" >> "${GITHUB_ENV}" - - - name: e2e/send-comment-results-in-pr - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 - with: - script: | - github.rest.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: process.env.PR_COMMENT, - }); - - - name: e2e/remove-label-from-pr + - name: Remove E2E test label uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 continue-on-error: true # Label might have been removed manually with: diff --git a/e2e/modules/environment.js b/e2e/modules/environment.js index 63555bdc..990ed686 100644 --- a/e2e/modules/environment.js +++ b/e2e/modules/environment.js @@ -268,23 +268,22 @@ module.exports = { await window.waitForSelector('#saveSetting'); let username = process.env.MM_TEST_USERNAME; - if (!username) { - switch (process.platform) { - case 'darwin': - username = 'success+sysadmin+macos@simulator.amazonses.com'; - break; - case 'linux': - username = 'success+sysadmin+linux@simulator.amazonses.com'; - break; - case 'win32': - username = 'success+sysadmin+windows@simulator.amazonses.com'; - break; - default: - throw new Error('Unsupported platform'); - } + switch (process.platform) { + case 'darwin': + username = 'success+sysadmin+macos@simulator.amazonses.com'; + break; + case 'linux': + username = 'success+sysadmin+linux@simulator.amazonses.com'; + break; + case 'win32': + username = 'success+sysadmin+windows@simulator.amazonses.com'; + break; + default: + throw new Error('Unsupported platform'); } await window.type('#input_loginId', username); + await window.type('#input_password-input', process.env.MM_TEST_PASSWORD); await window.click('#saveSetting'); }, diff --git a/e2e/save_report.js b/e2e/save_report.js index 35478a1b..07665ff2 100644 --- a/e2e/save_report.js +++ b/e2e/save_report.js @@ -27,6 +27,7 @@ * - TYPE=[type], e.g. "MASTER", "PR", "RELEASE", "CLOUD" */ +const fs = require('fs'); const path = require('path'); const generator = require('mochawesome-report-generator'); @@ -79,6 +80,9 @@ const saveReport = async () => { const result = await saveArtifacts(); if (result && result.success) { console.log('Successfully uploaded artifacts to S3:', result.reportLink); + + // save the report link to a file For CI to use + fs.writeFileSync('report-link.txt', result.reportLink); } // Create or use an existing test cycle @@ -98,8 +102,6 @@ const saveReport = async () => { if (ZEPHYR_ENABLE === 'true') { await createTestExecutions(jsonReport, testCycle); } - - // chai.expect(Boolean(jsonReport.stats.failures), FAILURE_MESSAGE).to.be.false; }; saveReport(); diff --git a/e2e/specs/mattermost/copy_link.test.js b/e2e/specs/mattermost/copy_link.test.js index 4b6a651c..72a93dad 100644 --- a/e2e/specs/mattermost/copy_link.test.js +++ b/e2e/specs/mattermost/copy_link.test.js @@ -32,29 +32,28 @@ describe('copylink', function desc() { await env.clearElectronInstances(); }); - it('MM-T125 Copy Link can be used from channel LHS', async () => { - const firstServer = this.serverMap[`${config.teams[0].name}___TAB_MESSAGING`].win; - await env.loginToMattermost(firstServer); - await asyncSleep(2000); - await firstServer.waitForSelector('#sidebarItem_town-square', {timeout: 5000}); - await firstServer.click('#sidebarItem_town-square', {button: 'right'}); - await asyncSleep(2000); - switch (process.platform) { - case 'linux': - robot.keyTap('c'); - break; - case 'win32': - robot.keyTap('down'); - robot.keyTap('down'); - break; - case 'darwin': - robot.keyTap('c'); - break; - } - robot.keyTap('enter'); - await firstServer.click('#sidebarItem_town-square'); - await firstServer.click('#post_textbox'); - const clipboardText = clipboard.readText(); - clipboardText.should.contain('/channels/town-square'); - }); + if (process.platform !== 'linux') { + it('MM-T125 Copy Link can be used from channel LHS', async () => { + const firstServer = this.serverMap[`${config.teams[0].name}___TAB_MESSAGING`].win; + await env.loginToMattermost(firstServer); + await asyncSleep(2000); + await firstServer.waitForSelector('#sidebarItem_town-square', {timeout: 5000}); + await firstServer.click('#sidebarItem_town-square', {button: 'right'}); + await asyncSleep(2000); + switch (process.platform) { + case 'win32': + robot.keyTap('down'); + robot.keyTap('down'); + break; + case 'darwin': + robot.keyTap('c'); + break; + } + robot.keyTap('enter'); + await firstServer.click('#sidebarItem_town-square'); + await firstServer.click('#post_textbox'); + const clipboardText = clipboard.readText(); + clipboardText.should.contain('/channels/town-square'); + }); + } }); diff --git a/e2e/specs/menu_bar/history_menu.test.js b/e2e/specs/menu_bar/history_menu.test.js index 9c76f17b..a36a1f45 100644 --- a/e2e/specs/menu_bar/history_menu.test.js +++ b/e2e/specs/menu_bar/history_menu.test.js @@ -34,16 +34,27 @@ describe('history_menu', function desc() { await env.loginToMattermost(firstServer); await firstServer.waitForSelector('#sidebarItem_off-topic'); - // click on Off topic channel + // Click on Off-Topic channel await firstServer.click('#sidebarItem_off-topic'); - // click on town square channel + // Click on Town Square channel await firstServer.click('#sidebarItem_town-square'); await firstServer.locator('[aria-label="Back"]').click(); - let channelHeaderText = await firstServer.$eval('#channelHeaderTitle', (el) => el.firstChild.innerHTML); + + // Wait for navigation + await firstServer.waitForSelector('#channelHeaderTitle'); + + // Get channel header text + let channelHeaderText = await firstServer.$eval('#channelHeaderTitle', (el) => el.textContent.trim()); channelHeaderText.should.equal('Off-Topic'); + await firstServer.locator('[aria-label="Forward"]').click(); - channelHeaderText = await firstServer.$eval('#channelHeaderTitle', (el) => el.firstChild.innerHTML); + await asyncSleep(3000); + + // Wait for navigation + await firstServer.waitForSelector('#channelHeaderTitle'); + + channelHeaderText = await firstServer.$eval('#channelHeaderTitle', (el) => el.textContent.trim()); channelHeaderText.should.equal('Town Square'); }); }); diff --git a/e2e/specs/menu_bar/view_menu.test.js b/e2e/specs/menu_bar/view_menu.test.js index 3032fcc6..8f28aae4 100644 --- a/e2e/specs/menu_bar/view_menu.test.js +++ b/e2e/specs/menu_bar/view_menu.test.js @@ -187,31 +187,33 @@ describe('menu/view', function desc() { }); }); - it('MM-T820 should open Developer Tools For Application Wrapper for main window', async () => { - const mainWindow = this.app.windows().find((window) => window.url().includes('index.html')); - const browserWindow = await this.app.browserWindow(mainWindow); + if (process.platform !== 'linux') { + it('MM-T820 should open Developer Tools For Application Wrapper for main window', async () => { + const mainWindow = this.app.windows().find((window) => window.url().includes('index.html')); + const browserWindow = await this.app.browserWindow(mainWindow); - let isDevToolsOpen = await browserWindow.evaluate((window) => { - return window.webContents.isDevToolsOpened(); + let isDevToolsOpen = await browserWindow.evaluate((window) => { + return window.webContents.isDevToolsOpened(); + }); + isDevToolsOpen.should.be.false; + + if (process.platform === 'darwin') { + // Press Command + Option + I + robot.keyTap('i', ['command', 'alt']); + await asyncSleep(3000); + } + + if (process.platform === 'win32') { + robot.keyToggle('shift', 'down'); + robot.keyToggle('control', 'down'); + robot.keyTap('i'); + } + + await asyncSleep(1000); + isDevToolsOpen = await browserWindow.evaluate((window) => { + return window.webContents.isDevToolsOpened(); + }); + isDevToolsOpen.should.be.true; }); - isDevToolsOpen.should.be.false; - - if (process.platform === 'darwin') { - // Press Command + Option + I - robot.keyTap('i', ['command', 'alt']); - await asyncSleep(3000); - } - - if (process.platform === 'win32') { - robot.keyToggle('shift', 'down'); - robot.keyToggle('control', 'down'); - robot.keyTap('i'); - } - - await asyncSleep(1000); - isDevToolsOpen = await browserWindow.evaluate((window) => { - return window.webContents.isDevToolsOpened(); - }); - isDevToolsOpen.should.be.true; - }); + } }); diff --git a/e2e/specs/menu_bar/window_menu.test.js b/e2e/specs/menu_bar/window_menu.test.js index 6b02dd8e..3d0f8b90 100644 --- a/e2e/specs/menu_bar/window_menu.test.js +++ b/e2e/specs/menu_bar/window_menu.test.js @@ -147,39 +147,42 @@ describe('Menu/window_menu', function desc() { await afterFunc(); }); - it('MM-T824 should be minimized when keyboard shortcuts are pressed', async () => { - await beforeFunc(); + if (process.platform !== 'linux') { + it('MM-T824 should be minimized when keyboard shortcuts are pressed', async () => { + await beforeFunc(); - const mainWindow = this.app.windows().find((window) => window.url().includes('index')); - const browserWindow = await this.app.browserWindow(mainWindow); - if (process.platform === 'darwin') { - robot.keyTap('m', [env.cmdOrCtrl]); - } else { - await mainWindow.click('button.three-dot-menu'); - robot.keyTap('w'); - robot.keyTap('m'); - robot.keyTap('enter'); - } + const mainWindow = this.app.windows().find((window) => window.url().includes('index')); + const browserWindow = await this.app.browserWindow(mainWindow); + if (process.platform === 'darwin') { + robot.keyTap('m', [env.cmdOrCtrl]); + } else { + await mainWindow.click('button.three-dot-menu'); + robot.keyTap('w'); + robot.keyTap('m'); + robot.keyTap('enter'); + } - await asyncSleep(2000); - const isMinimized = await browserWindow.evaluate((window) => window.isMinimized()); - isMinimized.should.be.true; + await asyncSleep(2000); + const isMinimized = await browserWindow.evaluate((window) => window.isMinimized()); + isMinimized.should.be.true; - await afterFunc(); - }); + await afterFunc(); + }); + } + if (process.platform !== 'linux') { + it('MM-T825 should be hidden when keyboard shortcuts are pressed', async () => { + await beforeFunc(); - it('MM-T825 should be hidden when keyboard shortcuts are pressed', async () => { - await beforeFunc(); + const mainWindow = this.app.windows().find((window) => window.url().includes('index')); + const browserWindow = await this.app.browserWindow(mainWindow); + robot.keyTap('w', [env.cmdOrCtrl]); + await asyncSleep(2000); + const isVisible = await browserWindow.evaluate((window) => window.isVisible()); + isVisible.should.be.false; + const isDestroyed = await browserWindow.evaluate((window) => window.isDestroyed()); + isDestroyed.should.be.false; - const mainWindow = this.app.windows().find((window) => window.url().includes('index')); - const browserWindow = await this.app.browserWindow(mainWindow); - robot.keyTap('w', [env.cmdOrCtrl]); - await asyncSleep(2000); - const isVisible = await browserWindow.evaluate((window) => window.isVisible()); - isVisible.should.be.false; - const isDestroyed = await browserWindow.evaluate((window) => window.isDestroyed()); - isDestroyed.should.be.false; - - await afterFunc(); - }); + await afterFunc(); + }); + } }); diff --git a/e2e/specs/server_management/header.test.js b/e2e/specs/server_management/header.test.js index 8167f812..6554930a 100644 --- a/e2e/specs/server_management/header.test.js +++ b/e2e/specs/server_management/header.test.js @@ -37,13 +37,15 @@ describe('header', function desc() { await env.clearElectronInstances(); }); - it('MM-T2637_1 should maximize on double-clicking the header', async () => { - const headerBounds = await header.boundingBox(); - await header.dblclick({position: {x: headerBounds.width / 2, y: headerBounds.y / 2}}); - await asyncSleep(1000); - const isMaximized = await browserWindow.evaluate((window) => window.isMaximized()); - isMaximized.should.be.equal(true); - }); + if (process.platform !== 'linux') { + it('MM-T2637_1 should maximize on double-clicking the header', async () => { + const headerBounds = await header.boundingBox(); + await header.dblclick({position: {x: headerBounds.width / 2, y: headerBounds.y / 2}}); + await asyncSleep(1000); + const isMaximized = await browserWindow.evaluate((window) => window.isMaximized()); + isMaximized.should.be.equal(true); + }); + } it('MM-T2637_2 should restore on double-clicking the header when maximized', async () => { const maximizedHeaderBounds = await header.boundingBox(); diff --git a/e2e/utils/analyze-flaky-test.js b/e2e/utils/analyze-flaky-test.js index 68bf7ca0..1c9b757d 100644 --- a/e2e/utils/analyze-flaky-test.js +++ b/e2e/utils/analyze-flaky-test.js @@ -24,53 +24,13 @@ function analyzeFlakyTests() { // Filter out the known flaky tests from the failed test titles const newFailedTests = failedFullTitles.filter((test) => !knownFlakyTestsForOS.has(test)); - // Check if any known failed tests are fixed - const fixedTests = [...knownFlakyTestsForOS].filter((test) => !failedFullTitles.includes(test)); - - const commentBody = generateCommentBodyFunctionalTest(newFailedTests, fixedTests); - - // Print on CI - console.log(commentBody); - - return {commentBody, newFailedTests}; + return {newFailedTests, os}; } catch (error) { console.error('Error analyzing failures:', error); return {}; } } -function generateCommentBodyFunctionalTest(newFailedTests, fixedTests) { - const osName = process.env.RUNNER_OS; - const build = process.env.BUILD_TAG; - - let commentBody = ` -## Test Summary for ${osName} on commit ${build} - `; - - if (newFailedTests.length === 0 && fixedTests.length === 0) { - commentBody += ` -All stable tests passed on ${osName}. - `; - return commentBody; - } - - if (newFailedTests.length > 0) { - const newTestFailure = `New failed tests found on ${osName}:\n${newFailedTests.map((test) => `- ${test}`).join('\n')}`; - commentBody += ` -${newTestFailure} - `; - } - - if (fixedTests.length > 0) { - const fixedTestMessage = `The following known failed tests have been fixed on ${osName}:\n\t${fixedTests.map((test) => `- ${test}`).join('\n\t')}`; - commentBody += ` -${fixedTestMessage} - `; - } - - return commentBody; -} - module.exports = { analyzeFlakyTests, };