phlux/mattermostest
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Spin server when Tests start (#3316)

Browse Source
This commit is contained in:
yasser khan
2025-03-26 23:43:10 +05:30
committed by GitHub
parent 081d9c5ad5
commit aa4ee31da3
11 changed files with 477 additions and 322 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
.github/actions/install-os-dependencies/action.yaml vendored Normal file
View File

@@ -0,0 +1,56 @@
name: 'Install OS Dependencies'
description: 'Installs OS-specific dependencies for E2E tests'
inputs:
os:
description: 'Operating system (Linux, macOS, Windows)'
required: true
runs:
using: "composite"
steps:
- name: Install Linux Dependencies
if: inputs.os == 'Linux'
shell: bash
run: |
wget -qO - https://download.opensuse.org/repositories/Emulators:/Wine:/Debian/xUbuntu_22.04/Release.key | sudo apt-key add -
wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/v4.20.1/yq_linux_amd64 && chmod a+x /usr/local/bin/yq
sudo apt-get update || true
sudo apt-get install -y \
ca-certificates \
libxtst-dev \
libpng++-dev \
gcc-aarch64-linux-gnu \
g++-aarch64-linux-gnu \
jq \
icnsutils \
graphicsmagick \
tzdata \
xsel \
xvfb \
libgtk-3-0 \
libnss3 \
libxss1 \
libasound2
- name: Install macOS Dependencies
if: inputs.os == 'macOS'
shell: bash
run: |
brew install nss
jq '.mac.target=["zip"]' electron-builder.json | \
jq '.mac.gatekeeperAssess=false' > /tmp/electron-builder.json && \
cp /tmp/electron-builder.json .
- name: Install Windows Dependencies
if: inputs.os == 'Windows'
shell: bash
run: |
choco install yq --version 4.15.1 -y
choco install vcredist-all -y
npm i -g node-gyp
ELECTRON_VERSION=$(jq -r .devDependencies.electron package.json)
node-gyp install
node-gyp install --devdir="C:\Users\runneradmin\.electron-gyp" --target=$ELECTRON_VERSION --dist-url="https://electronjs.org/headers"
node-gyp install --devdir="C:\Users\runneradmin\.electron-gyp" --target=$ELECTRON_VERSION --dist-url="https://electronjs.org/headers" --arch arm64
npm install mochawesome-report-generator

240
.github/workflows/e2e-functional-template.yml vendored
View File

@@ -35,14 +35,87 @@ on:
description: "True if this is Compatibility Matrix Testing" description: "True if this is Compatibility Matrix Testing"
required: false required: false
default: false default: false
TYPE:
type: boolean
description: "True if this is Triggered from PR"
required: false
default: false
MM_SERVER_VERSION: MM_SERVER_VERSION:
type: string type: string
required: false required: false
default: "9.9.1" default: "9.9.1"
outputs: outputs:
COMMENT_BODY: NEW_FAILURES_LINUX:
description: "The output to comment" description: "The output to comment"
value: ${{ jobs.e2e.outputs.COMMENT_BODY }} value: ${{ jobs.e2e.outputs.NEW_FAILURES_LINUX }}
NEW_FAILURES_MACOS:
description: "The output to comment"
value: ${{ jobs.e2e.outputs.NEW_FAILURES_MACOS }}
NEW_FAILURES_WINDOWS:
description: "The output to comment"
value: ${{ jobs.e2e.outputs.NEW_FAILURES_WINDOWS }}
REPORT_LINK_LINUX:
description: "Link to Linux report"
value: ${{ jobs.e2e.outputs.REPORT_LINK_LINUX }}
REPORT_LINK_MACOS:
description: "Link to MacOS report"
value: ${{ jobs.e2e.outputs.REPORT_LINK_MACOS }}
REPORT_LINK_WINDOWS:
description: "Link to Windows report"
value: ${{ jobs.e2e.outputs.REPORT_LINK_WINDOWS }}
STATUS_LINUX:
description: "The status of the linux test"
value: ${{ jobs.e2e.outputs.STATUS_LINUX }}
STATUS_MACOS:
description: "The status of the macOS test"
value: ${{ jobs.e2e.outputs.STATUS_MACOS }}
STATUS_WINDOWS:
description: "The status of the windows test"
value: ${{ jobs.e2e.outputs.STATUS_WINDOWS }}
workflow_dispatch:
inputs:
MM_TEST_SERVER_URL:
description: "The test server URL"
required: false
type: string
MM_TEST_USER_NAME:
description: "The admin username of the test instance"
required: false
type: string
MM_TEST_PASSWORD:
description: "The admin password of the test instance"
required: false
type: string
DESKTOP_VERSION:
description: "The desktop version to test"
required: false
default: "master"
type: string
runs-on:
type: string
description: "The E2E tests underlying OS"
required: true
default: "ubuntu-22.04"
nightly:
type: boolean
description: "True if this is nightly build"
required: false
default: false
cmt:
type: boolean
description: "True if this is Compatibility Matrix Testing"
required: false
default: false
TYPE:
type: boolean
description: "True if this is Triggered from PR"
required: false
default: false
MM_SERVER_VERSION:
type: string
required: false
default: "9.9.1"
env: env:
AWS_S3_BUCKET: "mattermost-cypress-report" AWS_S3_BUCKET: "mattermost-cypress-report"
@@ -62,49 +135,73 @@ env:
WEBHOOK_URL: ${{ secrets.MM_DESKTOP_E2E_WEBHOOK_URL }} WEBHOOK_URL: ${{ secrets.MM_DESKTOP_E2E_WEBHOOK_URL }}
ZEPHYR_API_KEY: ${{ secrets.MM_DESKTOP_E2E_ZEPHYR_API_KEY }} ZEPHYR_API_KEY: ${{ secrets.MM_DESKTOP_E2E_ZEPHYR_API_KEY }}
REPORT_LINK: "none" REPORT_LINK: "none"
NODE_VERSION: 18
SERVER_VERSION: ${{ inputs.MM_SERVER_VERSION || '9.9.1' }}
DESKTOP_VERSION: ${{ inputs.DESKTOP_VERSION }}
jobs: jobs:
e2e: e2e:
name: e2e-on-${{ inputs.runs-on }}
runs-on: ${{ inputs.runs-on }} runs-on: ${{ inputs.runs-on }}
defaults: defaults:
run: run:
shell: bash shell: bash
outputs: outputs:
COMMENT_BODY: ${{ steps.analyze-flaky-tests.outputs.COMMENT_BODY }} NEW_FAILURES_LINUX: ${{ steps.analyze-flaky-tests.outputs.NEW_FAILURES_LINUX }}
NEW_FAILURES_MACOS: ${{ steps.analyze-flaky-tests.outputs.NEW_FAILURES_MACOS }}
NEW_FAILURES_WINDOWS: ${{ steps.analyze-flaky-tests.outputs.NEW_FAILURES_WINDOWS }}
REPORT_LINK_LINUX: ${{ steps.analyze-flaky-tests.outputs.REPORT_LINK_LINUX }}
REPORT_LINK_MACOS: ${{ steps.analyze-flaky-tests.outputs.REPORT_LINK_MACOS }}
REPORT_LINK_WINDOWS: ${{ steps.analyze-flaky-tests.outputs.REPORT_LINK_WINDOWS }}
STATUS_LINUX: ${{ steps.analyze-flaky-tests.outputs.STATUS_LINUX }}
STATUS_WINDOWS: ${{ steps.analyze-flaky-tests.outputs.STATUS_WINDOWS }}
STATUS_MACOS: ${{ steps.analyze-flaky-tests.outputs.STATUS_MACOS }}
steps: steps:
- name: e2e/set-required-variables - name: e2e/set-required-variables
id: variables id: variables
run: | run: |
RUNNER_OS=$(echo "${{ runner.os }}" | tr '[:upper:]' '[:lower:]') RUNNER_OS=$(echo "${{ runner.os }}" | tr '[:upper:]' '[:lower:]')
echo "RUNNER_OS=${RUNNER_OS}" >> $GITHUB_ENV
if [ "${{ github.event_name }}" == "pull_request" ]; then
echo "BUILD_SUFFIX=desktop-pr-${RUNNER_OS}" >> $GITHUB_OUTPUT # Define build type and suffix
echo "TYPE=PR" >> $GITHUB_ENV case "${{ github.event_name }}" in
"pull_request")
elif [ "${{ github.event_name }}" == "workflow_dispatch" && "${{ inputs.cmt }}" ]; then echo "BUILD_SUFFIX=desktop-pr-${RUNNER_OS}" >> $GITHUB_OUTPUT
echo "BUILD_SUFFIX=desktop-release-${RUNNER_OS}" >> $GITHUB_OUTPUT echo "TYPE=PR" >> $GITHUB_ENV
echo "TYPE=CMT" >> $GITHUB_ENV ;;
echo "ZEPHYR_ENABLE=true" >> $GITHUB_ENV "workflow_dispatch")
echo "ZEPHYR_FOLDER_LINUX_REPORT=12358649" >> $GITHUB_ENV if ${{ inputs.cmt }}; then
echo "ZEPHYR_FOLDER_MACOS_REPORT=12358650" >> $GITHUB_ENV echo "BUILD_SUFFIX=desktop-release-${RUNNER_OS}" >> $GITHUB_OUTPUT
echo "ZEPHYR_FOLDER_WIN_REPORT=12358651" >> $GITHUB_ENV echo "TYPE=CMT" >> $GITHUB_ENV
echo "ZEPHYR_ENABLE=true" >> $GITHUB_ENV
elif [ "${{ github.event_name }}" == "workflow_dispatch" ] && ! ${{ inputs.nightly }}; then echo "ZEPHYR_FOLDER_${RUNNER_OS^^}_REPORT=${{
echo "BUILD_SUFFIX=desktop-manual-trigger-${RUNNER_OS}" >> $GITHUB_OUTPUT runner.os == 'Linux' && '12358649' ||
echo "TYPE=MANUAL" >> $GITHUB_ENV runner.os == 'macOS' && '12358650' ||
'12358651'
elif [ "${{ github.event_name }}" == "push" ] && [ "${{ github.ref }}" == "refs/heads/master" ]; then }}" >> $GITHUB_ENV
echo "BUILD_SUFFIX=desktop-master-push-${RUNNER_OS}" >> $GITHUB_OUTPUT else
echo "TYPE=MASTER" >> $GITHUB_ENV echo "BUILD_SUFFIX=desktop-manual-trigger-${RUNNER_OS}" >> $GITHUB_OUTPUT
echo "ZEPHYR_ENABLE=true" >> $GITHUB_ENV echo "TYPE=MANUAL" >> $GITHUB_ENV
fi
elif ${{ inputs.nightly }}; then ;;
"push")
if [[ "${{ github.ref }}" == "refs/heads/master" ]]; then
echo "BUILD_SUFFIX=desktop-master-push-${RUNNER_OS}" >> $GITHUB_OUTPUT
echo "TYPE=MASTER" >> $GITHUB_ENV
echo "ZEPHYR_ENABLE=true" >> $GITHUB_ENV
fi
;;
esac
if ${{ inputs.nightly }}; then
echo "BUILD_SUFFIX=desktop-nightly-${RUNNER_OS}" >> $GITHUB_OUTPUT echo "BUILD_SUFFIX=desktop-nightly-${RUNNER_OS}" >> $GITHUB_OUTPUT
echo "TYPE=NIGHTLY" >> $GITHUB_ENV echo "TYPE=NIGHTLY" >> $GITHUB_ENV
echo "ZEPHYR_ENABLE=true" >> $GITHUB_ENV echo "ZEPHYR_ENABLE=true" >> $GITHUB_ENV
echo "ZEPHYR_FOLDER_LINUX_REPORT=12363689" >> $GITHUB_ENV echo "ZEPHYR_FOLDER_${RUNNER_OS^^}_REPORT=${{
echo "ZEPHYR_FOLDER_MACOS_REPORT=12363687" >> $GITHUB_ENV runner.os == 'Linux' && '12363689' ||
echo "ZEPHYR_FOLDER_WIN_REPORT=12363690" >> $GITHUB_ENV runner.os == 'macOS' && '12363687' ||
'12363690'
}}" >> $GITHUB_ENV
fi fi
- name: e2e/set-build-id - name: e2e/set-build-id
@@ -140,52 +237,16 @@ jobs:
with: with:
python-version: "3.10" python-version: "3.10"
## Linux Dependencies - name: e2e/install-os-dependencies
- name: e2e/install-dependencies-linux uses: ./.github/actions/install-os-dependencies
if: runner.os == 'Linux' with:
env: os: ${{ runner.os }}
PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 0
run: |
wget -qO - https://download.opensuse.org/repositories/Emulators:/Wine:/Debian/xUbuntu_22.04/Release.key | sudo apt-key add -
wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/v4.20.1/yq_linux_amd64 && chmod a+x /usr/local/bin/yq
sudo apt-get update || true && sudo apt-get install -y ca-certificates libxtst-dev libpng++-dev gcc-aarch64-linux-gnu g++-aarch64-linux-gnu jq icnsutils graphicsmagick tzdata xsel libgtk-3-0 libnss3 libxss1 libasound2
sudo apt-get install -y xvfb
export DISPLAY=:99
Xvfb :99 -screen 0 1024x768x24 &
npm ci
cd e2e
npm ci
npx electron-rebuild --platform=linux -f -t prod,optional,dev -w robotjs --module-dir ../
- name: e2e/install-dependencies-macos - name: e2e/install-node-dependencies
if: runner.os == 'macOS'
env:
PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
run: | run: |
brew install nss
jq '.mac.target=["zip"]' electron-builder.json | jq '.mac.gatekeeperAssess=false' > /tmp/electron-builder.json && cp /tmp/electron-builder.json .
npm ci npm ci
cd e2e cd e2e && npm ci
npm ci npx electron-rebuild --platform=${{ env.RUNNER_OS }} -f -t prod,optional,dev -w robotjs --module-dir ../
npx electron-rebuild --platform=darwin -f -t prod,optional,dev -w robotjs --module-dir ../
## Windows Dependencies
- name: e2e/install-dependencies-windows
if: runner.os == 'windows'
env:
PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
run: |
choco install yq --version 4.15.1 -y
choco install vcredist-all -y
npm i -g node-gyp
node-gyp install
node-gyp install --devdir="C:\Users\runneradmin\.electron-gyp" --target=$(jq -r .devDependencies.electron package.json) --dist-url="https://electronjs.org/headers"
node-gyp install --devdir="C:\Users\runneradmin\.electron-gyp" --target=$(jq -r .devDependencies.electron package.json) --dist-url="https://electronjs.org/headers" --arch arm64
npm ci --openssl_fips=''
cd e2e
npm ci
npx electron-rebuild --platform=win32 -f -t prod,optional,dev -w robotjs --module-dir ../
npm install mochawesome-report-generator
- name: e2e/run-playright-tests-linux - name: e2e/run-playright-tests-linux
if: runner.os == 'Linux' if: runner.os == 'Linux'
@@ -224,6 +285,9 @@ jobs:
SERVER_VERSION: ${{ inputs.MM_SERVER_VERSION }} SERVER_VERSION: ${{ inputs.MM_SERVER_VERSION }}
DESKTOP_VERSION: ${{ inputs.DESKTOP_VERSION }} DESKTOP_VERSION: ${{ inputs.DESKTOP_VERSION }}
- name: Capture Report Link
run: echo "REPORT_LINK=$(cat e2e/report-link.txt)" >> $GITHUB_ENV
- name: e2e/analyze-flaky-tests - name: e2e/analyze-flaky-tests
id: analyze-flaky-tests id: analyze-flaky-tests
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
@@ -231,8 +295,32 @@ jobs:
script: | script: |
process.chdir('./e2e'); process.chdir('./e2e');
const { analyzeFlakyTests } = require('./utils/analyze-flaky-test.js'); const { analyzeFlakyTests } = require('./utils/analyze-flaky-test.js');
const { commentBody, newFailedTests } = analyzeFlakyTests(); const { newFailedTests, os } = analyzeFlakyTests();
core.setOutput('COMMENT_BODY', commentBody);
if (newFailedTests.length > 0) { switch (os) {
core.setFailed('E2E tests failed.'); case 'linux':
core.setOutput('NEW_FAILURES_LINUX', newFailedTests.length);
core.setOutput('REPORT_LINK_LINUX', "${{ env.REPORT_LINK}}");
newFailedTests.length > 0 ? core.setOutput('STATUS_LINUX', 'failure') : core.setOutput('STATUS_LINUX', 'success');
break;
case 'darwin':
core.setOutput('NEW_FAILURES_MACOS', newFailedTests.length);
core.setOutput('REPORT_LINK_MACOS', "${{ env.REPORT_LINK}}");
newFailedTests.length > 0 ? core.setOutput('STATUS_MACOS', 'failure') : core.setOutput('STATUS_MACOS', 'success');
break;
case 'win32':
core.setOutput('NEW_FAILURES_WINDOWS', newFailedTests.length);
core.setOutput('REPORT_LINK_WINDOWS', "${{ env.REPORT_LINK}}");
newFailedTests.length > 0 ? core.setOutput('STATUS_WINDOWS', 'failure') : core.setOutput('STATUS_WINDOWS', 'success');
break;
default:
throw new Error(`Unsupported OS: ${os}`);
} }
- name: Upload test results
if: always()
uses: actions/upload-artifact@v4
with:
name: test-results-${{ runner.os }}
path: e2e/mochawesome-report
retention-days: 7

229
.github/workflows/e2e-functional.yml vendored
View File

@@ -2,126 +2,159 @@ name: Electron Playwright Tests
on: on:
push: push:
branches: branches: [master]
- master
pull_request: pull_request:
types: types: [labeled]
- labeled
workflow_dispatch: workflow_dispatch:
inputs: inputs:
version_name: version_name:
type: string type: string
description: "Desktop Version name eg: 5.6" description: "Desktop Version name eg: 5.6"
required: true required: true
job_name: platform:
type: choice type: choice
description: "Job name" description: "Choose platform to run tests"
required: true required: true
default: "All" default: "all"
options: options: [linux, macos, windows, all]
- "e2e-linux"
- "e2e-macos"
- "e2e-windows"
- "All"
jobs: jobs:
e2e-linux: update-initial-status-linux:
if: ${{ runs-on: ubuntu-22.04
( if: contains(github.event.label.name, 'Run Desktop E2E Tests')
(inputs.job_name == 'e2e-linux' || inputs.job_name == 'All') steps:
&& - uses: mattermost/actions/delivery/update-commit-status@main
github.event_name == 'workflow_dispatch' env:
) || GITHUB_TOKEN: ${{ github.token }}
( with:
github.event_name == 'push' repository_full_name: ${{ github.repository }}
) || commit_sha: ${{ github.event.pull_request.head.sha }}
( context: e2e/linux
github.event_name == 'pull_request' && description: E2E tests for Mattermost desktop app on Linux have started...
github.event.pull_request.labels && status: pending
contains(github.event.pull_request.labels.*.name, 'Run Desktop E2E Tests')
) update-initial-status-macos:
}} runs-on: ubuntu-22.04
if: contains(github.event.label.name, 'Run Desktop E2E Tests')
steps:
- uses: mattermost/actions/delivery/update-commit-status@main
env:
GITHUB_TOKEN: ${{ github.token }}
with:
repository_full_name: ${{ github.repository }}
commit_sha: ${{ github.event.pull_request.head.sha }}
context: e2e/macOS
description: E2E tests for Mattermost desktop app on macOS have started...
status: pending
# TODO: uncomment when https://mattermost.atlassian.net/browse/MM-63397 is fixed
# update-initial-status-windows:
# runs-on: ubuntu-22.04
# if: contains(github.event.label.name, 'Run Desktop E2E Tests')
# steps:
# - uses: mattermost/actions/delivery/update-commit-status@main
# env:
# GITHUB_TOKEN: ${{ github.token }}
# with:
# repository_full_name: ${{ github.repository }}
# commit_sha: ${{ github.event.pull_request.head.sha }}
# context: e2e/windows
# description: E2E tests for Mattermost desktop app on Windows have started...
# status: pending
determine-platforms:
if: contains(github.event.label.name, 'Run Desktop E2E Tests')
needs: [update-initial-status-linux, update-initial-status-macos] #, update-initial-status-windows]
runs-on: ubuntu-latest
outputs:
platforms: ${{ steps.set-matrix.outputs.platforms }}
steps:
- id: set-matrix
run: |
if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
if [[ "${{ inputs.platform }}" == "all" ]]; then
echo "platforms=['ubuntu-22.04', 'macos-13', 'windows-2022']" >> $GITHUB_OUTPUT
else
case "${{ inputs.platform }}" in
linux) echo "platforms=['ubuntu-22.04']" ;;
macos) echo "platforms=['macos-13']" ;;
# windows) echo "platforms=['windows-2022']" ;; TODO: uncomment when https://mattermost.atlassian.net/browse/MM-63397 is fixed
esac >> $GITHUB_OUTPUT
fi
elif [[ "${{ github.event_name }}" == "push" ]] || [[ "${{ contains(github.event.pull_request.labels.*.name, 'Run Desktop E2E Tests') }}" == "true" ]]; then
echo "platforms=['ubuntu-22.04', 'macos-13']" >> $GITHUB_OUTPUT
# echo "platforms=['ubuntu-22.04', 'macos-13', 'windows-2022']" >> $GITHUB_OUTPUT
else
echo "platforms=[]" >> $GITHUB_OUTPUT
fi
e2e-tests:
needs: determine-platforms
strategy:
matrix:
os: ${{ fromJson(needs.determine-platforms.outputs.platforms) }}
fail-fast: false
uses: ./.github/workflows/e2e-functional-template.yml uses: ./.github/workflows/e2e-functional-template.yml
secrets: inherit secrets: inherit
with: with:
runs-on: ubuntu-22.04 runs-on: ${{ matrix.os }}
DESKTOP_VERSION: ${{ inputs.version_name || github.head_ref || github.ref }} DESKTOP_VERSION: ${{ inputs.version_name || github.head_ref || github.ref }}
e2e-macos: update-final-status-linux:
if: ${{ runs-on: ubuntu-22.04
( needs:
(inputs.job_name == 'e2e-macos' || inputs.job_name == 'All') - e2e-tests
&& steps:
github.event_name == 'workflow_dispatch' - uses: mattermost/actions/delivery/update-commit-status@main
) || env:
( GITHUB_TOKEN: ${{ github.token }}
github.event_name == 'push' with:
) || repository_full_name: ${{ github.repository }}
( commit_sha: ${{ github.event.pull_request.head.sha }}
github.event_name == 'pull_request' && context: e2e/linux
github.event.pull_request.labels && description: Completed with ${{ needs.e2e-tests.outputs.NEW_FAILURES_LINUX }} failures
contains(github.event.pull_request.labels.*.name, 'Run Desktop E2E Tests') status: ${{ needs.e2e-tests.outputs.STATUS_LINUX }}
) target_url: ${{ needs.e2e-tests.outputs.REPORT_LINK_LINUX }}
}}
uses: ./.github/workflows/e2e-functional-template.yml
secrets: inherit
with:
runs-on: macos-13
DESKTOP_VERSION: ${{ inputs.version_name || github.head_ref || github.ref }}
# TODO: Uncomment when https://mattermost.atlassian.net/browse/MM-63397 is resolved update-final-status-macos:
# e2e-windows: runs-on: ubuntu-22.04
# if: ${{ needs:
# ( - e2e-tests
# (inputs.job_name == 'e2e-windows' || inputs.job_name == 'All') steps:
# && - uses: mattermost/actions/delivery/update-commit-status@main
# github.event_name == 'workflow_dispatch' env:
# ) || GITHUB_TOKEN: ${{ github.token }}
# ( with:
# github.event_name == 'push' repository_full_name: ${{ github.repository }}
# ) || commit_sha: ${{ github.event.pull_request.head.sha }}
# ( context: e2e/macOS
# github.event_name == 'pull_request' && description: Completed with ${{ needs.e2e-tests.outputs.NEW_FAILURES_MACOS }} failures
# github.event.pull_request.labels && status: ${{ needs.e2e-tests.outputs.STATUS_MACOS }}
# contains(github.event.pull_request.labels.*.name, 'Run Desktop E2E Tests') target_url: ${{ needs.e2e-tests.outputs.REPORT_LINK_MACOS }}
# )
# }} # TODO: uncomment when https://mattermost.atlassian.net/browse/MM-63397 is fixed
# uses: ./.github/workflows/e2e-functional-template.yml # update-final-status-windows:
# secrets: inherit # runs-on: ubuntu-22.04
# with: # needs:
# runs-on: windows-2022 # - e2e-tests
# DESKTOP_VERSION: ${{ inputs.version_name || github.head_ref || github.ref }} # steps:
# - uses: mattermost/actions/delivery/update-commit-status@main
# env:
# GITHUB_TOKEN: ${{ github.token }}
# with:
# repository_full_name: ${{ github.repository }}
# commit_sha: ${{ github.event.pull_request.head.sha }}
# context: e2e/windows
# description: Completed with ${{ needs.e2e-tests.outputs.NEW_FAILURES_WINDOWS }} failures
# status: ${{ needs.e2e-tests.outputs.STATUS_WINDOWS }}
# target_url: ${{ needs.e2e-tests.outputs.REPORT_LINK_WINDOWS }}
e2e-remove-label: e2e-remove-label:
if: ${{ always() && contains(github.event.pull_request.labels.*.name, 'Run Desktop E2E Tests') }} if: ${{ always() && contains(github.event.pull_request.labels.*.name, 'Run Desktop E2E Tests') }}
needs: [e2e-linux, e2e-macos] #, e2e-windows] TODO: Uncomment when https://mattermost.atlassian.net/browse/MM-63397 is resolved needs: e2e-tests
runs-on: ubuntu-22.04 runs-on: ubuntu-22.04
env:
COMMENT_BODY_LINUX: "${{ needs.e2e-linux.outputs.COMMENT_BODY }}"
COMMENT_BODY_MACOS: "${{ needs.e2e-macos.outputs.COMMENT_BODY }}"
# COMMENT_BODY_WINDOWS: "${{ needs.e2e-windows.outputs.COMMENT_BODY }}"
steps: steps:
- name: e2e/unify-comments-in-single-comment - name: Remove E2E test label
run: |
echo "PR_COMMENT<<EOF" >> "${GITHUB_ENV}"
echo "Here are the test results below:" >> "${GITHUB_ENV}"
echo "$COMMENT_BODY_LINUX" >> "${GITHUB_ENV}"
echo "$COMMENT_BODY_MACOS" >> "${GITHUB_ENV}"
echo "$COMMENT_BODY_WINDOWS" >> "${GITHUB_ENV}"
echo "EOF" >> "${GITHUB_ENV}"
- name: e2e/send-comment-results-in-pr
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
with:
script: |
github.rest.issues.createComment({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
body: process.env.PR_COMMENT,
});
- name: e2e/remove-label-from-pr
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
continue-on-error: true # Label might have been removed manually continue-on-error: true # Label might have been removed manually
with: with:

27
e2e/modules/environment.js
View File

@@ -268,23 +268,22 @@ module.exports = {
await window.waitForSelector('#saveSetting'); await window.waitForSelector('#saveSetting');
let username = process.env.MM_TEST_USERNAME; let username = process.env.MM_TEST_USERNAME;
if (!username) { switch (process.platform) {
switch (process.platform) { case 'darwin':
case 'darwin': username = 'success+sysadmin+macos@simulator.amazonses.com';
username = 'success+sysadmin+macos@simulator.amazonses.com'; break;
break; case 'linux':
case 'linux': username = 'success+sysadmin+linux@simulator.amazonses.com';
username = 'success+sysadmin+linux@simulator.amazonses.com'; break;
break; case 'win32':
case 'win32': username = 'success+sysadmin+windows@simulator.amazonses.com';
username = 'success+sysadmin+windows@simulator.amazonses.com'; break;
break; default:
default: throw new Error('Unsupported platform');
throw new Error('Unsupported platform');
}
} }
await window.type('#input_loginId', username); await window.type('#input_loginId', username);
await window.type('#input_password-input', process.env.MM_TEST_PASSWORD); await window.type('#input_password-input', process.env.MM_TEST_PASSWORD);
await window.click('#saveSetting'); await window.click('#saveSetting');
}, },

6
e2e/save_report.js
View File

@@ -27,6 +27,7 @@
* - TYPE=[type], e.g. "MASTER", "PR", "RELEASE", "CLOUD" * - TYPE=[type], e.g. "MASTER", "PR", "RELEASE", "CLOUD"
*/ */
const fs = require('fs');
const path = require('path'); const path = require('path');
const generator = require('mochawesome-report-generator'); const generator = require('mochawesome-report-generator');
@@ -79,6 +80,9 @@ const saveReport = async () => {
const result = await saveArtifacts(); const result = await saveArtifacts();
if (result && result.success) { if (result && result.success) {
console.log('Successfully uploaded artifacts to S3:', result.reportLink); console.log('Successfully uploaded artifacts to S3:', result.reportLink);
// save the report link to a file For CI to use
fs.writeFileSync('report-link.txt', result.reportLink);
} }
// Create or use an existing test cycle // Create or use an existing test cycle
@@ -98,8 +102,6 @@ const saveReport = async () => {
if (ZEPHYR_ENABLE === 'true') { if (ZEPHYR_ENABLE === 'true') {
await createTestExecutions(jsonReport, testCycle); await createTestExecutions(jsonReport, testCycle);
} }
// chai.expect(Boolean(jsonReport.stats.failures), FAILURE_MESSAGE).to.be.false;
}; };
saveReport(); saveReport();

49
e2e/specs/mattermost/copy_link.test.js
View File

@@ -32,29 +32,28 @@ describe('copylink', function desc() {
await env.clearElectronInstances(); await env.clearElectronInstances();
}); });
it('MM-T125 Copy Link can be used from channel LHS', async () => { if (process.platform !== 'linux') {
const firstServer = this.serverMap[`${config.teams[0].name}___TAB_MESSAGING`].win; it('MM-T125 Copy Link can be used from channel LHS', async () => {
await env.loginToMattermost(firstServer); const firstServer = this.serverMap[`${config.teams[0].name}___TAB_MESSAGING`].win;
await asyncSleep(2000); await env.loginToMattermost(firstServer);
await firstServer.waitForSelector('#sidebarItem_town-square', {timeout: 5000}); await asyncSleep(2000);
await firstServer.click('#sidebarItem_town-square', {button: 'right'}); await firstServer.waitForSelector('#sidebarItem_town-square', {timeout: 5000});
await asyncSleep(2000); await firstServer.click('#sidebarItem_town-square', {button: 'right'});
switch (process.platform) { await asyncSleep(2000);
case 'linux': switch (process.platform) {
robot.keyTap('c'); case 'win32':
break; robot.keyTap('down');
case 'win32': robot.keyTap('down');
robot.keyTap('down'); break;
robot.keyTap('down'); case 'darwin':
break; robot.keyTap('c');
case 'darwin': break;
robot.keyTap('c'); }
break; robot.keyTap('enter');
} await firstServer.click('#sidebarItem_town-square');
robot.keyTap('enter'); await firstServer.click('#post_textbox');
await firstServer.click('#sidebarItem_town-square'); const clipboardText = clipboard.readText();
await firstServer.click('#post_textbox'); clipboardText.should.contain('/channels/town-square');
const clipboardText = clipboard.readText(); });
clipboardText.should.contain('/channels/town-square'); }
});
}); });

19
e2e/specs/menu_bar/history_menu.test.js
View File

@@ -34,16 +34,27 @@ describe('history_menu', function desc() {
await env.loginToMattermost(firstServer); await env.loginToMattermost(firstServer);
await firstServer.waitForSelector('#sidebarItem_off-topic'); await firstServer.waitForSelector('#sidebarItem_off-topic');
// click on Off topic channel // Click on Off-Topic channel
await firstServer.click('#sidebarItem_off-topic'); await firstServer.click('#sidebarItem_off-topic');
// click on town square channel // Click on Town Square channel
await firstServer.click('#sidebarItem_town-square'); await firstServer.click('#sidebarItem_town-square');
await firstServer.locator('[aria-label="Back"]').click(); await firstServer.locator('[aria-label="Back"]').click();
let channelHeaderText = await firstServer.$eval('#channelHeaderTitle', (el) => el.firstChild.innerHTML);
// Wait for navigation
await firstServer.waitForSelector('#channelHeaderTitle');
// Get channel header text
let channelHeaderText = await firstServer.$eval('#channelHeaderTitle', (el) => el.textContent.trim());
channelHeaderText.should.equal('Off-Topic'); channelHeaderText.should.equal('Off-Topic');
await firstServer.locator('[aria-label="Forward"]').click(); await firstServer.locator('[aria-label="Forward"]').click();
channelHeaderText = await firstServer.$eval('#channelHeaderTitle', (el) => el.firstChild.innerHTML); await asyncSleep(3000);
// Wait for navigation
await firstServer.waitForSelector('#channelHeaderTitle');
channelHeaderText = await firstServer.$eval('#channelHeaderTitle', (el) => el.textContent.trim());
channelHeaderText.should.equal('Town Square'); channelHeaderText.should.equal('Town Square');
}); });
}); });

52
e2e/specs/menu_bar/view_menu.test.js
View File

@@ -187,31 +187,33 @@ describe('menu/view', function desc() {
}); });
}); });
it('MM-T820 should open Developer Tools For Application Wrapper for main window', async () => { if (process.platform !== 'linux') {
const mainWindow = this.app.windows().find((window) => window.url().includes('index.html')); it('MM-T820 should open Developer Tools For Application Wrapper for main window', async () => {
const browserWindow = await this.app.browserWindow(mainWindow); const mainWindow = this.app.windows().find((window) => window.url().includes('index.html'));
const browserWindow = await this.app.browserWindow(mainWindow);
let isDevToolsOpen = await browserWindow.evaluate((window) => { let isDevToolsOpen = await browserWindow.evaluate((window) => {
return window.webContents.isDevToolsOpened(); return window.webContents.isDevToolsOpened();
});
isDevToolsOpen.should.be.false;
if (process.platform === 'darwin') {
// Press Command + Option + I
robot.keyTap('i', ['command', 'alt']);
await asyncSleep(3000);
}
if (process.platform === 'win32') {
robot.keyToggle('shift', 'down');
robot.keyToggle('control', 'down');
robot.keyTap('i');
}
await asyncSleep(1000);
isDevToolsOpen = await browserWindow.evaluate((window) => {
return window.webContents.isDevToolsOpened();
});
isDevToolsOpen.should.be.true;
}); });
isDevToolsOpen.should.be.false; }
if (process.platform === 'darwin') {
// Press Command + Option + I
robot.keyTap('i', ['command', 'alt']);
await asyncSleep(3000);
}
if (process.platform === 'win32') {
robot.keyToggle('shift', 'down');
robot.keyToggle('control', 'down');
robot.keyTap('i');
}
await asyncSleep(1000);
isDevToolsOpen = await browserWindow.evaluate((window) => {
return window.webContents.isDevToolsOpened();
});
isDevToolsOpen.should.be.true;
});
}); });

63
e2e/specs/menu_bar/window_menu.test.js
View File

@@ -147,39 +147,42 @@ describe('Menu/window_menu', function desc() {
await afterFunc(); await afterFunc();
}); });
it('MM-T824 should be minimized when keyboard shortcuts are pressed', async () => { if (process.platform !== 'linux') {
await beforeFunc(); it('MM-T824 should be minimized when keyboard shortcuts are pressed', async () => {
await beforeFunc();
const mainWindow = this.app.windows().find((window) => window.url().includes('index')); const mainWindow = this.app.windows().find((window) => window.url().includes('index'));
const browserWindow = await this.app.browserWindow(mainWindow); const browserWindow = await this.app.browserWindow(mainWindow);
if (process.platform === 'darwin') { if (process.platform === 'darwin') {
robot.keyTap('m', [env.cmdOrCtrl]); robot.keyTap('m', [env.cmdOrCtrl]);
} else { } else {
await mainWindow.click('button.three-dot-menu'); await mainWindow.click('button.three-dot-menu');
robot.keyTap('w'); robot.keyTap('w');
robot.keyTap('m'); robot.keyTap('m');
robot.keyTap('enter'); robot.keyTap('enter');
} }
await asyncSleep(2000); await asyncSleep(2000);
const isMinimized = await browserWindow.evaluate((window) => window.isMinimized()); const isMinimized = await browserWindow.evaluate((window) => window.isMinimized());
isMinimized.should.be.true; isMinimized.should.be.true;
await afterFunc(); await afterFunc();
}); });
}
if (process.platform !== 'linux') {
it('MM-T825 should be hidden when keyboard shortcuts are pressed', async () => {
await beforeFunc();
it('MM-T825 should be hidden when keyboard shortcuts are pressed', async () => { const mainWindow = this.app.windows().find((window) => window.url().includes('index'));
await beforeFunc(); const browserWindow = await this.app.browserWindow(mainWindow);
robot.keyTap('w', [env.cmdOrCtrl]);
await asyncSleep(2000);
const isVisible = await browserWindow.evaluate((window) => window.isVisible());
isVisible.should.be.false;
const isDestroyed = await browserWindow.evaluate((window) => window.isDestroyed());
isDestroyed.should.be.false;
const mainWindow = this.app.windows().find((window) => window.url().includes('index')); await afterFunc();
const browserWindow = await this.app.browserWindow(mainWindow); });
robot.keyTap('w', [env.cmdOrCtrl]); }
await asyncSleep(2000);
const isVisible = await browserWindow.evaluate((window) => window.isVisible());
isVisible.should.be.false;
const isDestroyed = await browserWindow.evaluate((window) => window.isDestroyed());
isDestroyed.should.be.false;
await afterFunc();
});
}); });

16
e2e/specs/server_management/header.test.js
View File

@@ -37,13 +37,15 @@ describe('header', function desc() {
await env.clearElectronInstances(); await env.clearElectronInstances();
}); });
it('MM-T2637_1 should maximize on double-clicking the header', async () => { if (process.platform !== 'linux') {
const headerBounds = await header.boundingBox(); it('MM-T2637_1 should maximize on double-clicking the header', async () => {
await header.dblclick({position: {x: headerBounds.width / 2, y: headerBounds.y / 2}}); const headerBounds = await header.boundingBox();
await asyncSleep(1000); await header.dblclick({position: {x: headerBounds.width / 2, y: headerBounds.y / 2}});
const isMaximized = await browserWindow.evaluate((window) => window.isMaximized()); await asyncSleep(1000);
isMaximized.should.be.equal(true); const isMaximized = await browserWindow.evaluate((window) => window.isMaximized());
}); isMaximized.should.be.equal(true);
});
}
it('MM-T2637_2 should restore on double-clicking the header when maximized', async () => { it('MM-T2637_2 should restore on double-clicking the header when maximized', async () => {
const maximizedHeaderBounds = await header.boundingBox(); const maximizedHeaderBounds = await header.boundingBox();

42
e2e/utils/analyze-flaky-test.js
View File

@@ -24,53 +24,13 @@ function analyzeFlakyTests() {
// Filter out the known flaky tests from the failed test titles // Filter out the known flaky tests from the failed test titles
const newFailedTests = failedFullTitles.filter((test) => !knownFlakyTestsForOS.has(test)); const newFailedTests = failedFullTitles.filter((test) => !knownFlakyTestsForOS.has(test));
// Check if any known failed tests are fixed return {newFailedTests, os};
const fixedTests = [...knownFlakyTestsForOS].filter((test) => !failedFullTitles.includes(test));
const commentBody = generateCommentBodyFunctionalTest(newFailedTests, fixedTests);
// Print on CI
console.log(commentBody);
return {commentBody, newFailedTests};
} catch (error) { } catch (error) {
console.error('Error analyzing failures:', error); console.error('Error analyzing failures:', error);
return {}; return {};
} }
} }
function generateCommentBodyFunctionalTest(newFailedTests, fixedTests) {
const osName = process.env.RUNNER_OS;
const build = process.env.BUILD_TAG;
let commentBody = `
## Test Summary for ${osName} on commit ${build}
`;
if (newFailedTests.length === 0 && fixedTests.length === 0) {
commentBody += `
All stable tests passed on ${osName}.
`;
return commentBody;
}
if (newFailedTests.length > 0) {
const newTestFailure = `New failed tests found on ${osName}:\n${newFailedTests.map((test) => `- ${test}`).join('\n')}`;
commentBody += `
${newTestFailure}
`;
}
if (fixedTests.length > 0) {
const fixedTestMessage = `The following known failed tests have been fixed on ${osName}:\n\t${fixedTests.map((test) => `- ${test}`).join('\n\t')}`;
commentBody += `
${fixedTestMessage}
`;
}
return commentBody;
}
module.exports = { module.exports = {
analyzeFlakyTests, analyzeFlakyTests,
}; };