phlux/mattermostest
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Modify run conditions for nightly e2e job to create test cycle and post in channel. (#2971)

Browse Source
This commit is contained in:
yasserfaraazkhan
2024-03-19 22:18:29 +05:30
committed by GitHub
parent d3b43b5b64
commit 7c2b4ec502
18 changed files with 480 additions and 268 deletions
Download Patch File Download Diff File Expand all files Collapse all files

240
.github/workflows/e2e-functional.yml vendored
View File

@@ -32,28 +32,28 @@ on:
env:
AWS_S3_BUCKET: "mattermost-cypress-report"
BRANCH: ${{ github.ref }}
BUILD_SUFFIX: 'desktop-pr'
BRANCH: ${{ github.head_ref || github.ref_name }}
BUILD_TAG: ${{ github.sha }}
JIRA_PROJECT_KEY: 'MM'
MM_TEST_SERVER_URL: ${{ secrets.MM_DESKTOP_E2E_SERVER_URL }}
MM_TEST_USER_NAME: ${{ secrets.MM_DESKTOP_E2E_USER_NAME }}
MM_TEST_PASSWORD: ${{ secrets.MM_DESKTOP_E2E_USER_CREDENTIALS }}
PULL_REQUEST_BASE_URL: "https://github.com/mattermost/desktop/pull/"
PULL_REQUEST: "https://github.com/mattermost/desktop/pull/${{ github.event.number }}"
ZEPHYR_ENVIRONMENT_NAME: 'Desktop app'
ZEPHYR_FOLDER_ID: "3256491"
ZEPHYR_FOLDER_ID: "12413253"
TEST_CYCLE_LINK_PREFIX: ${{ secrets.MM_DESKTOP_E2E_TEST_CYCLE_LINK_PREFIX }}
AWS_ACCESS_KEY_ID: ${{ secrets.MM_DESKTOP_E2E_AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.MM_DESKTOP_E2E_AWS_SECRET_ACCESS_KEY }}
AWS_REGION: "us-east-1"
WEBHOOK_URL: ${{ secrets.MM_DESKTOP_E2E_WEBHOOK_URL }}
ZEPHYR_API_KEY: ${{ secrets.MM_DESKTOP_E2E_ZEPHYR_API_KEY }}
REPORT_LINK: "none"
jobs:
e2e-linux:
if: ${{
(
(inputs.job_name == 'e2e-linux' ||
inputs.job_name == 'All')
(inputs.job_name == 'e2e-linux' || inputs.job_name == 'All')
&&
github.event_name == 'workflow_dispatch'
) ||
@@ -65,17 +65,16 @@ jobs:
github.event.pull_request.labels &&
contains(github.event.pull_request.labels.*.name, 'Run Desktop E2E Tests')
) ||
(
github.event_name == 'schedule' || github.event_name == 'workflow_call'
)
(
(inputs.tag != '')
)
}}
runs-on: ubuntu-latest
runs-on: ubuntu-22.04
steps:
- name: ci/checkout-repo
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
- name: ci/setup-node
uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c # v3.6.0
uses: actions/setup-node@v4
with:
node-version-file: "package.json"
cache: "npm"
@@ -87,52 +86,86 @@ jobs:
run: |
wget -qO - https://download.opensuse.org/repositories/Emulators:/Wine:/Debian/xUbuntu_22.04/Release.key | sudo apt-key add -
wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/v4.20.1/yq_linux_amd64 && chmod a+x /usr/local/bin/yq
sudo apt-get update || true && sudo apt-get install -y ca-certificates libxtst-dev libpng++-dev gcc-aarch64-linux-gnu g++-aarch64-linux-gnu jq icnsutils graphicsmagick tzdata
sudo apt-get update || true && sudo apt-get install -y ca-certificates libxtst-dev libpng++-dev gcc-aarch64-linux-gnu g++-aarch64-linux-gnu jq icnsutils graphicsmagick tzdata xsel
npm ci
npx electron-rebuild --platform=linux -f -t prod,optional,dev -w robotjs
- name: Set Environment Variables
run: |
if [ "${{ github.event_name }}" == "pull_request" ]; then
echo "BRANCH=${{ github.event.pull_request.head.ref }}" >> $GITHUB_ENV
echo "BUILD_SUFFIX=desktop-pr" >> $GITHUB_ENV
echo "BUILD_SUFFIX=desktop-pr-linux" >> $GITHUB_ENV
echo "TYPE=PR" >> $GITHUB_ENV
elif [ "${{ github.event_name }}" == "release" ]; then
echo "BRANCH=${{ github.ref }}" >> $GITHUB_ENV
echo "BUILD_SUFFIX=desktop-release" >> $GITHUB_ENV
echo "BUILD_SUFFIX=desktop-release-linux" >> $GITHUB_ENV
echo "TYPE=RELEASE" >> $GITHUB_ENV
echo "ZEPHYR_ENABLE=true" >> $GITHUB_ENV
echo "ZEPHYR_FOLDER_LINUX_REPORT=12358649" >> $GITHUB_ENV
elif [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
echo "BRANCH=${{ github.ref }}" >> $GITHUB_ENV
echo "BUILD_SUFFIX=desktop-manual-trigger" >> $GITHUB_ENV
elif [ "${{ github.event_name }}" == "workflow_dispatch" ] && [ "${{ inputs.tag }}" == "" ]; then
echo "BUILD_SUFFIX=desktop-manual-trigger-linux" >> $GITHUB_ENV
echo "TYPE=MANUAL" >> $GITHUB_ENV
elif [ "${{ github.event_name }}" == "push" ] && [ "${{ github.ref }}" == "refs/heads/master" ]; then
echo "BRANCH=${{ github.ref }}" >> $GITHUB_ENV
echo "BUILD_SUFFIX=desktop-master-push-linux" >> $GITHUB_ENV
echo "TYPE=MASTER" >> $GITHUB_ENV
echo "BUILD_SUFFIX=desktop-master-push" >> $GITHUB_ENV
echo "ZEPHYR_ENABLE=true" >> $GITHUB_ENV
elif [ "${{ github.event_name }}" == "schedule" || "${{ github.event_name }}" == "workflow_call" ]; then
echo "BRANCH=${{ github.ref }}" >> $GITHUB_ENV
echo "BUILD_SUFFIX=desktop-nightly" >> $GITHUB_ENV
elif [ "${{ inputs.tag }}" != "" ]; then
echo "BUILD_SUFFIX=desktop-nightly-linux" >> $GITHUB_ENV
echo "TYPE=NIGHTLY" >> $GITHUB_ENV
echo "ZEPHYR_ENABLE=true" >> $GITHUB_ENV
echo "ZEPHYR_FOLDER_LINUX_REPORT=12363689" >> $GITHUB_ENV
fi
- name: Set Build ID
run: echo "BUILD_ID=${{ github.run_id }}-${{ env.BUILD_SUFFIX }}" >> $GITHUB_ENV
- name: Run Playwright tests (Ubuntu OS)
run: |
export DISPLAY=:99
Xvfb $DISPLAY -screen 0 1024x768x24 > /dev/null 2>&1 &
Xvfb $DISPLAY -screen 0 1280x960x24 > /dev/null 2>&1 &
npm run test:e2e || true # making job pass even if the tests fail due to flakyness
npm run test:e2e:send-report
- name: Analyze flaky tests
id: analyze-flaky-tests
uses: actions/github-script@v7
with:
script: |
const { analyzeFlakyTests } = require('./e2e/utils/analyze-flaky-test.js');
const { newFailedTests } = analyzeFlakyTests();
if (newFailedTests.length === 0) {
process.env.NEW_FAILED_TESTS = 'true';
}
env:
NEW_FAILED_TESTS: false
- name: Add results in PR comment
if: github.event_name == 'pull_request'
uses: actions/github-script@v7
with:
script: |
const { analyzeFlakyTests } = require('./e2e/utils/analyze-flaky-test.js');
const { commentBody } = analyzeFlakyTests();
github.rest.issues.createComment({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
body: commentBody,
});
- name: Mark job as failed if there are new failed tests
if: env.NEW_FAILED_TESTS == 'true'
run: exit 1
shell: bash
e2e-macos:
if: ${{
(
(inputs.job_name == 'e2e-macos' ||
inputs.job_name == 'All')
(inputs.job_name == 'e2e-macos' || inputs.job_name == 'All')
&&
github.event_name == 'workflow_dispatch'
) ||
@@ -144,16 +177,16 @@ jobs:
github.event.pull_request.labels &&
contains(github.event.pull_request.labels.*.name, 'Run Desktop E2E Tests')
) ||
(
github.event_name == 'schedule' || github.event_name == 'workflow_call'
)
(
(inputs.tag != '')
)
}}
runs-on: macos-13
steps:
- name: ci/checkout-repo
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
- name: ci/setup-node
uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c # v3.6.0
uses: actions/setup-node@v4
with:
node-version-file: "package.json"
cache: "npm"
@@ -170,59 +203,88 @@ jobs:
uses: actions/setup-python@v5
with:
python-version: '3.10'
- name: ci/install-dependencies
env:
PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
run: |
npm cache clean --force
jq '.mac.target=["zip"]' electron-builder.json | jq '.mac.gatekeeperAssess=false' > /tmp/electron-builder.json && cp /tmp/electron-builder.json .
npm ci
npm install -g node-gyp
npm i robotjs
npx electron-rebuild -f -t prod,optional,dev -w robotjs
npx electron-rebuild --platform=darwin -f -t prod,optional,dev -w robotjs
- name: Set Environment Variables
run: |
if [ "${{ github.event_name }}" == "pull_request" ]; then
echo "BRANCH=${{ github.event.pull_request.head.ref }}" >> $GITHUB_ENV
echo "BUILD_SUFFIX=desktop-pr" >> $GITHUB_ENV
echo "BUILD_SUFFIX=desktop-pr-macos" >> $GITHUB_ENV
echo "TYPE=PR" >> $GITHUB_ENV
elif [ "${{ github.event_name }}" == "release" ]; then
echo "BRANCH=${{ github.ref }}" >> $GITHUB_ENV
echo "BUILD_SUFFIX=desktop-release" >> $GITHUB_ENV
echo "BUILD_SUFFIX=desktop-release-macos" >> $GITHUB_ENV
echo "TYPE=RELEASE" >> $GITHUB_ENV
echo "ZEPHYR_ENABLE=true" >> $GITHUB_ENV
echo "ZEPHYR_FOLDER_MACOS_REPORT=12358650" >> $GITHUB_ENV
elif [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
echo "BRANCH=${{ github.ref }}" >> $GITHUB_ENV
echo "BUILD_SUFFIX=desktop-manual-trigger" >> $GITHUB_ENV
elif [ "${{ github.event_name }}" == "workflow_dispatch" ] && [ "${{ inputs.tag }}" == "" ]; then
echo "BUILD_SUFFIX=desktop-manual-trigger-macos" >> $GITHUB_ENV
echo "TYPE=MANUAL" >> $GITHUB_ENV
elif [ "${{ github.event_name }}" == "push" ] && [ "${{ github.ref }}" == "refs/heads/master" ]; then
echo "BRANCH=${{ github.ref }}" >> $GITHUB_ENV
echo "BUILD_SUFFIX=desktop-master-push-macos" >> $GITHUB_ENV
echo "TYPE=MASTER" >> $GITHUB_ENV
echo "BUILD_SUFFIX=desktop-master-push" >> $GITHUB_ENV
echo "ZEPHYR_ENABLE=true" >> $GITHUB_ENV
elif [ "${{ github.event_name }}" == "schedule" || "${{ github.event_name }}" == "workflow_call" ]; then
echo "BRANCH=${{ github.ref }}" >> $GITHUB_ENV
echo "BUILD_SUFFIX=desktop-nightly" >> $GITHUB_ENV
elif [ "${{ inputs.tag }}" != "" ]; then
echo "BUILD_SUFFIX=desktop-nightly-macos" >> $GITHUB_ENV
echo "TYPE=NIGHTLY" >> $GITHUB_ENV
echo "ZEPHYR_ENABLE=true" >> $GITHUB_ENV
echo "ZEPHYR_FOLDER_MACOS_REPORT=12363687" >> $GITHUB_ENV
fi
- name: Set Build ID
run: echo "BUILD_ID=${{ github.run_id }}-${{ env.BUILD_SUFFIX }}" >> $GITHUB_ENV
- name: Run Playwright tests (macOS)
run: |
cat $GITHUB_ENV
npm run test:e2e || true # making job pass even if the tests fail due to flakyness
npm run test:e2e:send-report
- name: Analyze flaky tests
id: analyze-flaky-tests
uses: actions/github-script@v7
with:
script: |
const { analyzeFlakyTests } = require('./e2e/utils/analyze-flaky-test.js');
const { newFailedTests } = analyzeFlakyTests();
if (newFailedTests.length === 0) {
process.env.NEW_FAILED_TESTS = 'true';
}
- name: Add results in PR comment
if: github.event_name == 'pull_request'
uses: actions/github-script@v7
with:
script: |
const { analyzeFlakyTests } = require('./e2e/utils/analyze-flaky-test.js');
const { commentBody } = analyzeFlakyTests();
github.rest.issues.createComment({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
body: commentBody,
});
- name: Mark job as failed if there are new failed tests
if: env.NEW_FAILED_TESTS == 'true'
run: exit 1
shell: bash
e2e-windows:
if: ${{
(
(inputs.job_name == 'e2e-windows' ||
inputs.job_name == 'All')
(inputs.job_name == 'e2e-windows' || inputs.job_name == 'All')
&&
github.event_name == 'workflow_dispatch'
) ||
@@ -234,16 +296,16 @@ jobs:
github.event.pull_request.labels &&
contains(github.event.pull_request.labels.*.name, 'Run Desktop E2E Tests')
) ||
(
github.event_name == 'schedule' || github.event_name == 'workflow_call'
)
(
(inputs.tag != '')
)
}}
runs-on: windows-2022
steps:
- name: ci/checkout-repo
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
- name: ci/setup-node
uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c # v3.6.0
uses: actions/setup-node@v4
with:
node-version-file: "package.json"
cache: "npm"
@@ -266,59 +328,93 @@ jobs:
PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
run: |
choco install yq --version 4.15.1 -y
npm install -g windows-build-tools
npm i -g node-gyp
node-gyp install
node-gyp install --devdir="C:\Users\runneradmin\.electron-gyp" --target=$(jq -r .devDependencies.electron package.json) --dist-url="https://electronjs.org/headers"
node-gyp install --devdir="C:\Users\runneradmin\.electron-gyp" --target=$(jq -r .devDependencies.electron package.json) --dist-url="https://electronjs.org/headers" --arch arm64
node-gyp install --devdir="C:\Users\runneradmin\.electron-gyp" --target=$(jq -r .devDependencies.electron package.json) --dist-url="https://electronjs.org/headers" --arch ia32
npm ci --openssl_fips=''
npm i robotjs
npx electron-rebuild --platform=win32 -f -t prod,optional,dev -w robotjs
- name: Set Environment Variables
shell: bash
run: |
if [ "${{ github.event_name }}" == "pull_request" ]; then
echo "BRANCH=${{ github.event.pull_request.head.ref }}" >> $GITHUB_ENV
echo "BUILD_SUFFIX=desktop-pr" >> $GITHUB_ENV
echo "BUILD_SUFFIX=desktop-pr-windows" >> $GITHUB_ENV
echo "TYPE=PR" >> $GITHUB_ENV
elif [ "${{ github.event_name }}" == "release" ]; then
echo "BRANCH=${{ github.ref }}" >> $GITHUB_ENV
echo "BUILD_SUFFIX=desktop-release" >> $GITHUB_ENV
echo "BUILD_SUFFIX=desktop-release-windows" >> $GITHUB_ENV
echo "TYPE=RELEASE" >> $GITHUB_ENV
echo "ZEPHYR_ENABLE=true" >> $GITHUB_ENV
echo "ZEPHYR_FOLDER_WIN_REPORT=12358651" >> $GITHUB_ENV
elif [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
echo "BRANCH=${{ github.ref }}" >> $GITHUB_ENV
echo "BUILD_SUFFIX=desktop-manual-trigger" >> $GITHUB_ENV
elif [ "${{ github.event_name }}" == "workflow_dispatch" ] && [ "${{ inputs.tag }}" == "" ]; then
echo "BUILD_SUFFIX=desktop-manual-trigger-windows" >> $GITHUB_ENV
echo "TYPE=MANUAL" >> $GITHUB_ENV
elif [ "${{ github.event_name }}" == "push" ] && [ "${{ github.ref }}" == "refs/heads/master" ]; then
echo "BRANCH=${{ github.ref }}" >> $GITHUB_ENV
echo "TYPE=MASTER" >> $GITHUB_ENV
echo "BUILD_SUFFIX=desktop-master-push" >> $GITHUB_ENV
echo "BUILD_SUFFIX=desktop-master-push-windows" >> $GITHUB_ENV
echo "ZEPHYR_ENABLE=true" >> $GITHUB_ENV
elif [ "${{ github.event_name }}" == "schedule" || "${{ github.event_name }}" == "workflow_call" ]; then
echo "BRANCH=${{ github.ref }}" >> $GITHUB_ENV
echo "BUILD_SUFFIX=desktop-nightly" >> $GITHUB_ENV
elif [ "${{ inputs.tag }}" != "" ]; then
echo "TYPE=NIGHTLY" >> $GITHUB_ENV
echo "BUILD_SUFFIX=desktop-nightly-windows" >> $GITHUB_ENV
echo "ZEPHYR_ENABLE=true" >> $GITHUB_ENV
echo "ZEPHYR_FOLDER_WIN_REPORT=12363690" >> $GITHUB_ENV
fi
- name: Set Build ID
run: echo "BUILD_ID=${{ github.run_id }}-${{ env.BUILD_SUFFIX }}" >> $GITHUB_ENV
shell: bash
- name: Run Playwright tests (Windows OS)
run: |
cat $GITHUB_ENV
npm run test:e2e || true
npm run test:e2e:send-report
shell: bash
- name: Analyze flaky tests
id: analyze-flaky-tests
uses: actions/github-script@v7
with:
script: |
const { analyzeFlakyTests } = require('./e2e/utils/analyze-flaky-test.js');
const { newFailedTests } = analyzeFlakyTests();
if (newFailedTests.length === 0) {
process.env.NEW_FAILED_TESTS = 'true';
}
- name: Add results in PR comment
if: github.event_name == 'pull_request'
uses: actions/github-script@v7
with:
script: |
const { analyzeFlakyTests } = require('./e2e/utils/analyze-flaky-test.js');
const { commentBody } = analyzeFlakyTests();
github.rest.issues.createComment({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
body: commentBody,
});
- name: Mark job as failed if there are new failed tests
if: env.NEW_FAILED_TESTS == 'true'
run: exit 1
shell: bash
e2e-remove-label:
if: ${{ github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'Run Desktop E2E Tests') }}
if: ${{ always() && contains(github.event.pull_request.labels.*.name, 'Run Desktop E2E Tests') }}
needs: [e2e-linux, e2e-macos, e2e-windows]
runs-on: ubuntu-latest
steps:
- name: Remove "Run Desktop E2E Tests" label
uses: actions-ecosystem/action-remove-labels@v1
with:
labels: |
Run Desktop E2E Tests
labels: Run Desktop E2E Tests