[MM-46305] Add reporting code for E2E tests (#2222)

* E2E reporting

* Test report

* Fix artifacts.js

* Revert "Test report"

This reverts commit f4d44b881a19c0e9d63066807f5cb6b9fe9017ee.

* PR feedback

e2e/utils/test_cases.js

@@ -0,0 +1,202 @@
+// Copyright (c) 2016-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+
+/* eslint-disable no-console */
+
+// See reference: https://support.smartbear.com/tm4j-cloud/api-docs/
+
+const axios = require('axios');
+const chalk = require('chalk');
+
+const {getAllTests} = require('./report');
+
+const status = {
+    passed: 'Pass',
+    failed: 'Fail',
+    pending: 'Pending',
+    skipped: 'Skip',
+};
+
+const environment = {
+    chrome: 'Chrome',
+    firefox: 'Firefox',
+};
+
+function getStepStateResult(steps = []) {
+    return steps.reduce((acc, item) => {
+        if (acc[item.state]) {
+            acc[item.state] += 1;
+        } else {
+            acc[item.state] = 1;
+        }
+
+        return acc;
+    }, {});
+}
+
+function getStepStateSummary(steps = []) {
+    const result = getStepStateResult(steps);
+
+    return Object.entries(result).map(([key, value]) => `${value} ${key}`).join(',');
+}
+
+function getZEPHYRTestCases(report) {
+    return getAllTests(report.results).
+        filter((item) => /^(MM-T)\w+/g.test(item.title)). // eslint-disable-line wrap-regex
+        map((item) => {
+            return {
+                title: item.title,
+                duration: item.duration,
+                incrementalDuration: item.incrementalDuration,
+                state: item.state,
+                pass: item.pass,
+                fail: item.fail,
+                pending: item.pending,
+            };
+        }).
+        reduce((acc, item) => {
+            // Extract the key to exactly match with "MM-T[0-9]+"
+            const key = item.title.match(/(MM-T\d+)/)[0];
+
+            if (acc[key]) {
+                acc[key].push(item);
+            } else {
+                acc[key] = [item];
+            }
+
+            return acc;
+        }, {});
+}
+
+function saveToEndpoint(url, data) {
+    return axios({
+        method: 'POST',
+        url,
+        headers: {
+            'Content-Type': 'application/json; charset=utf-8',
+            Authorization: process.env.ZEPHYR_API_KEY,
+        },
+        data,
+    }).catch((error) => {
+        console.log('Something went wrong:', error.response.data.message);
+        return error.response.data;
+    });
+}
+
+async function createTestCycle(startDate, endDate) {
+    const {
+        BRANCH,
+        BUILD_ID,
+        JIRA_PROJECT_KEY,
+        ZEPHYR_CYCLE_NAME,
+        ZEPHYR_FOLDER_ID,
+    } = process.env;
+
+    const testCycle = {
+        projectKey: JIRA_PROJECT_KEY,
+        name: ZEPHYR_CYCLE_NAME ? `${ZEPHYR_CYCLE_NAME} (${BUILD_ID}-${BRANCH})` : `${BUILD_ID}-${BRANCH}`,
+        description: `Cypress automated test with ${BRANCH}`,
+        plannedStartDate: startDate,
+        plannedEndDate: endDate,
+        statusName: 'Done',
+        folderId: ZEPHYR_FOLDER_ID,
+    };
+
+    const response = await saveToEndpoint('https://api.zephyrscale.smartbear.com/v2/testcycles', testCycle);
+    return response.data;
+}
+
+async function createTestExecutions(report, testCycle) {
+    const {
+        BROWSER,
+        JIRA_PROJECT_KEY,
+        ZEPHYR_ENVIRONMENT_NAME,
+    } = process.env;
+
+    const testCases = getZEPHYRTestCases(report);
+    const startDate = new Date(report.stats.start);
+    const startTime = startDate.getTime();
+
+    const promises = [];
+    Object.entries(testCases).forEach(([key, steps], index) => {
+        const testScriptResults = steps.
+            sort((a, b) => a.title.localeCompare(b.title)).
+            map((item) => {
+                return {
+                    statusName: status[item.state],
+                    actualEndDate: new Date(startTime + item.incrementalDuration).toISOString(),
+                    actualResult: 'Cypress automated test completed',
+                };
+            });
+
+        const stateResult = getStepStateResult(steps);
+
+        const testExecution = {
+            projectKey: JIRA_PROJECT_KEY,
+            testCaseKey: key,
+            testCycleKey: testCycle.key,
+            statusName: stateResult.passed && stateResult.passed === steps.length ? 'Pass' : 'Fail',
+            testScriptResults,
+            environmentName: ZEPHYR_ENVIRONMENT_NAME || environment[BROWSER] || 'Chrome',
+            actualEndDate: testScriptResults[testScriptResults.length - 1].actualEndDate,
+            executionTime: steps.reduce((acc, prev) => {
+                acc += prev.duration; // eslint-disable-line no-param-reassign
+                return acc;
+            }, 0),
+            comment: `Cypress automated test - ${getStepStateSummary(steps)}`,
+        };
+
+        // Temporarily log to verify cases that were being saved.
+        console.log(index, key); // eslint-disable-line no-console
+
+        promises.push(saveTestExecution(testExecution, index));
+    });
+
+    await Promise.all(promises);
+    console.log('Successfully saved test cases into the Test Management System');
+}
+
+const saveTestCases = async (allReport) => {
+    const {start, end} = allReport.stats;
+
+    const testCycle = await createTestCycle(start, end);
+
+    await createTestExecutions(allReport, testCycle);
+};
+
+const RETRY = [];
+
+async function saveTestExecution(testExecution, index) {
+    await axios({
+        method: 'POST',
+        url: 'https://api.zephyrscale.smartbear.com/v2/testexecutions',
+        headers: {
+            'Content-Type': 'application/json; charset=utf-8',
+            Authorization: process.env.ZEPHYR_API_KEY,
+        },
+        data: testExecution,
+    }).then(() => {
+        console.log(chalk.green('Success:', index, testExecution.testCaseKey));
+    }).catch((error) => {
+        // Retry on 500 error code / internal server error
+        if (!error.response || error.response.data.errorCode === 500) {
+            if (RETRY[testExecution.testCaseKey]) {
+                RETRY[testExecution.testCaseKey] += 1;
+            } else {
+                RETRY[testExecution.testCaseKey] = 1;
+            }
+
+            saveTestExecution(testExecution, index);
+            console.log(chalk.magenta('Retry:', index, testExecution.testCaseKey, `(${RETRY[testExecution.testCaseKey]}x)`));
+        } else {
+            console.log(chalk.red('Error:', index, testExecution.testCaseKey, error.response.data.message));
+        }
+    });
+}
+
+module.exports = {
+    createTestCycle,
+    saveTestCases,
+    createTestExecutions,
+};