Modify run conditions for nightly e2e job to create test cycle and post in channel. (#2971)

e2e/utils/analyze-flaky-test.js

@@ -0,0 +1,81 @@
+// Copyright (c) 2016-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+/* eslint-disable no-console */
+const path = require('path');
+
+const {MOCHAWESOME_REPORT_DIR} = require('./constants');
+const knownFlakyTests = require('./known_flaky_tests.json');
+const {
+    generateShortSummary,
+    readJsonFromFile,
+} = require('./report');
+
+function analyzeFlakyTests() {
+    const os = process.platform;
+    try {
+        // Import
+        const jsonReport = readJsonFromFile(path.join(MOCHAWESOME_REPORT_DIR, 'mochawesome.json'));
+
+        const {failedFullTitles} = generateShortSummary(jsonReport);
+
+        // Get the list of known flaky tests for the provided operating system
+        const knownFlakyTestsForOS = new Set(knownFlakyTests[os] || []);
+
+        // Filter out the known flaky tests from the failed test titles
+        const newFailedTests = failedFullTitles.filter((test) => !knownFlakyTestsForOS.has(test));
+
+        // Check if any known failed tests are fixed
+        const fixedTests = [...knownFlakyTestsForOS].filter((test) => !failedFullTitles.includes(test));
+
+        const commentBody = generateCommentBodyFunctionalTest(newFailedTests, fixedTests);
+
+        // Print on CI
+        console.log(commentBody);
+
+        return {commentBody, newFailedTests};
+    } catch (error) {
+        console.error('Error analyzing failures:', error);
+    }
+}
+
+function generateCommentBodyFunctionalTest(newFailedTests, fixedTests) {
+    const osName = process.env.RUNNER_OS;
+    const build = process.env.BUILD_TAG;
+
+    let commentBody = `
+    ## Test Summary for ${osName} on commit ${build}
+`;
+
+    if (newFailedTests.length === 0 && fixedTests.length === 0) {
+        commentBody += `
+        All stable tests passed on ${osName}.
+    `;
+        return commentBody;
+    }
+
+    if (newFailedTests.length > 0) {
+        const newTestFailure = `New failed tests found on ${osName}:\n\t${newFailedTests.map((test) => `- ${test}`).join('\n\t')}`;
+        commentBody += `
+    ### New Failed Tests
+    
+    | Test |
+    | --- |
+       ${newTestFailure}
+    `;
+    }
+
+    if (fixedTests.length > 0) {
+        const fixedTestMessage = `The following known failed tests have been fixed on ${osName}:\n\t${fixedTests.map((test) => `- ${test}`).join('\n\t')}`;
+        commentBody += `
+    ### Fixed Tests
+    
+    ${fixedTestMessage}
+    `;
+    }
+
+    return commentBody;
+}
+
+module.exports = {
+    analyzeFlakyTests,
+};

e2e/utils/known_flaky_tests.json

@@ -0,0 +1,14 @@
+{
+    "darwin": [
+        "popup MM-T2827_1 should be able to select all in popup windows"
+    ],
+    "linux": [
+        "menu/view MM-T820 should open Developer Tools For Application Wrapper for main window",
+        "Menu/window_menu MM-T824 should be minimized when keyboard shortcuts are pressed",
+        "Menu/window_menu MM-T825 should be hidden when keyboard shortcuts are pressed",
+        "header MM-T2637 Double-Clicking on the header should minimize/maximize the app MM-T2637_1 should maximize on double-clicking the header"
+    ],
+    "win32": [
+        "application MM-T1304/MM-T1306 should open the app on the requested deep link"
+    ]
+}

e2e/utils/pr-e2e-durations-report.js

@@ -1,7 +1,7 @@
 // Copyright (c) 2016-present Mattermost, Inc. All Rights Reserved.
 // See LICENSE.txt for license information.
 
-function generateCommentBody(fileContents) {
+function generateCommentBodyPerformanceTest(fileContents) {
     const data = JSON.parse(fileContents);
 
     return `
@@ -32,5 +32,5 @@ ${fileContents}
 }
 
 module.exports = {
-    generateCommentBody,
+    generateCommentBodyPerformanceTest,
 };

e2e/utils/report.js

@@ -76,6 +76,7 @@ function generateShortSummary(report) {
     return {
         stats,
         statsFieldValue,
+        failedFullTitles,
     };
 }
 

e2e/utils/test_cases.js

@@ -5,6 +5,8 @@
 
 // See reference: https://support.smartbear.com/tm4j-cloud/api-docs/
 
+const os = require('os');
+
 const axios = require('axios');
 const chalk = require('chalk');
 
@@ -83,13 +85,50 @@ function saveToEndpoint(url, data) {
     });
 }
 
+async function getZEPHYRFolderID() {
+    const {
+        TYPE,
+        ZEPHYR_FOLDER_ID,
+        ZEPHYR_FOLDER_LINUX_REPORT,
+        ZEPHYR_FOLDER_MACOS_REPORT,
+        ZEPHYR_FOLDER_WIN_REPORT,
+    } = process.env;
+    if (TYPE === 'MASTER') {
+        return ZEPHYR_FOLDER_ID;
+    }
+    const platform = os.platform();
+
+    // Define Zephyr folder IDs for different run types and platforms.
+    // For PR we dont generate reports.
+    // Post Merge to master branch, default folderID will be used.
+    const folderIDs = {
+        RELEASE: {
+            darwin: ZEPHYR_FOLDER_MACOS_REPORT,
+            win32: ZEPHYR_FOLDER_WIN_REPORT,
+            linux: ZEPHYR_FOLDER_LINUX_REPORT,
+            default: ZEPHYR_FOLDER_ID,
+        },
+        NIGHTLY: {
+            darwin: ZEPHYR_FOLDER_MACOS_REPORT,
+            win32: ZEPHYR_FOLDER_WIN_REPORT,
+            linux: ZEPHYR_FOLDER_LINUX_REPORT,
+            default: ZEPHYR_FOLDER_ID,
+        },
+    };
+
+    // Get the folder ID based on the type and platform
+    const typeFolderIDs = folderIDs[TYPE];
+    const folderID = typeFolderIDs?.[platform] ?? typeFolderIDs?.default ?? ZEPHYR_FOLDER_ID;
+
+    return folderID;
+}
+
 async function createTestCycle(startDate, endDate) {
     const {
         BRANCH,
         BUILD_ID,
         JIRA_PROJECT_KEY,
         ZEPHYR_CYCLE_NAME,
-        ZEPHYR_FOLDER_ID,
     } = process.env;
 
     const testCycle = {
@@ -99,7 +138,7 @@ async function createTestCycle(startDate, endDate) {
         plannedStartDate: startDate,
         plannedEndDate: endDate,
         statusName: 'Done',
-        folderId: ZEPHYR_FOLDER_ID,
+        folderId: await getZEPHYRFolderID(),
     };
 
     const response = await saveToEndpoint('https://api.zephyrscale.smartbear.com/v2/testcycles', testCycle);