From 059ae79ce2479819307437af439f4f9909f712e6 Mon Sep 17 00:00:00 2001 From: eduponz Date: Wed, 12 Jun 2024 15:44:50 +0200 Subject: [PATCH] Refs #20843: Migrate to flakiness_report action Signed-off-by: eduponz --- .github/resources/check_flakes.py | 384 ------------------ .github/resources/ctest-to-junit-reducted.xsl | 36 -- .github/resources/ctest2junit.py | 156 ------- .github/workflows/nightly-ubuntu-ci.yml | 1 + .../workflows/nightly-ubuntu-detect-flaky.yml | 135 ------ .github/workflows/reusable-mac-ci.yml | 16 + .github/workflows/reusable-ubuntu-ci.yml | 46 ++- .github/workflows/reusable-windows-ci.yml | 16 + 8 files changed, 64 insertions(+), 726 deletions(-) delete mode 100644 .github/resources/check_flakes.py delete mode 100644 .github/resources/ctest-to-junit-reducted.xsl delete mode 100644 .github/resources/ctest2junit.py delete mode 100644 .github/workflows/nightly-ubuntu-detect-flaky.yml diff --git a/.github/resources/check_flakes.py b/.github/resources/check_flakes.py deleted file mode 100644 index 9e13e08a663..00000000000 --- a/.github/resources/check_flakes.py +++ /dev/null @@ -1,384 +0,0 @@ -# Copyright 2024 Proyectos y Sistemas de Mantenimiento SL (eProsima). -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Script to analyze and publish flaky tests from JUnit test history files. -""" -import argparse -import os -import re -from datetime import datetime -from decimal import getcontext, Decimal, ROUND_UP -from typing import Dict - -from junitparser import TestSuite -import pandas as pd -import xml.etree.ElementTree as ET - - -class FlakyTestsPublisher: - """ - Class to analyze and publish flaky tests from JUnit test history files. - """ - # Regex pattern to match the timestamp part of the filename - _timestamp_rstr = r'\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}' - - def __init__(self, junit_files_dir: str, window_size: int, delete_old_files: bool = True): - """ - Args: - junit_files_dir (str): Path to the directory containing the JUnit test history files. - window_size (int): Number of test runs to consider for the fliprate calculation. - delete_old_files (bool): Whether to delete old files taking window_size into account. - """ - self._flip_rates = {} - self._window_size = window_size - self._junit_files = FlakyTestsPublisher._find_test_reports(junit_files_dir) - - if delete_old_files: - self._delete_old_files() - - def analyze(self) -> int: - """ - Analyze the test history files and calculate the fliprate for each test. - - Returns: - int: Number of flaky tests found. - """ - df = FlakyTestsPublisher._get_test_results(self._junit_files) - fliprate_table = FlakyTestsPublisher._calculate_fliprate_table(df, self._window_size) - self._flip_rates = FlakyTestsPublisher._get_top_fliprates(fliprate_table, 0, 4) - return len(self._flip_rates) - - def publish_summary(self, output_file: str): - """ - Publish the summary of the flaky tests found to a markdown file. - - Args: - output_file (str): Path to the output file. - """ - # Test summary - summary = '## Flaky tests\n' - - # Table header - # TODO(eduponz): Add a column for the failures/runs ratio - summary += '|#|Flaky tests|Fliprate score %|Consecutive failures|Consecutive passes|Total failures|\n' - summary += '|-|-|-|-|-|\n' - - i = 1 - for test_name, (flip_rate, consecutive_failures, consecutive_passes, failures) in self._flip_rates.items(): - summary += f'| {i} | {test_name} | {round(flip_rate, 2)*100} | {consecutive_failures} | {consecutive_passes} | {failures} |\n' - i += 1 - - with open(output_file, 'w') as file: - file.write(summary) - - @staticmethod - def _find_test_reports(directory): - """ - Find all JUnit test history files in the given directory. - - Args: - directory (str): Path to the directory containing the JUnit test history files. - - Returns: - list: List of paths to the JUnit test history files. - """ - pattern = re.compile(r'^test_report.*' + FlakyTestsPublisher._timestamp_rstr + r'\.xml$') - - files = os.listdir(directory) - matched_files = [ - os.path.join(directory, f) for f in files if pattern.match(f) - ] - matched_files.sort(key=FlakyTestsPublisher._extract_timestamp) - - return matched_files - - @staticmethod - def _extract_timestamp(file_path): - """ - Extract the timestamp from the filename. - - Args: - file_path (str): Path to the file. - - Returns: - datetime: Timestamp extracted from the filename. - """ - filename = os.path.basename(file_path) - timestamp_str = re.search(FlakyTestsPublisher._timestamp_rstr, filename).group() - return datetime.strptime(timestamp_str, '%Y-%m-%dT%H-%M-%S') - - def _delete_old_files(self): - """ - Delete old files taking window_size into account. - """ - if len(self._junit_files) > self._window_size: - # Calculate the number of files to delete - files_to_delete = self._junit_files[:-self._window_size] - - # Update the list of kept files - self._junit_files = self._junit_files[-self._window_size:] - - # Delete the older files - for file in files_to_delete: - print(f'Deleting old file: {file}') - os.remove(file) - - @staticmethod - def _get_test_results(junit_files: list): - """ - Parse JUnit test history files and return a DataFrame with the test results. - - Args: - junit_files (list): List of paths to the JUnit test history files. - - Returns: - pd.DataFrame: DataFrame with the test results. - - Raises: - RuntimeError: If no valid JUnit files are found. - """ - dataframe_entries = [] - - for junit_file in junit_files: - xml = ET.parse(junit_file) - root = xml.getroot() - for suite in root.findall('.//testsuite'): - dataframe_entries += FlakyTestsPublisher._junit_suite_to_df(suite) - - if dataframe_entries: - df = pd.DataFrame(dataframe_entries) - df["timestamp"] = pd.to_datetime(df["timestamp"]) - df = df.set_index("timestamp") - return df.sort_index() - else: - raise RuntimeError(f"No Junit files found in {junit_files}") - - @staticmethod - def _junit_suite_to_df(suite: TestSuite) -> list: - """ - Convert a JUnit test suite to a list of DataFrame entries. - - Args: - suite (TestSuite): JUnit test suite. - - Returns: - list: List of DataFrame entries. - """ - dataframe_entries = [] - time = suite.attrib.get('timestamp') - - for testcase in suite.findall('.//testcase'): - test_identifier = testcase.attrib.get('name') - - status = testcase.attrib.get('status') - - # Convert status to "pass" if it's "passed" - if status == "passed": - test_status = "pass" - else: - test_status = status - if test_status == "skipped": - continue - - dataframe_entries.append( - { - "timestamp": time, - "test_identifier": test_identifier, - "test_status": test_status, - } - ) - return dataframe_entries - - @staticmethod - def _calculate_fliprate_table(testrun_table: pd.DataFrame, window_size: int) -> pd.DataFrame: - """ - Calculate the fliprate for each test in the testrun_table. - - Args: - testrun_table (pd.DataFrame): DataFrame with the test results. - window_size (int): Number of test runs to consider for the fliprate calculation. - - Returns: - pd.DataFrame: DataFrame with the fliprates for each test. - """ - # Apply non_overlapping_window_fliprate to each group in testrun_table - fliprates = testrun_table.groupby("test_identifier")["test_status"].apply( - lambda x: FlakyTestsPublisher._non_overlapping_window_fliprate(x, window_size) - ) - - # Convert fliprates Series of DataFrames to a DataFrame - fliprate_table = fliprates.reset_index() - - # Rename the index level to "window" - fliprate_table = fliprate_table.rename(columns={"level_1": "window"}) - - # Filter out rows where flip_rate is zero - # TODO(eduponz): I'm seeing tests with 5 consecutive failures that are not showing here. - # Seems it's because they are not flaky, they are just always failing. - fliprate_table = fliprate_table[fliprate_table.flip_rate != 0] - - return fliprate_table - - @staticmethod - def _non_overlapping_window_fliprate(testruns: pd.Series, window_size: int) -> pd.Series: - """ - Calculate the fliprate for a test in a non-overlapping window. - - Args: - testruns (pd.Series): Series with the test results. - window_size (int): Number of test runs to consider for the fliprate calculation. - - Returns: - pd.Series: Series with the fliprate for the test. - """ - # Apply _calc_fliprate directly to the last selected rows - testruns_last = testruns.iloc[-window_size:] - fliprate_groups = FlakyTestsPublisher._calc_fliprate(testruns_last) - - return fliprate_groups.reset_index(drop=True) - - @staticmethod - def _calc_fliprate(testruns: pd.Series) -> pd.DataFrame: - """ - Calculate the fliprate for a test. - - Args: - testruns (pd.Series): Series with the test results. - - Returns: - pd.DataFrame: DataFrame with the fliprate, consecutive failures - and consecutive passes for the test. - """ - if len(testruns) < 2: - return pd.DataFrame( - { - 'flip_rate': [0.0], - 'consecutive_failures': [0], - 'consecutive_passes': [0], - 'failures': [0], - } - ) - - first = True - previous = None - flips = 0 - consecutive_failures = 0 - consecutive_passes = 0 - failures = 0 - possible_flips = len(testruns) - 1 - - for _, val in testruns.items(): - if first: - first = False - previous = val - continue - - if val != previous: - flips += 1 - - if val != "pass": - consecutive_failures += 1 - consecutive_passes = 0 - failures += 1 - else: - consecutive_failures = 0 - consecutive_passes += 1 - - previous = val - - flip_rate = flips / possible_flips - - return pd.DataFrame( - { - 'flip_rate': [flip_rate], - 'consecutive_failures': [consecutive_failures], - 'consecutive_passes': [consecutive_passes], - 'failures': [failures], - } - ) - - @staticmethod - def _get_top_fliprates(fliprate_table: pd.DataFrame, top_n: int, precision: int) -> Dict[str, tuple[Decimal, int, int, int]]: - """ - Get the top N fliprates from the fliprate_table. - - Args: - fliprate_table (pd.DataFrame): DataFrame with the fliprates. - top_n (int): Number of top fliprates to get. - precision (int): Number of decimal places to round the fliprates. - - Returns: - Dict[str, tuple[Decimal, int, int, int]]: Dictionary with the top fliprates - and their consecutive and total failures. - """ - context = getcontext() - context.prec = precision - context.rounding = ROUND_UP - last_window_values = fliprate_table.groupby("test_identifier").last() - - if top_n != 0: - top_fliprates = last_window_values.nlargest(top_n, "flip_rate") - else : - top_fliprates = last_window_values.nlargest(len(last_window_values), "flip_rate") - - # Create a dictionary with test_identifier as keys and a tuple of flip_rate and consecutive_failures as values - results = {} - for test_id, row in top_fliprates.iterrows(): - results[test_id] = ( - Decimal(row["flip_rate"]), - int(row["consecutive_failures"]), - int(row["consecutive_passes"]), - int(row["failures"]) - ) - - return results - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - group = parser.add_mutually_exclusive_group(required=True) - group.add_argument( - "--junit-files", - help="Path for a folder with JUnit xml test history files", - type=str - ) - parser.add_argument( - "--window-size", - type=int, - help="flip rate calculation window size", - required=True, - ) - parser.add_argument( - '-d', - '--delete-old-files', - action='store_true', - help='Delete old files taking window size into account.' - ) - parser.add_argument( - '-o', - '--output-file', - type=str, - required=False, - help='Path to output file.' - ) - args = parser.parse_args() - - publisher = FlakyTestsPublisher(args.junit_files, args.window_size, args.delete_old_files) - flaky_test_count = publisher.analyze() - - if args.output_file: - publisher.publish_summary(args.output_file) - - ret = 0 if flaky_test_count == 0 else 1 - exit(ret) diff --git a/.github/resources/ctest-to-junit-reducted.xsl b/.github/resources/ctest-to-junit-reducted.xsl deleted file mode 100644 index 4f2fa7d988e..00000000000 --- a/.github/resources/ctest-to-junit-reducted.xsl +++ /dev/null @@ -1,36 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/.github/resources/ctest2junit.py b/.github/resources/ctest2junit.py deleted file mode 100644 index b98086060ee..00000000000 --- a/.github/resources/ctest2junit.py +++ /dev/null @@ -1,156 +0,0 @@ -# Copyright 2024 Proyectos y Sistemas de Mantenimiento SL (eProsima). -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Script to transform CTest native XML reports to jUnit if they do not exist""" - -import argparse - -from lxml import etree -import os - -DESCRIPTION = """Script to transform CTest native XML reports to jUnit""" -USAGE = ('python3 ctest2junit.py') - - -def parse_options(): - """ - Parse arguments. - :return: The arguments parsed. - """ - parser = argparse.ArgumentParser( - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - add_help=True, - description=(DESCRIPTION), - usage=(USAGE) - ) - - required_args = parser.add_argument_group('required arguments') - required_args.add_argument( - '-b', - '--build-dir', - type=str, - required=True, - help='Path to build directory' - ) - required_args.add_argument( - '-x', - '--xslt', - type=str, - required=True, - help='Path to XSLT translation file.' - ) - required_args.add_argument( - '-o', - '--output-junit', - type=str, - required=True, - help='Path to output jUnit file. If the file exists, the script takes no action' - ) - required_args.add_argument( - '-t', - '--timestamp', - type=str, - required=True, - help='Timestamp' - ) - return parser.parse_args() - - -def find_ctest_report(build_dir): - """ - Find the last ctest report in a given CMake build directory. - - :param build_dir: The CMake build directory - - :return: The path to latests CTest report on build_dir if any; else None - """ - ret = None - testing_dir = os.path.join(build_dir, 'Testing') - - # Check if build_dir/Testing exists - if os.path.exists(testing_dir): - tag_file = os.path.join(testing_dir, 'TAG') - - # CTest generates the reports if build_dir/Testing/{label}/Test.xml, - # where {label} is specified in the first line of a file build_dir/Testing/TAG - # Look for that build_dir/Testing/{label}/Test.xml. - if os.path.isfile(tag_file): - # Extract the {label} from the TAG file - with open(tag_file, 'r') as f: - tag_content = f.readline().strip() - test_dir = os.path.join(testing_dir, tag_content) - - # Check that Test.xml exists - test_xml = os.path.join(test_dir, 'Test.xml') - - if os.path.isfile(test_xml): - ret = test_xml - - return ret - - -def translate(original_xml, xsl_file, timestamp): - """ - Translate an XML from one spec to another using an XSLT file. - - :param original_xml: The XML to translate - :param xsl_file: The XSLT transformation file - - :return: A stream containing the translated XML - """ - var = timestamp - xml = etree.parse(original_xml) - xslt = etree.parse(xsl_file) - transform = etree.XSLT(xslt) - - try: - # Apply the transformation with the new context - result_tree = transform(xml, var1=etree.XSLT.strparam(var)) - return str(result_tree) - except Exception as e: - for error in transform.error_log: - print(error.message, error.line) - raise e - - -def write_to_file(stream, filename): - """ - Write a stream to a file. - - :param stream: The stream to write - :param filename: The destination file - """ - with open(filename, 'w') as f: - f.write(stream) - - -if __name__ == '__main__': - - args = parse_options() - print(args.timestamp) - exit_code = 0 - - if os.path.isfile(args.output_junit): - print(f'File {args.output_junit} already exists. No action taken.') - - else: - exit_code = 1 - ctest_report = find_ctest_report(args.build_dir) - - if ctest_report: - junit = translate(ctest_report, args.xslt, args.timestamp) - write_to_file(junit, args.output_junit) - exit_code = 0 - - exit(exit_code) diff --git a/.github/workflows/nightly-ubuntu-ci.yml b/.github/workflows/nightly-ubuntu-ci.yml index b4216f87349..e42cd9ecd3e 100644 --- a/.github/workflows/nightly-ubuntu-ci.yml +++ b/.github/workflows/nightly-ubuntu-ci.yml @@ -22,6 +22,7 @@ jobs: cmake-args: "-DSECURITY=${{ matrix.security }}" ctest-args: "-LE xfail" fastdds-branch: 'master' + flakiness-report: true nightly-ubuntu-ci-2_14_x: strategy: diff --git a/.github/workflows/nightly-ubuntu-detect-flaky.yml b/.github/workflows/nightly-ubuntu-detect-flaky.yml deleted file mode 100644 index 22931b56116..00000000000 --- a/.github/workflows/nightly-ubuntu-detect-flaky.yml +++ /dev/null @@ -1,135 +0,0 @@ -name: Detect flakiness Ubuntu workflow - -on: - schedule: - - cron: '0 1 * * *' - pull_request: - types: [opened, synchronize] - workflow_dispatch: - inputs: - label: - description: 'ID associated to the workflow' - required: false - type: string - colcon-args: - description: 'Extra arguments for colcon cli' - required: false - type: string - -defaults: - run: - shell: bash - -jobs: - test-ci: - runs-on: 'ubuntu-22.04' - strategy: - fail-fast: false - matrix: - fastdds-branch: - - 'feature/detect_flaky_tests' - cmake-build-type: - - 'RelWithDebInfo' - steps: - - - name: Sync eProsima/Fast-DDS repository - uses: eProsima/eProsima-CI/external/checkout@v0 - with: - path: src/fastdds - ref: ${{ matrix.fastdds-branch }} - - - name: Get minimum supported version of CMake - uses: eProsima/eProsima-CI/external/get-cmake@v0 - with: - cmakeVersion: '3.22.6' - - - name: Install apt dependencies - uses: eProsima/eProsima-CI/ubuntu/install_apt_packages@v0 - with: - packages: libasio-dev libtinyxml2-dev libssl-dev - update: false - upgrade: false - - - name: Install colcon - uses: eProsima/eProsima-CI/ubuntu/install_colcon@v0 - - - name: Install Python dependencies - run: | - pip install pytest pytest-json-report requests vcstool lxml junitparser pandas numpy seaborn - - - name: Setup CCache - uses: eProsima/eProsima-CI/external/setup-ccache-action@v0 - - - name: Fetch Fast DDS dependencies - uses: eProsima/eProsima-CI/multiplatform/vcs_import@v0 - with: - vcs_repos_file: ${{ github.workspace }}/src/fastdds/fastdds.repos - destination_workspace: src - skip_existing: 'true' - - - name: Fetch googletest - uses: eProsima/eProsima-CI/multiplatform/vcs_import@v0 - with: - vcs_repos_file: ${{ github.workspace }}/src/fastdds/.github/workflows/config/ci.repos - destination_workspace: src - skip_existing: 'true' - - - name: Colcon build - continue-on-error: false - uses: eProsima/eProsima-CI/multiplatform/colcon_build@v0 - with: - colcon_meta_file: ${{ github.workspace }}/src/fastdds/.github/workflows/config/ci.meta - colcon_build_args: ${{ inputs.colcon-args }} - cmake_args: -DSECURITY=ON - cmake_args_default: -DCMAKE_CXX_FLAGS="-Werror -Wall -Wextra -Wpedantic -Wunused-value -Woverloaded-virtual" - cmake_build_type: ${{ matrix.cmake-build-type }} - workspace: ${{ github.workspace }} - - - name: Set current date as env variable - id: date - run: | - echo "TODAY=$(date +'%Y-%m-%dT%H-%M-%S')" >> $GITHUB_ENV - echo "NOW=$(date +'%Y-%m-%dT%H:%M:%S')" >> $GITHUB_ENV - - - name: Colcon test - run: | - cd ${{ github.workspace }} - test_results_file="${{ github.workspace }}/test_report-$TODAY.xml" - colcon test --packages-select=fastdds --event-handlers=console_direct+ --ctest-args --timeout=300 --label-exclude="xfail" - # Translate CTest XML reports to jUnit - if [ ! -f ${test_results_file} ] - then - echo "::group::Translating test report to jUnit specific format" - python3 ${{ github.workspace }}/src/fastdds/.github/resources/ctest2junit.py \ - --build-dir ${{ github.workspace }}/build/fastdds \ - --xslt ${{ github.workspace }}/src/fastdds/.github/resources/ctest-to-junit-reducted.xsl \ - --output-junit ${test_results_file} \ - --timestamp "$NOW" - echo "::endgroup::" - fi - - - name: Download previous test results with plugin - if: always() - uses: dawidd6/action-download-artifact@v2 - with: - workflow: nightly-ubuntu-detect-flaky.yml - name: test-results-all - workflow_conclusion: completed - check_artifacts: true - search_artifacts: true - - - name: Flaky test analysis on [window-size] runs history - id: flaky_analysis - run: | - # CHANGE --window-size TO THE DESIRED NUMBER OF RUNS - python ${{ github.workspace }}/src/fastdds/.github/resources/check_flakes.py --junit-files=. --window-size=30 --delete-old-files --output-file=$GITHUB_STEP_SUMMARY - # Do not fail the workflow if there are flaky tests - exit 0 - - - name: Upload test results - if: always() - uses: actions/upload-artifact@v2 - with: - name: test-results-all - path: | - *.xml diff --git a/.github/workflows/reusable-mac-ci.yml b/.github/workflows/reusable-mac-ci.yml index 93c9886dd59..c2259d0a5a0 100644 --- a/.github/workflows/reusable-mac-ci.yml +++ b/.github/workflows/reusable-mac-ci.yml @@ -23,6 +23,11 @@ on: description: 'Branch or tag of Fast DDS repository (https://github.com/eProsima/Fast-DDS)' required: true type: string + flakiness-report: + description: 'Whether to generate a flakiness report' + required: false + type: boolean + default: false defaults: run: @@ -145,6 +150,17 @@ jobs: workspace: ${{ github.workspace }} test_report_artifact: ${{ format('test_report_{0}_{1}_{2}', inputs.label, github.job, join(matrix.*, '_')) }} + - name: Flakiness report + uses: eProsima/eProsima-CI/multiplatform/flakiness_report@feature/detect_flaky_tests + if: ${{ inputs.flakiness-report }} + with: + junit_reports_dir: "${{ steps.test.outputs.ctest_results_path }}" + junit_archive_artifact: ${{ format('flakiness_archive_{0}_{1}_{2}', inputs.label, github.job, join(matrix.*, '_')) }} + print_report: 'True' + window_size: '30' + fail_on_flaky_tests: 'False' + github_token: ${{ secrets.GITHUB_TOKEN }} + - name: Test summary uses: eProsima/eProsima-CI/multiplatform/junit_summary@v0 if: ${{ !cancelled() && !contains(github.event.pull_request.labels.*.name, 'no-test') }} diff --git a/.github/workflows/reusable-ubuntu-ci.yml b/.github/workflows/reusable-ubuntu-ci.yml index 0c1c44024cb..e38cb62fbc2 100644 --- a/.github/workflows/reusable-ubuntu-ci.yml +++ b/.github/workflows/reusable-ubuntu-ci.yml @@ -27,6 +27,11 @@ on: description: 'Branch or tag of Fast DDS repository (https://github.com/eProsima/Fast-DDS)' required: true type: string + flakiness-report: + description: 'Whether to generate a flakiness report' + required: false + type: boolean + default: false defaults: run: @@ -44,46 +49,46 @@ jobs: steps: - name: Add ci-pending label if PR if: ${{ github.event_name == 'pull_request' }} - uses: eProsima/eProsima-CI/external/add_labels@v0 + uses: eProsima/eProsima-CI/external/add_labels@feature/detect_flaky_tests with: labels: ci-pending number: ${{ github.event.number }} repo: eProsima/Fast-DDS - name: Sync eProsima/Fast-DDS repository - uses: eProsima/eProsima-CI/external/checkout@v0 + uses: eProsima/eProsima-CI/external/checkout@feature/detect_flaky_tests with: path: src/fastdds ref: ${{ inputs.fastdds-branch }} - name: Install Fix Python version - uses: eProsima/eProsima-CI/external/setup-python@v0 + uses: eProsima/eProsima-CI/external/setup-python@feature/detect_flaky_tests with: python-version: '3.11' - name: Get minimum supported version of CMake - uses: eProsima/eProsima-CI/external/get-cmake@v0 + uses: eProsima/eProsima-CI/external/get-cmake@feature/detect_flaky_tests with: cmakeVersion: '3.22.6' - name: Install apt dependencies - uses: eProsima/eProsima-CI/ubuntu/install_apt_packages@v0 + uses: eProsima/eProsima-CI/ubuntu/install_apt_packages@feature/detect_flaky_tests with: packages: libasio-dev libtinyxml2-dev libssl-dev update: false upgrade: false - name: Install colcon - uses: eProsima/eProsima-CI/ubuntu/install_colcon@v0 + uses: eProsima/eProsima-CI/ubuntu/install_colcon@feature/detect_flaky_tests - name: Install Python dependencies - uses: eProsima/eProsima-CI/multiplatform/install_python_packages@v0 + uses: eProsima/eProsima-CI/multiplatform/install_python_packages@feature/detect_flaky_tests with: packages: vcstool xmlschema upgrade: false - name: Setup CCache - uses: eProsima/eProsima-CI/external/setup-ccache-action@v0 + uses: eProsima/eProsima-CI/external/setup-ccache-action@feature/detect_flaky_tests with: api_token: ${{ secrets.GITHUB_TOKEN }} @@ -103,14 +108,14 @@ jobs: # which entails logout/login or rebooting the machine. This is not feasible in a CI environment. - name: Fetch Fast DDS dependencies - uses: eProsima/eProsima-CI/multiplatform/vcs_import@v0 + uses: eProsima/eProsima-CI/multiplatform/vcs_import@feature/detect_flaky_tests with: vcs_repos_file: ${{ github.workspace }}/src/fastdds/fastdds.repos destination_workspace: src skip_existing: 'true' - name: Fetch Fast DDS CI dependencies - uses: eProsima/eProsima-CI/multiplatform/vcs_import@v0 + uses: eProsima/eProsima-CI/multiplatform/vcs_import@feature/detect_flaky_tests with: vcs_repos_file: ${{ github.workspace }}/src/fastdds/.github/workflows/config/ci.repos destination_workspace: src @@ -119,13 +124,13 @@ jobs: # Get Shapes Demo to make sure it keeps compiling - name: Get Shapes Demo branch id: get_shapes_demo_branch - uses: eProsima/eProsima-CI/ubuntu/get_related_branch_from_repo@v0 + uses: eProsima/eProsima-CI/ubuntu/get_related_branch_from_repo@feature/detect_flaky_tests with: remote_repository: eProsima/ShapesDemo fallback_branch: 'master' - name: Download Shapes Demo repo - uses: eProsima/eProsima-CI/external/checkout@v0 + uses: eProsima/eProsima-CI/external/checkout@feature/detect_flaky_tests with: repository: eProsima/ShapesDemo path: src/shapes-demo @@ -144,7 +149,7 @@ jobs: - name: Colcon build continue-on-error: false - uses: eProsima/eProsima-CI/multiplatform/colcon_build@v0 + uses: eProsima/eProsima-CI/multiplatform/colcon_build@feature/detect_flaky_tests with: colcon_meta_file: ${{ github.workspace }}/src/fastdds/.github/workflows/config/ci.meta colcon_build_args: ${{ inputs.colcon-args }} @@ -156,7 +161,7 @@ jobs: - name: Colcon test id: test if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-test') }} - uses: eProsima/eProsima-CI/multiplatform/colcon_test@v0 + uses: eProsima/eProsima-CI/multiplatform/colcon_test@feature/detect_flaky_tests with: colcon_test_args: ${{ inputs.colcon-args }} colcon_test_args_default: --event-handlers=console_direct+ @@ -166,8 +171,19 @@ jobs: workspace: ${{ github.workspace }} test_report_artifact: ${{ format('test_report_{0}_{1}_{2}', inputs.label, github.job, join(matrix.*, '_')) }} + - name: Flakiness report + uses: eProsima/eProsima-CI/multiplatform/flakiness_report@feature/detect_flaky_tests + if: ${{ inputs.flakiness-report }} + with: + junit_reports_dir: "${{ steps.test.outputs.ctest_results_path }}" + junit_archive_artifact: ${{ format('flakiness_archive_{0}_{1}_{2}', inputs.label, github.job, join(matrix.*, '_')) }} + print_report: 'True' + window_size: '30' + fail_on_flaky_tests: 'False' + github_token: ${{ secrets.GITHUB_TOKEN }} + - name: Test summary - uses: eProsima/eProsima-CI/multiplatform/junit_summary@v0 + uses: eProsima/eProsima-CI/multiplatform/junit_summary@feature/detect_flaky_tests if: ${{ !cancelled() && !contains(github.event.pull_request.labels.*.name, 'no-test') }} with: junit_reports_dir: "${{ steps.test.outputs.ctest_results_path }}" diff --git a/.github/workflows/reusable-windows-ci.yml b/.github/workflows/reusable-windows-ci.yml index b9fed900e37..ee26a630d7d 100644 --- a/.github/workflows/reusable-windows-ci.yml +++ b/.github/workflows/reusable-windows-ci.yml @@ -23,6 +23,11 @@ on: description: 'Branch or tag of Fast DDS repository (https://github.com/eProsima/Fast-DDS)' required: true type: string + flakiness-report: + description: 'Whether to generate a flakiness report' + required: false + type: boolean + default: false defaults: run: @@ -167,6 +172,17 @@ jobs: workspace: ${{ github.workspace }} test_report_artifact: ${{ format('test_report_{0}_{1}_{2}', inputs.label, github.job, join(matrix.*, '_')) }} + - name: Flakiness report + uses: eProsima/eProsima-CI/multiplatform/flakiness_report@feature/detect_flaky_tests + if: ${{ inputs.flakiness-report }} + with: + junit_reports_dir: "${{ steps.test.outputs.ctest_results_path }}" + junit_archive_artifact: ${{ format('flakiness_archive_{0}_{1}_{2}', inputs.label, github.job, join(matrix.*, '_')) }} + print_report: 'True' + window_size: '30' + fail_on_flaky_tests: 'False' + github_token: ${{ secrets.GITHUB_TOKEN }} + - name: Test summary uses: eProsima/eProsima-CI/windows/junit_summary@v0 if: ${{ !cancelled() && !contains(github.event.pull_request.labels.*.name, 'no-test') }}