From c4c05a6c570f9273ee1d12e89701ac625fb6bac1 Mon Sep 17 00:00:00 2001 From: Eleanor Boyd Date: Thu, 8 Feb 2024 11:49:44 -0800 Subject: [PATCH] update tree comparison for tests to be order independent for children (#22832) following the introduction of pytest 8, the order in which children were listed changed. Since the order is not important, this updates the tests to make the tests not consider order of children when comparing actual and expected outcomes of test runs. --- .../test_bottom_folder.py | 0 .../expected_discovery_test_output.py | 22 +++++++++---------- .../expected_execution_test_output.py | 14 ++++++------ .../tests/pytestadapter/test_discovery.py | 21 ++++++++++++------ .../tests/pytestadapter/test_execution.py | 4 ++-- .../helpers.py => tree_comparison_helper.py} | 9 +++----- .../expected_discovery_test_output.py | 4 +++- .../tests/unittestadapter/test_discovery.py | 9 +++++++- .../tests/unittestadapter/test_utils.py | 9 +++++++- 9 files changed, 56 insertions(+), 36 deletions(-) rename pythonFiles/tests/pytestadapter/.data/dual_level_nested_folder/{z_nested_folder_one => nested_folder_one}/test_bottom_folder.py (100%) rename pythonFiles/tests/{unittestadapter/helpers.py => tree_comparison_helper.py} (80%) diff --git a/pythonFiles/tests/pytestadapter/.data/dual_level_nested_folder/z_nested_folder_one/test_bottom_folder.py b/pythonFiles/tests/pytestadapter/.data/dual_level_nested_folder/nested_folder_one/test_bottom_folder.py similarity index 100% rename from pythonFiles/tests/pytestadapter/.data/dual_level_nested_folder/z_nested_folder_one/test_bottom_folder.py rename to pythonFiles/tests/pytestadapter/.data/dual_level_nested_folder/nested_folder_one/test_bottom_folder.py diff --git a/pythonFiles/tests/pytestadapter/expected_discovery_test_output.py b/pythonFiles/tests/pytestadapter/expected_discovery_test_output.py index d4e91f56b5fe..2d86710e776b 100644 --- a/pythonFiles/tests/pytestadapter/expected_discovery_test_output.py +++ b/pythonFiles/tests/pytestadapter/expected_discovery_test_output.py @@ -317,7 +317,7 @@ # └── test_top_folder.py # └── test_top_function_t # └── test_top_function_f -# └── z_nested_folder_one +# └── nested_folder_one # └── test_bottom_folder.py # └── test_bottom_function_t # └── test_bottom_function_f @@ -326,14 +326,14 @@ TEST_DATA_PATH / "dual_level_nested_folder" / "test_top_folder.py" ) -test_z_nested_folder_one_path = ( - TEST_DATA_PATH / "dual_level_nested_folder" / "z_nested_folder_one" +test_nested_folder_one_path = ( + TEST_DATA_PATH / "dual_level_nested_folder" / "nested_folder_one" ) test_bottom_folder_path = ( TEST_DATA_PATH / "dual_level_nested_folder" - / "z_nested_folder_one" + / "nested_folder_one" / "test_bottom_folder.py" ) @@ -392,10 +392,10 @@ ], }, { - "name": "z_nested_folder_one", - "path": os.fspath(test_z_nested_folder_one_path), + "name": "nested_folder_one", + "path": os.fspath(test_nested_folder_one_path), "type_": "folder", - "id_": os.fspath(test_z_nested_folder_one_path), + "id_": os.fspath(test_nested_folder_one_path), "children": [ { "name": "test_bottom_folder.py", @@ -412,11 +412,11 @@ ), "type_": "test", "id_": get_absolute_test_id( - "dual_level_nested_folder/z_nested_folder_one/test_bottom_folder.py::test_bottom_function_t", + "dual_level_nested_folder/nested_folder_one/test_bottom_folder.py::test_bottom_function_t", test_bottom_folder_path, ), "runID": get_absolute_test_id( - "dual_level_nested_folder/z_nested_folder_one/test_bottom_folder.py::test_bottom_function_t", + "dual_level_nested_folder/nested_folder_one/test_bottom_folder.py::test_bottom_function_t", test_bottom_folder_path, ), }, @@ -429,11 +429,11 @@ ), "type_": "test", "id_": get_absolute_test_id( - "dual_level_nested_folder/z_nested_folder_one/test_bottom_folder.py::test_bottom_function_f", + "dual_level_nested_folder/nested_folder_one/test_bottom_folder.py::test_bottom_function_f", test_bottom_folder_path, ), "runID": get_absolute_test_id( - "dual_level_nested_folder/z_nested_folder_one/test_bottom_folder.py::test_bottom_function_f", + "dual_level_nested_folder/nested_folder_one/test_bottom_folder.py::test_bottom_function_f", test_bottom_folder_path, ), }, diff --git a/pythonFiles/tests/pytestadapter/expected_execution_test_output.py b/pythonFiles/tests/pytestadapter/expected_execution_test_output.py index cf8997a252d3..44f3d3d0abce 100644 --- a/pythonFiles/tests/pytestadapter/expected_execution_test_output.py +++ b/pythonFiles/tests/pytestadapter/expected_execution_test_output.py @@ -308,7 +308,7 @@ # └── test_top_folder.py # └── test_top_function_t: success # └── test_top_function_f: failure -# └── z_nested_folder_one +# └── nested_folder_one # └── test_bottom_folder.py # └── test_bottom_function_t: success # └── test_bottom_function_f: failure @@ -318,7 +318,7 @@ dual_level_nested_folder_bottom_path = ( TEST_DATA_PATH / "dual_level_nested_folder" - / "z_nested_folder_one" + / "nested_folder_one" / "test_bottom_folder.py" ) dual_level_nested_folder_execution_expected_output = { @@ -345,11 +345,11 @@ "subtest": None, }, get_absolute_test_id( - "z_nested_folder_one/test_bottom_folder.py::test_bottom_function_t", + "nested_folder_one/test_bottom_folder.py::test_bottom_function_t", dual_level_nested_folder_bottom_path, ): { "test": get_absolute_test_id( - "z_nested_folder_one/test_bottom_folder.py::test_bottom_function_t", + "nested_folder_one/test_bottom_folder.py::test_bottom_function_t", dual_level_nested_folder_bottom_path, ), "outcome": "success", @@ -358,11 +358,11 @@ "subtest": None, }, get_absolute_test_id( - "z_nested_folder_one/test_bottom_folder.py::test_bottom_function_f", + "nested_folder_one/test_bottom_folder.py::test_bottom_function_f", dual_level_nested_folder_bottom_path, ): { "test": get_absolute_test_id( - "z_nested_folder_one/test_bottom_folder.py::test_bottom_function_f", + "nested_folder_one/test_bottom_folder.py::test_bottom_function_f", dual_level_nested_folder_bottom_path, ), "outcome": "failure", @@ -479,7 +479,7 @@ dual_level_nested_folder_bottom_path = ( TEST_DATA_PATH / "dual_level_nested_folder" - / "z_nested_folder_one" + / "nested_folder_one" / "test_bottom_folder.py" ) unittest_folder_add_path = TEST_DATA_PATH / "unittest_folder" / "test_add.py" diff --git a/pythonFiles/tests/pytestadapter/test_discovery.py b/pythonFiles/tests/pytestadapter/test_discovery.py index 2630ddef68b0..9956b82a6345 100644 --- a/pythonFiles/tests/pytestadapter/test_discovery.py +++ b/pythonFiles/tests/pytestadapter/test_discovery.py @@ -1,11 +1,18 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. import os +import pathlib import shutil +import sys from typing import Any, Dict, List, Optional import pytest +script_dir = pathlib.Path(__file__).parent.parent +sys.path.append(os.fspath(script_dir)) + +from tests.tree_comparison_helper import is_same_tree + from . import expected_discovery_test_output from .helpers import TEST_DATA_PATH, runner, runner_with_cwd @@ -195,7 +202,7 @@ def test_pytest_collect(file, expected_const): assert all(item in actual_item.keys() for item in ("status", "cwd", "error")) assert actual_item.get("status") == "success" assert actual_item.get("cwd") == os.fspath(TEST_DATA_PATH) - assert actual_item.get("tests") == expected_const + assert is_same_tree(actual_item.get("tests"), expected_const) def test_pytest_root_dir(): @@ -219,9 +226,9 @@ def test_pytest_root_dir(): assert all(item in actual_item.keys() for item in ("status", "cwd", "error")) assert actual_item.get("status") == "success" assert actual_item.get("cwd") == os.fspath(TEST_DATA_PATH / "root") - assert ( - actual_item.get("tests") - == expected_discovery_test_output.root_with_config_expected_output + assert is_same_tree( + actual_item.get("tests"), + expected_discovery_test_output.root_with_config_expected_output, ) @@ -245,7 +252,7 @@ def test_pytest_config_file(): assert all(item in actual_item.keys() for item in ("status", "cwd", "error")) assert actual_item.get("status") == "success" assert actual_item.get("cwd") == os.fspath(TEST_DATA_PATH / "root") - assert ( - actual_item.get("tests") - == expected_discovery_test_output.root_with_config_expected_output + assert is_same_tree( + actual_item.get("tests"), + expected_discovery_test_output.root_with_config_expected_output, ) diff --git a/pythonFiles/tests/pytestadapter/test_execution.py b/pythonFiles/tests/pytestadapter/test_execution.py index 767d54a6cabe..dd32b61fa262 100644 --- a/pythonFiles/tests/pytestadapter/test_execution.py +++ b/pythonFiles/tests/pytestadapter/test_execution.py @@ -193,8 +193,8 @@ def test_bad_id_error_execution(): [ "dual_level_nested_folder/test_top_folder.py::test_top_function_t", "dual_level_nested_folder/test_top_folder.py::test_top_function_f", - "dual_level_nested_folder/z_nested_folder_one/test_bottom_folder.py::test_bottom_function_t", - "dual_level_nested_folder/z_nested_folder_one/test_bottom_folder.py::test_bottom_function_f", + "dual_level_nested_folder/nested_folder_one/test_bottom_folder.py::test_bottom_function_t", + "dual_level_nested_folder/nested_folder_one/test_bottom_folder.py::test_bottom_function_f", ], expected_execution_test_output.dual_level_nested_folder_execution_expected_output, ), diff --git a/pythonFiles/tests/unittestadapter/helpers.py b/pythonFiles/tests/tree_comparison_helper.py similarity index 80% rename from pythonFiles/tests/unittestadapter/helpers.py rename to pythonFiles/tests/tree_comparison_helper.py index 303d021368f7..edf6aa8ff869 100644 --- a/pythonFiles/tests/unittestadapter/helpers.py +++ b/pythonFiles/tests/tree_comparison_helper.py @@ -1,10 +1,6 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -import pathlib - -TEST_DATA_PATH = pathlib.Path(__file__).parent / ".data" - def is_same_tree(tree1, tree2) -> bool: """Helper function to test if two test trees are the same. @@ -17,8 +13,9 @@ def is_same_tree(tree1, tree2) -> bool: # Compare child test nodes if they exist, otherwise compare test items. if "children" in tree1 and "children" in tree2: - children1 = tree1["children"] - children2 = tree2["children"] + # sort children by path before comparing since order doesn't matter of children + children1 = sorted(tree1["children"], key=lambda x: x["path"]) + children2 = sorted(tree2["children"], key=lambda x: x["path"]) # Compare test nodes. if len(children1) != len(children2): diff --git a/pythonFiles/tests/unittestadapter/expected_discovery_test_output.py b/pythonFiles/tests/unittestadapter/expected_discovery_test_output.py index db509ebeca3c..1007a8f42dfd 100644 --- a/pythonFiles/tests/unittestadapter/expected_discovery_test_output.py +++ b/pythonFiles/tests/unittestadapter/expected_discovery_test_output.py @@ -3,9 +3,11 @@ import os from unittestadapter.pvsc_utils import TestNodeTypeEnum -from .helpers import TEST_DATA_PATH import pathlib +TEST_DATA_PATH = pathlib.Path(__file__).parent / ".data" + + skip_unittest_folder_discovery_output = { "path": os.fspath(TEST_DATA_PATH / "unittest_skip"), "name": "unittest_skip", diff --git a/pythonFiles/tests/unittestadapter/test_discovery.py b/pythonFiles/tests/unittestadapter/test_discovery.py index 4249ca4faef2..a68774d3f2dc 100644 --- a/pythonFiles/tests/unittestadapter/test_discovery.py +++ b/pythonFiles/tests/unittestadapter/test_discovery.py @@ -3,14 +3,21 @@ import os import pathlib +import sys from typing import List import pytest from unittestadapter.discovery import discover_tests from unittestadapter.pvsc_utils import TestNodeTypeEnum, parse_unittest_args +script_dir = pathlib.Path(__file__).parent.parent +sys.path.append(os.fspath(script_dir)) + + from . import expected_discovery_test_output -from .helpers import TEST_DATA_PATH, is_same_tree +from tests.tree_comparison_helper import is_same_tree + +TEST_DATA_PATH = pathlib.Path(__file__).parent / ".data" @pytest.mark.parametrize( diff --git a/pythonFiles/tests/unittestadapter/test_utils.py b/pythonFiles/tests/unittestadapter/test_utils.py index 9fe2af8256cd..d5f6fbbe9f18 100644 --- a/pythonFiles/tests/unittestadapter/test_utils.py +++ b/pythonFiles/tests/unittestadapter/test_utils.py @@ -3,6 +3,7 @@ import os import pathlib +import sys import unittest import pytest @@ -15,7 +16,13 @@ get_test_case, ) -from .helpers import TEST_DATA_PATH, is_same_tree +script_dir = pathlib.Path(__file__).parent.parent +sys.path.append(os.fspath(script_dir)) + +from tests.tree_comparison_helper import is_same_tree + + +TEST_DATA_PATH = pathlib.Path(__file__).parent / ".data" @pytest.mark.parametrize(