Skip to content

Commit b68fa75

Browse files
eleanorjboydDonJayamanne
eleanorjboyd
authored and
DonJayamanne
committed
fix parameterized test duplicate function different classes (#23439)
fixes #23434 switched parameterized function IDs to now be `path/to/file::ClassIfExists::functionName` so it is an absolute ID and will not be confused across classes.
1 parent a8a0e59 commit b68fa75

File tree

7 files changed

+285
-31
lines changed

7 files changed

+285
-31
lines changed

‎python_files/tests/pytestadapter/.data/same_function_new_class_param.py

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
# Copyright (c) Microsoft Corporation. All rights reserved.
2+
# Licensed under the MIT License.
3+
import pytest
4+
5+
6+
class TestNotEmpty:
7+
@pytest.mark.parametrize("a, b", [(1, 1), (2, 2)]) # test_marker--TestNotEmpty::test_integer
8+
def test_integer(self, a, b):
9+
assert a == b
10+
11+
@pytest.mark.parametrize( # test_marker--TestNotEmpty::test_string
12+
"a, b", [("a", "a"), ("b", "b")]
13+
)
14+
def test_string(self, a, b):
15+
assert a == b
16+
17+
18+
class TestEmpty:
19+
@pytest.mark.parametrize("a, b", [(0, 0)]) # test_marker--TestEmpty::test_integer
20+
def test_integer(self, a, b):
21+
assert a == b
22+
23+
@pytest.mark.parametrize("a, b", [("", "")]) # test_marker--TestEmpty::test_string
24+
def test_string(self, a, b):
25+
assert a == b

‎python_files/tests/pytestadapter/expected_discovery_test_output.py

Lines changed: 201 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -541,7 +541,7 @@
541541
"name": "test_adding",
542542
"path": os.fspath(parameterize_tests_path),
543543
"type_": "function",
544-
"id_": "parametrize_tests.py::TestClass::test_adding",
544+
"id_": os.fspath(parameterize_tests_path) + "::TestClass::test_adding",
545545
"children": [
546546
{
547547
"name": "[3+5-8]",
@@ -638,7 +638,7 @@
638638
),
639639
},
640640
],
641-
"id_": "parametrize_tests.py::test_string",
641+
"id_": os.fspath(parameterize_tests_path) + "::test_string",
642642
},
643643
],
644644
},
@@ -760,7 +760,7 @@
760760
),
761761
},
762762
],
763-
"id_": "param_same_name/test_param1.py::test_odd_even",
763+
"id_": os.fspath(param1_path) + "::test_odd_even",
764764
}
765765
],
766766
},
@@ -818,7 +818,7 @@
818818
),
819819
},
820820
],
821-
"id_": "param_same_name/test_param2.py::test_odd_even",
821+
"id_": os.fspath(param2_path) + "::test_odd_even",
822822
}
823823
],
824824
},
@@ -1077,3 +1077,200 @@
10771077
],
10781078
"id_": str(SYMLINK_FOLDER_PATH),
10791079
}
1080+
1081+
same_function_new_class_param_expected_output = {
1082+
"name": ".data",
1083+
"path": TEST_DATA_PATH_STR,
1084+
"type_": "folder",
1085+
"children": [
1086+
{
1087+
"name": "same_function_new_class_param.py",
1088+
"path": os.fspath(TEST_DATA_PATH / "same_function_new_class_param.py"),
1089+
"type_": "file",
1090+
"id_": os.fspath(TEST_DATA_PATH / "same_function_new_class_param.py"),
1091+
"children": [
1092+
{
1093+
"name": "TestNotEmpty",
1094+
"path": os.fspath(TEST_DATA_PATH / "same_function_new_class_param.py"),
1095+
"type_": "class",
1096+
"children": [
1097+
{
1098+
"name": "test_integer",
1099+
"path": os.fspath(TEST_DATA_PATH / "same_function_new_class_param.py"),
1100+
"type_": "function",
1101+
"children": [
1102+
{
1103+
"name": "[1-1]",
1104+
"path": os.fspath(
1105+
TEST_DATA_PATH / "same_function_new_class_param.py"
1106+
),
1107+
"lineno": find_test_line_number(
1108+
"TestNotEmpty::test_integer",
1109+
os.fspath(
1110+
TEST_DATA_PATH / "same_function_new_class_param.py"
1111+
),
1112+
),
1113+
"type_": "test",
1114+
"id_": get_absolute_test_id(
1115+
"same_function_new_class_param.py::TestNotEmpty::test_integer[1-1]",
1116+
TEST_DATA_PATH / "same_function_new_class_param.py",
1117+
),
1118+
"runID": get_absolute_test_id(
1119+
"same_function_new_class_param.py::TestNotEmpty::test_integer[1-1]",
1120+
TEST_DATA_PATH / "same_function_new_class_param.py",
1121+
),
1122+
},
1123+
{
1124+
"name": "[2-2]",
1125+
"path": os.fspath(
1126+
TEST_DATA_PATH / "same_function_new_class_param.py"
1127+
),
1128+
"lineno": find_test_line_number(
1129+
"TestNotEmpty::test_integer",
1130+
os.fspath(
1131+
TEST_DATA_PATH / "same_function_new_class_param.py"
1132+
),
1133+
),
1134+
"type_": "test",
1135+
"id_": get_absolute_test_id(
1136+
"same_function_new_class_param.py::TestNotEmpty::test_integer[2-2]",
1137+
TEST_DATA_PATH / "same_function_new_class_param.py",
1138+
),
1139+
"runID": get_absolute_test_id(
1140+
"same_function_new_class_param.py::TestNotEmpty::test_integer[2-2]",
1141+
TEST_DATA_PATH / "same_function_new_class_param.py",
1142+
),
1143+
},
1144+
],
1145+
"id_": os.fspath(TEST_DATA_PATH / "same_function_new_class_param.py")
1146+
+ "::TestNotEmpty::test_integer",
1147+
},
1148+
{
1149+
"name": "test_string",
1150+
"path": os.fspath(TEST_DATA_PATH / "same_function_new_class_param.py"),
1151+
"type_": "function",
1152+
"children": [
1153+
{
1154+
"name": "[a-a]",
1155+
"path": os.fspath(
1156+
TEST_DATA_PATH / "same_function_new_class_param.py"
1157+
),
1158+
"lineno": find_test_line_number(
1159+
"TestNotEmpty::test_string",
1160+
os.fspath(
1161+
TEST_DATA_PATH / "same_function_new_class_param.py"
1162+
),
1163+
),
1164+
"type_": "test",
1165+
"id_": get_absolute_test_id(
1166+
"same_function_new_class_param.py::TestNotEmpty::test_string[a-a]",
1167+
TEST_DATA_PATH / "same_function_new_class_param.py",
1168+
),
1169+
"runID": get_absolute_test_id(
1170+
"same_function_new_class_param.py::TestNotEmpty::test_string[a-a]",
1171+
TEST_DATA_PATH / "same_function_new_class_param.py",
1172+
),
1173+
},
1174+
{
1175+
"name": "[b-b]",
1176+
"path": os.fspath(
1177+
TEST_DATA_PATH / "same_function_new_class_param.py"
1178+
),
1179+
"lineno": find_test_line_number(
1180+
"TestNotEmpty::test_string",
1181+
os.fspath(
1182+
TEST_DATA_PATH / "same_function_new_class_param.py"
1183+
),
1184+
),
1185+
"type_": "test",
1186+
"id_": get_absolute_test_id(
1187+
"same_function_new_class_param.py::TestNotEmpty::test_string[b-b]",
1188+
TEST_DATA_PATH / "same_function_new_class_param.py",
1189+
),
1190+
"runID": get_absolute_test_id(
1191+
"same_function_new_class_param.py::TestNotEmpty::test_string[b-b]",
1192+
TEST_DATA_PATH / "same_function_new_class_param.py",
1193+
),
1194+
},
1195+
],
1196+
"id_": os.fspath(TEST_DATA_PATH / "same_function_new_class_param.py")
1197+
+ "::TestNotEmpty::test_string",
1198+
},
1199+
],
1200+
"id_": "same_function_new_class_param.py::TestNotEmpty",
1201+
},
1202+
{
1203+
"name": "TestEmpty",
1204+
"path": os.fspath(TEST_DATA_PATH / "same_function_new_class_param.py"),
1205+
"type_": "class",
1206+
"children": [
1207+
{
1208+
"name": "test_integer",
1209+
"path": os.fspath(TEST_DATA_PATH / "same_function_new_class_param.py"),
1210+
"type_": "function",
1211+
"children": [
1212+
{
1213+
"name": "[0-0]",
1214+
"path": os.fspath(
1215+
TEST_DATA_PATH / "same_function_new_class_param.py"
1216+
),
1217+
"lineno": find_test_line_number(
1218+
"TestEmpty::test_integer",
1219+
os.fspath(
1220+
TEST_DATA_PATH / "same_function_new_class_param.py"
1221+
),
1222+
),
1223+
"type_": "test",
1224+
"id_": get_absolute_test_id(
1225+
"same_function_new_class_param.py::TestEmpty::test_integer[0-0]",
1226+
TEST_DATA_PATH / "same_function_new_class_param.py",
1227+
),
1228+
"runID": get_absolute_test_id(
1229+
"same_function_new_class_param.py::TestEmpty::test_integer[0-0]",
1230+
TEST_DATA_PATH / "same_function_new_class_param.py",
1231+
),
1232+
},
1233+
],
1234+
"id_": os.fspath(TEST_DATA_PATH / "same_function_new_class_param.py")
1235+
+ "::TestEmpty::test_integer",
1236+
},
1237+
{
1238+
"name": "test_string",
1239+
"path": os.fspath(TEST_DATA_PATH / "same_function_new_class_param.py"),
1240+
"type_": "function",
1241+
"children": [
1242+
{
1243+
"name": "[-]",
1244+
"path": os.fspath(
1245+
TEST_DATA_PATH / "same_function_new_class_param.py"
1246+
),
1247+
"lineno": find_test_line_number(
1248+
"TestEmpty::test_string",
1249+
os.fspath(
1250+
TEST_DATA_PATH / "same_function_new_class_param.py"
1251+
),
1252+
),
1253+
"type_": "test",
1254+
"id_": get_absolute_test_id(
1255+
"same_function_new_class_param.py::TestEmpty::test_string[-]",
1256+
TEST_DATA_PATH / "same_function_new_class_param.py",
1257+
),
1258+
"runID": get_absolute_test_id(
1259+
"same_function_new_class_param.py::TestEmpty::test_string[-]",
1260+
TEST_DATA_PATH / "same_function_new_class_param.py",
1261+
),
1262+
},
1263+
],
1264+
"id_": os.fspath(TEST_DATA_PATH / "same_function_new_class_param.py")
1265+
+ "::TestEmpty::test_string",
1266+
},
1267+
],
1268+
"id_": "same_function_new_class_param.py::TestEmpty",
1269+
},
1270+
],
1271+
}
1272+
],
1273+
"id_": TEST_DATA_PATH_STR,
1274+
}
1275+
1276+
print(param_same_name_expected_output)

‎python_files/tests/pytestadapter/test_discovery.py

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,10 @@ def test_parameterized_error_collect():
113113
"test_multi_class_nest.py",
114114
expected_discovery_test_output.nested_classes_expected_test_output,
115115
),
116+
(
117+
"same_function_new_class_param.py",
118+
expected_discovery_test_output.same_function_new_class_param_expected_output,
119+
),
116120
(
117121
"test_multi_class_nest.py",
118122
expected_discovery_test_output.nested_classes_expected_test_output,
@@ -187,7 +191,9 @@ def test_pytest_collect(file, expected_const):
187191
), f"Status is not 'success', error is: {actual_item.get('error')}"
188192
assert actual_item.get("cwd") == os.fspath(helpers.TEST_DATA_PATH)
189193
assert is_same_tree(
190-
actual_item.get("tests"), expected_const
194+
actual_item.get("tests"),
195+
expected_const,
196+
["id_", "lineno", "name", "runID"],
191197
), f"Tests tree does not match expected value. \n Expected: {json.dumps(expected_const, indent=4)}. \n Actual: {json.dumps(actual_item.get('tests'), indent=4)}"
192198

193199

@@ -255,6 +261,7 @@ def test_pytest_root_dir():
255261
assert is_same_tree(
256262
actual_item.get("tests"),
257263
expected_discovery_test_output.root_with_config_expected_output,
264+
["id_", "lineno", "name", "runID"],
258265
), f"Tests tree does not match expected value. \n Expected: {json.dumps(expected_discovery_test_output.root_with_config_expected_output, indent=4)}. \n Actual: {json.dumps(actual_item.get('tests'), indent=4)}"
259266

260267

@@ -281,6 +288,7 @@ def test_pytest_config_file():
281288
assert is_same_tree(
282289
actual_item.get("tests"),
283290
expected_discovery_test_output.root_with_config_expected_output,
291+
["id_", "lineno", "name", "runID"],
284292
), f"Tests tree does not match expected value. \n Expected: {json.dumps(expected_discovery_test_output.root_with_config_expected_output, indent=4)}. \n Actual: {json.dumps(actual_item.get('tests'), indent=4)}"
285293

286294

‎python_files/tests/tree_comparison_helper.py

Lines changed: 22 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,29 +1,39 @@
1-
# Copyright (c) Microsoft Corporation. All rights reserved.
2-
# Licensed under the MIT License.
3-
4-
5-
def is_same_tree(tree1, tree2) -> bool:
6-
"""Helper function to test if two test trees are the same.
1+
def is_same_tree(tree1, tree2, test_key_arr, path="root") -> bool:
2+
"""Helper function to test if two test trees are the same with detailed error logs.
73
84
`is_same_tree` starts by comparing the root attributes, and then checks if all children are the same.
95
"""
106
# Compare the root.
11-
if any(tree1[key] != tree2[key] for key in ["path", "name", "type_"]):
12-
return False
7+
for key in ["path", "name", "type_", "id_"]:
8+
if tree1.get(key) != tree2.get(key):
9+
print(
10+
f"Difference found at {path}: '{key}' is '{tree1.get(key)}' in tree1 and '{tree2.get(key)}' in tree2."
11+
)
12+
return False
1313

1414
# Compare child test nodes if they exist, otherwise compare test items.
1515
if "children" in tree1 and "children" in tree2:
16-
# sort children by path before comparing since order doesn't matter of children
16+
# Sort children by path before comparing since order doesn't matter of children
1717
children1 = sorted(tree1["children"], key=lambda x: x["path"])
1818
children2 = sorted(tree2["children"], key=lambda x: x["path"])
1919

2020
# Compare test nodes.
2121
if len(children1) != len(children2):
22+
print(
23+
f"Difference in number of children at {path}: {len(children1)} in tree1 and {len(children2)} in tree2."
24+
)
2225
return False
2326
else:
24-
return all(is_same_tree(*pair) for pair in zip(children1, children2))
27+
for i, (child1, child2) in enumerate(zip(children1, children2)):
28+
if not is_same_tree(child1, child2, test_key_arr, path=f"{path} -> child {i}"):
29+
return False
2530
elif "id_" in tree1 and "id_" in tree2:
2631
# Compare test items.
27-
return all(tree1[key] == tree2[key] for key in ["id_", "lineno"])
32+
for key in test_key_arr:
33+
if tree1.get(key) != tree2.get(key):
34+
print(
35+
f"Difference found at {path}: '{key}' is '{tree1.get(key)}' in tree1 and '{tree2.get(key)}' in tree2."
36+
)
37+
return False
2838

29-
return False
39+
return True

‎python_files/tests/unittestadapter/test_discovery.py

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -129,7 +129,7 @@ def test_simple_discovery() -> None:
129129
actual = discover_tests(start_dir, pattern, None)
130130

131131
assert actual["status"] == "success"
132-
assert is_same_tree(actual.get("tests"), expected)
132+
assert is_same_tree(actual.get("tests"), expected, ["id_", "lineno", "name"])
133133
assert "error" not in actual
134134

135135

@@ -185,7 +185,7 @@ def test_simple_discovery_with_top_dir_calculated() -> None:
185185
actual = discover_tests(start_dir, pattern, None)
186186

187187
assert actual["status"] == "success"
188-
assert is_same_tree(actual.get("tests"), expected)
188+
assert is_same_tree(actual.get("tests"), expected, ["id_", "lineno", "name"])
189189
assert "error" not in actual
190190

191191

@@ -256,7 +256,7 @@ def test_error_discovery() -> None:
256256
actual = discover_tests(start_dir, pattern, None)
257257

258258
assert actual["status"] == "error"
259-
assert is_same_tree(expected, actual.get("tests"))
259+
assert is_same_tree(expected, actual.get("tests"), ["id_", "lineno", "name"])
260260
assert len(actual.get("error", [])) == 1
261261

262262

@@ -274,6 +274,7 @@ def test_unit_skip() -> None:
274274
assert is_same_tree(
275275
actual.get("tests"),
276276
expected_discovery_test_output.skip_unittest_folder_discovery_output,
277+
["id_", "lineno", "name"],
277278
)
278279
assert "error" not in actual
279280

@@ -296,4 +297,5 @@ def test_complex_tree() -> None:
296297
assert is_same_tree(
297298
actual.get("tests"),
298299
expected_discovery_test_output.complex_tree_expected_output,
300+
["id_", "lineno", "name"],
299301
)

0 commit comments

Comments
 (0)