Skip to content

Commit 11f5c1c

Browse files
CadairCadair
authoredApr 22, 2022
Merge pull request #145 from ConorMacBride/test-shape-mismatch
Add test for image shape mismatch and fix bug
·
v0.17.0v0.15.1
2 parents 503e6f9 + 06475e8 commit 11f5c1c

24 files changed

+505
-1
lines changed
 

Diff for: ‎pytest_mpl/summary/html.py

+2
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,8 @@ def _rms_sort(self):
130130
if self.image_status == 'match':
131131
return "000000"
132132
elif self.image_status == 'diff':
133+
if self.rms is None: # Shape mismatch
134+
return "999999"
133135
# RMS will be in [0, 255]
134136
return f"{(self.rms + 2) * 1000:06.0f}"
135137
else: # Missing baseline image

Diff for: ‎pytest_mpl/summary/templates/result.html

+5-1
Original file line numberDiff line numberDiff line change
@@ -8,14 +8,18 @@
88
<div class="card">
99
<a class="btn" data-bs-toggle="offcanvas" href="#offcanvas{{ r.id }}" role="button"
1010
aria-controls="offcanvas{{ r.id }}">
11-
{% if r.diff_image -%}
11+
{% if r.image_status and r.image_status == "diff" -%}
1212
<div class="hover-image">
13+
{% if r.diff_image -%}
1314
<div class="diff-image">
1415
<img src="{{ r.diff_image }}" class="card-img-top" alt="diff image">
1516
</div>
1617
<div class="result-image">
1718
<img src="{{ r.result_image }}" class="card-img-top" alt="result image">
1819
</div>
20+
{%- else -%}
21+
<img src="{{ r.result_image }}" class="card-img-top" alt="result image">
22+
{%- endif %}
1923
</div>
2024
{%- elif r.result_image -%}
2125
<img src="{{ r.result_image }}" class="card-img-top" alt="result image">

Diff for: ‎tests/subtests/baseline/test_hdiff_idiffshape.png

16.1 KB
Loading

Diff for: ‎tests/subtests/baseline/test_hmatch_idiffshape.png

15.8 KB
Loading

Diff for: ‎tests/subtests/baseline/test_hmissing_idiffshape.png

16.9 KB
Loading

Diff for: ‎tests/subtests/hashes/mpl33_ft261.json

+2
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
{
22
"subtests.subtest.test_hmatch_imatch": "42c391b37022e2c4edb53f5fd988e94f421905b40cea1544e62ffb3c049292a8",
33
"subtests.subtest.test_hmatch_idiff": "c14ba098dbda0988e35be5724ffb15b8e666253a4b37dec6a21203607c17473d",
4+
"subtests.subtest.test_hmatch_idiffshape": "d23fa57068c6888307575623e5bdbe5e577d935910fee8d41deab426677acecb",
45
"subtests.subtest.test_hmatch_imissing": "6c07931bac1a926c88bea5d07c40c8c1ce30648712e3fc963028193863e3ae65",
56
"subtests.subtest.test_hdiff_imatch": "d1ff383721a0c395c856302be7de8a8138a2693651425dc181ede262860aef7b",
67
"subtests.subtest.test_hdiff_idiff": "d1fff55ace5ef7e45dcd9913b54e0d9970028cae59666e937ccb3586d0f76e9a",
8+
"subtests.subtest.test_hdiff_idiffshape": "d1ff76e20951e78fd3dedfff6a6f8f2eab4c569860d1a0da7867114cdcdf7c2c",
79
"subtests.subtest.test_hdiff_imissing": "d1ff35845c5887c034230e02aa4b60e053c779c693867e4803e1d72dde9240f7",
810
"subtests.subtest.test_hdiff_imatch_tolerance": "d1ff6912989a4b47ea910b04edfa58cf5d756d60825ea52ad59dcde8e03d4d8b",
911
"subtests.subtest.test_hdiff_idiff_tolerance": "d1ff6912989a4b47ea910b04edfa58cf5d756d60825ea52ad59dcde8e03d4d8b",

Diff for: ‎tests/subtests/hashes/mpl34_ft261.json

+2
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
{
22
"subtests.subtest.test_hmatch_imatch": "573f4c1482192b7b15bbe4f2bd370ae3b5ab40c9afa441543b87e15a71e9f672",
33
"subtests.subtest.test_hmatch_idiff": "8b5c00365e6f784d5c8947091f09a92fd7d222e790431f297b4848f173e64857",
4+
"subtests.subtest.test_hmatch_idiffshape": "2ee75301c4de2dcb9f839b278c6371be2e751de40b131213e375d4dcc5542382",
45
"subtests.subtest.test_hmatch_imissing": "fd069e642e3b154c24077a4996b545e1c4dbffdbed34ea5ad34c7b36873af68f",
56
"subtests.subtest.test_hdiff_imatch": "d1ffdde5a6682dc6abba7121f5df702c3664b1ce09593534fc0d7c3514eb07e1",
67
"subtests.subtest.test_hdiff_idiff": "d1ff61bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43",
8+
"subtests.subtest.test_hdiff_idiffshape": "d1ffae8ab2b65de3fa297be17ce973ff871e703c9550679e9566179dd785f6eb",
79
"subtests.subtest.test_hdiff_imissing": "d1ff63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e",
810
"subtests.subtest.test_hdiff_imatch_tolerance": "d1ffe85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96",
911
"subtests.subtest.test_hdiff_idiff_tolerance": "d1ffe85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96",

Diff for: ‎tests/subtests/hashes/mpl35_ft261.json

+2
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
{
22
"subtests.subtest.test_hmatch_imatch": "4a47c9b7920779cc83eabe2bbb64b9c40745d9d8abfa57857f93a5d8f12a5a03",
33
"subtests.subtest.test_hmatch_idiff": "2b48790b0a2cee4b41cdb9820336acaf229ba811ae21c6a92b4b92838843adfa",
4+
"subtests.subtest.test_hmatch_idiffshape": "e3fed4ad2d22aff2cd771c5503dcb30c6161b21d154430ededa5faa1ec54366e",
45
"subtests.subtest.test_hmatch_imissing": "e937fa1997d088c904ca35b1ab542e2285ea47b84df976490380f9c5f5b5f8ae",
56
"subtests.subtest.test_hdiff_imatch": "d1ff8f315d44b06de8f45d937af46a67bd1389edd6e4cde32f9feb4b7472284f",
67
"subtests.subtest.test_hdiff_idiff": "d1ff21206ef454a25417e3ba0bd3235c84518cb202c2d1fa7afcfdfcde5fdcde",
8+
"subtests.subtest.test_hdiff_idiffshape": "d1ff745bbdf2aac6743dbd830afb1877c2ac25a5f926d4f6483c1e24d19a0580",
79
"subtests.subtest.test_hdiff_imissing": "d1ff11cfa34db3a5819ac4127704e86acf27d24d1ea2410718853d3d7e1d6ae0",
810
"subtests.subtest.test_hdiff_imatch_tolerance": "d1ff3273d63a2a26a27e788ff0f090e86c9df7f9f191b7c566321c57de8266d6",
911
"subtests.subtest.test_hdiff_idiff_tolerance": "d1ff3273d63a2a26a27e788ff0f090e86c9df7f9f191b7c566321c57de8266d6",

Diff for: ‎tests/subtests/result_hashes/mpl33_ft261.json

+3
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,15 @@
11
{
22
"subtests.subtest.test_hmatch_imatch": "42c391b37022e2c4edb53f5fd988e94f421905b40cea1544e62ffb3c049292a8",
33
"subtests.subtest.test_hmatch_idiff": "c14ba098dbda0988e35be5724ffb15b8e666253a4b37dec6a21203607c17473d",
4+
"subtests.subtest.test_hmatch_idiffshape": "d23fa57068c6888307575623e5bdbe5e577d935910fee8d41deab426677acecb",
45
"subtests.subtest.test_hmatch_imissing": "6c07931bac1a926c88bea5d07c40c8c1ce30648712e3fc963028193863e3ae65",
56
"subtests.subtest.test_hdiff_imatch": "b48a383721a0c395c856302be7de8a8138a2693651425dc181ede262860aef7b",
67
"subtests.subtest.test_hdiff_idiff": "44edf55ace5ef7e45dcd9913b54e0d9970028cae59666e937ccb3586d0f76e9a",
8+
"subtests.subtest.test_hdiff_idiffshape": "1c6176e20951e78fd3dedfff6a6f8f2eab4c569860d1a0da7867114cdcdf7c2c",
79
"subtests.subtest.test_hdiff_imissing": "042235845c5887c034230e02aa4b60e053c779c693867e4803e1d72dde9240f7",
810
"subtests.subtest.test_hmissing_imatch": "d87e9aa72d2cd0ccd9959ec7af2dc00ab86e2fe8f06a6c54a017cde62f7e5c1f",
911
"subtests.subtest.test_hmissing_idiff": "00b074ef61b5061b8ec50dbe32b3214ffd21c9489ae212671558f338ad792e5f",
12+
"subtests.subtest.test_hmissing_idiffshape": "3abf54dbc88ce374b40d677597685fc83b4dc05c9d9a50fa1580f348d71fb99d",
1013
"subtests.subtest.test_hmissing_imissing": "3e5c4a4386e32133083c36ca0c95f5e38be904ed238e49ab1e06edd35603abfc",
1114
"subtests.subtest.test_hdiff_imatch_tolerance": "04eb6912989a4b47ea910b04edfa58cf5d756d60825ea52ad59dcde8e03d4d8b",
1215
"subtests.subtest.test_hdiff_idiff_tolerance": "04eb6912989a4b47ea910b04edfa58cf5d756d60825ea52ad59dcde8e03d4d8b",

Diff for: ‎tests/subtests/result_hashes/mpl34_ft261.json

+3
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,15 @@
11
{
22
"subtests.subtest.test_hmatch_imatch": "573f4c1482192b7b15bbe4f2bd370ae3b5ab40c9afa441543b87e15a71e9f672",
33
"subtests.subtest.test_hmatch_idiff": "8b5c00365e6f784d5c8947091f09a92fd7d222e790431f297b4848f173e64857",
4+
"subtests.subtest.test_hmatch_idiffshape": "2ee75301c4de2dcb9f839b278c6371be2e751de40b131213e375d4dcc5542382",
45
"subtests.subtest.test_hmatch_imissing": "fd069e642e3b154c24077a4996b545e1c4dbffdbed34ea5ad34c7b36873af68f",
56
"subtests.subtest.test_hdiff_imatch": "6e2fdde5a6682dc6abba7121f5df702c3664b1ce09593534fc0d7c3514eb07e1",
67
"subtests.subtest.test_hdiff_idiff": "443361bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43",
8+
"subtests.subtest.test_hdiff_idiffshape": "3379ae8ab2b65de3fa297be17ce973ff871e703c9550679e9566179dd785f6eb",
79
"subtests.subtest.test_hdiff_imissing": "301e63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e",
810
"subtests.subtest.test_hmissing_imatch": "eabd8a2e22afd88682990bfb8e4a0700a942fe68e5114e8da4ab6bd93c47b824",
911
"subtests.subtest.test_hmissing_idiff": "e69570c4a70b2cc88ddee0f0a82312cae4f394b7f62e5760245feda1364c03ab",
12+
"subtests.subtest.test_hmissing_idiffshape": "fa566a1620537f5c10aa8bcbebed55065f13fa9f8e8e82d7f7c67f7b39edd552",
1013
"subtests.subtest.test_hmissing_imissing": "5c8a9c7412e4e098f6f2683ee247c08bd50805a93df4d4b6d8fccf3579b4c56b",
1114
"subtests.subtest.test_hdiff_imatch_tolerance": "aaf4e85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96",
1215
"subtests.subtest.test_hdiff_idiff_tolerance": "aaf4e85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96",

Diff for: ‎tests/subtests/result_hashes/mpl35_ft261.json

+3
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,15 @@
11
{
22
"subtests.subtest.test_hmatch_imatch": "4a47c9b7920779cc83eabe2bbb64b9c40745d9d8abfa57857f93a5d8f12a5a03",
33
"subtests.subtest.test_hmatch_idiff": "2b48790b0a2cee4b41cdb9820336acaf229ba811ae21c6a92b4b92838843adfa",
4+
"subtests.subtest.test_hmatch_idiffshape": "e3fed4ad2d22aff2cd771c5503dcb30c6161b21d154430ededa5faa1ec54366e",
45
"subtests.subtest.test_hmatch_imissing": "e937fa1997d088c904ca35b1ab542e2285ea47b84df976490380f9c5f5b5f8ae",
56
"subtests.subtest.test_hdiff_imatch": "2cae8f315d44b06de8f45d937af46a67bd1389edd6e4cde32f9feb4b7472284f",
67
"subtests.subtest.test_hdiff_idiff": "927521206ef454a25417e3ba0bd3235c84518cb202c2d1fa7afcfdfcde5fdcde",
8+
"subtests.subtest.test_hdiff_idiffshape": "5d37745bbdf2aac6743dbd830afb1877c2ac25a5f926d4f6483c1e24d19a0580",
79
"subtests.subtest.test_hdiff_imissing": "afc411cfa34db3a5819ac4127704e86acf27d24d1ea2410718853d3d7e1d6ae0",
810
"subtests.subtest.test_hmissing_imatch": "7ee8370efdc4b767634d12355657ca4f2460176670c07b31f3fb72cea0e79856",
911
"subtests.subtest.test_hmissing_idiff": "4eeda1d349f4b0f26df97df41ba5410dce2b1c7ed520062d58f3c5f0e3790ebd",
12+
"subtests.subtest.test_hmissing_idiffshape": "204d43b643538cafafa2d0fb7dafac9cf7009dbb421d4b32c71da2e400ceb59d",
1013
"subtests.subtest.test_hmissing_imissing": "5101e60ac100cf2c2f418a0a6a382aae0060339e76718730344f539b61f7dc7e",
1114
"subtests.subtest.test_hdiff_imatch_tolerance": "510b3273d63a2a26a27e788ff0f090e86c9df7f9f191b7c566321c57de8266d6",
1215
"subtests.subtest.test_hdiff_idiff_tolerance": "510b3273d63a2a26a27e788ff0f090e86c9df7f9f191b7c566321c57de8266d6",

Diff for: ‎tests/subtests/subtest.py

+15
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,11 @@ def test_hmatch_idiff():
2525
return plot([1, 3, 2, 4])
2626

2727

28+
@pytest.mark.mpl_image_compare()
29+
def test_hmatch_idiffshape():
30+
return plot([4, 2, 3, 1, 2])
31+
32+
2833
@pytest.mark.mpl_image_compare()
2934
def test_hmatch_imissing():
3035
return plot([4, 3, 2, 1])
@@ -42,6 +47,11 @@ def test_hdiff_idiff():
4247
return plot([1, 2, 4, 3])
4348

4449

50+
@pytest.mark.mpl_image_compare()
51+
def test_hdiff_idiffshape():
52+
return plot([4, 2, 3, 1, 3])
53+
54+
4555
@pytest.mark.mpl_image_compare()
4656
def test_hdiff_imissing():
4757
return plot([3, 2, 4, 1])
@@ -59,6 +69,11 @@ def test_hmissing_idiff():
5969
return plot([1, 4, 3, 2])
6070

6171

72+
@pytest.mark.mpl_image_compare()
73+
def test_hmissing_idiffshape():
74+
return plot([4, 2, 3, 1, 4])
75+
76+
6277
@pytest.mark.mpl_image_compare()
6378
def test_hmissing_imissing():
6479
return plot([2, 4, 3, 1])

Diff for: ‎tests/subtests/summaries/test_default.json

+39
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,19 @@
2525
"baseline_hash": null,
2626
"result_hash": null
2727
},
28+
"subtests.subtest.test_hmatch_idiffshape": {
29+
"status": "failed",
30+
"image_status": "diff",
31+
"hash_status": null,
32+
"status_msg": "REGEX:Error: Image dimensions did not match\\.\n Expected shape: \\(400, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png",
33+
"baseline_image": "subtests.subtest.test_hmatch_idiffshape/baseline.png",
34+
"diff_image": null,
35+
"rms": null,
36+
"tolerance": null,
37+
"result_image": "subtests.subtest.test_hmatch_idiffshape/result.png",
38+
"baseline_hash": null,
39+
"result_hash": null
40+
},
2841
"subtests.subtest.test_hmatch_imissing": {
2942
"status": "failed",
3043
"image_status": "missing",
@@ -64,6 +77,19 @@
6477
"baseline_hash": null,
6578
"result_hash": null
6679
},
80+
"subtests.subtest.test_hdiff_idiffshape": {
81+
"status": "failed",
82+
"image_status": "diff",
83+
"hash_status": null,
84+
"status_msg": "REGEX:Error: Image dimensions did not match\\.\n Expected shape: \\(400, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png",
85+
"baseline_image": "subtests.subtest.test_hdiff_idiffshape/baseline.png",
86+
"diff_image": null,
87+
"rms": null,
88+
"tolerance": null,
89+
"result_image": "subtests.subtest.test_hdiff_idiffshape/result.png",
90+
"baseline_hash": null,
91+
"result_hash": null
92+
},
6793
"subtests.subtest.test_hdiff_imissing": {
6894
"status": "failed",
6995
"image_status": "missing",
@@ -103,6 +129,19 @@
103129
"baseline_hash": null,
104130
"result_hash": null
105131
},
132+
"subtests.subtest.test_hmissing_idiffshape": {
133+
"status": "failed",
134+
"image_status": "diff",
135+
"hash_status": null,
136+
"status_msg": "REGEX:Error: Image dimensions did not match\\.\n Expected shape: \\(400, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png",
137+
"baseline_image": "subtests.subtest.test_hmissing_idiffshape/baseline.png",
138+
"diff_image": null,
139+
"rms": null,
140+
"tolerance": null,
141+
"result_image": "subtests.subtest.test_hmissing_idiffshape/result.png",
142+
"baseline_hash": null,
143+
"result_hash": null
144+
},
106145
"subtests.subtest.test_hmissing_imissing": {
107146
"status": "failed",
108147
"image_status": "missing",

Diff for: ‎tests/subtests/summaries/test_generate.json

+39
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,19 @@
2525
"baseline_hash": "2b48790b0a2cee4b41cdb9820336acaf229ba811ae21c6a92b4b92838843adfa",
2626
"result_hash": null
2727
},
28+
"subtests.subtest.test_hmatch_idiffshape": {
29+
"status": "skipped",
30+
"image_status": "generated",
31+
"hash_status": "generated",
32+
"status_msg": "Skipped test, since generating image.",
33+
"baseline_image": null,
34+
"diff_image": null,
35+
"rms": null,
36+
"tolerance": null,
37+
"result_image": null,
38+
"baseline_hash": "e3fed4ad2d22aff2cd771c5503dcb30c6161b21d154430ededa5faa1ec54366e",
39+
"result_hash": null
40+
},
2841
"subtests.subtest.test_hmatch_imissing": {
2942
"status": "skipped",
3043
"image_status": "generated",
@@ -64,6 +77,19 @@
6477
"baseline_hash": "927521206ef454a25417e3ba0bd3235c84518cb202c2d1fa7afcfdfcde5fdcde",
6578
"result_hash": null
6679
},
80+
"subtests.subtest.test_hdiff_idiffshape": {
81+
"status": "skipped",
82+
"image_status": "generated",
83+
"hash_status": "generated",
84+
"status_msg": "Skipped test, since generating image.",
85+
"baseline_image": null,
86+
"diff_image": null,
87+
"rms": null,
88+
"tolerance": null,
89+
"result_image": null,
90+
"baseline_hash": "5d37745bbdf2aac6743dbd830afb1877c2ac25a5f926d4f6483c1e24d19a0580",
91+
"result_hash": null
92+
},
6793
"subtests.subtest.test_hdiff_imissing": {
6894
"status": "skipped",
6995
"image_status": "generated",
@@ -103,6 +129,19 @@
103129
"baseline_hash": "4eeda1d349f4b0f26df97df41ba5410dce2b1c7ed520062d58f3c5f0e3790ebd",
104130
"result_hash": null
105131
},
132+
"subtests.subtest.test_hmissing_idiffshape": {
133+
"status": "skipped",
134+
"image_status": "generated",
135+
"hash_status": "generated",
136+
"status_msg": "Skipped test, since generating image.",
137+
"baseline_image": null,
138+
"diff_image": null,
139+
"rms": null,
140+
"tolerance": null,
141+
"result_image": null,
142+
"baseline_hash": "204d43b643538cafafa2d0fb7dafac9cf7009dbb421d4b32c71da2e400ceb59d",
143+
"result_hash": null
144+
},
106145
"subtests.subtest.test_hmissing_imissing": {
107146
"status": "skipped",
108147
"image_status": "generated",

Diff for: ‎tests/subtests/summaries/test_generate_hashes_only.json

+39
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,19 @@
2525
"baseline_hash": "2b48790b0a2cee4b41cdb9820336acaf229ba811ae21c6a92b4b92838843adfa",
2626
"result_hash": null
2727
},
28+
"subtests.subtest.test_hmatch_idiffshape": {
29+
"status": "failed",
30+
"image_status": "diff",
31+
"hash_status": "generated",
32+
"status_msg": "REGEX:Error: Image dimensions did not match\\.\n Expected shape: \\(400, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png",
33+
"baseline_image": "subtests.subtest.test_hmatch_idiffshape/baseline.png",
34+
"diff_image": null,
35+
"rms": null,
36+
"tolerance": null,
37+
"result_image": "subtests.subtest.test_hmatch_idiffshape/result.png",
38+
"baseline_hash": "e3fed4ad2d22aff2cd771c5503dcb30c6161b21d154430ededa5faa1ec54366e",
39+
"result_hash": null
40+
},
2841
"subtests.subtest.test_hmatch_imissing": {
2942
"status": "failed",
3043
"image_status": "missing",
@@ -64,6 +77,19 @@
6477
"baseline_hash": "927521206ef454a25417e3ba0bd3235c84518cb202c2d1fa7afcfdfcde5fdcde",
6578
"result_hash": null
6679
},
80+
"subtests.subtest.test_hdiff_idiffshape": {
81+
"status": "failed",
82+
"image_status": "diff",
83+
"hash_status": "generated",
84+
"status_msg": "REGEX:Error: Image dimensions did not match\\.\n Expected shape: \\(400, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png",
85+
"baseline_image": "subtests.subtest.test_hdiff_idiffshape/baseline.png",
86+
"diff_image": null,
87+
"rms": null,
88+
"tolerance": null,
89+
"result_image": "subtests.subtest.test_hdiff_idiffshape/result.png",
90+
"baseline_hash": "5d37745bbdf2aac6743dbd830afb1877c2ac25a5f926d4f6483c1e24d19a0580",
91+
"result_hash": null
92+
},
6793
"subtests.subtest.test_hdiff_imissing": {
6894
"status": "failed",
6995
"image_status": "missing",
@@ -103,6 +129,19 @@
103129
"baseline_hash": "4eeda1d349f4b0f26df97df41ba5410dce2b1c7ed520062d58f3c5f0e3790ebd",
104130
"result_hash": null
105131
},
132+
"subtests.subtest.test_hmissing_idiffshape": {
133+
"status": "failed",
134+
"image_status": "diff",
135+
"hash_status": "generated",
136+
"status_msg": "REGEX:Error: Image dimensions did not match\\.\n Expected shape: \\(400, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png",
137+
"baseline_image": "subtests.subtest.test_hmissing_idiffshape/baseline.png",
138+
"diff_image": null,
139+
"rms": null,
140+
"tolerance": null,
141+
"result_image": "subtests.subtest.test_hmissing_idiffshape/result.png",
142+
"baseline_hash": "204d43b643538cafafa2d0fb7dafac9cf7009dbb421d4b32c71da2e400ceb59d",
143+
"result_hash": null
144+
},
106145
"subtests.subtest.test_hmissing_imissing": {
107146
"status": "failed",
108147
"image_status": "missing",

Diff for: ‎tests/subtests/summaries/test_generate_images_only.json

+39
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,19 @@
2525
"baseline_hash": null,
2626
"result_hash": null
2727
},
28+
"subtests.subtest.test_hmatch_idiffshape": {
29+
"status": "skipped",
30+
"image_status": "generated",
31+
"hash_status": null,
32+
"status_msg": "Skipped test, since generating image.",
33+
"baseline_image": null,
34+
"diff_image": null,
35+
"rms": null,
36+
"tolerance": null,
37+
"result_image": null,
38+
"baseline_hash": null,
39+
"result_hash": null
40+
},
2841
"subtests.subtest.test_hmatch_imissing": {
2942
"status": "skipped",
3043
"image_status": "generated",
@@ -64,6 +77,19 @@
6477
"baseline_hash": null,
6578
"result_hash": null
6679
},
80+
"subtests.subtest.test_hdiff_idiffshape": {
81+
"status": "skipped",
82+
"image_status": "generated",
83+
"hash_status": null,
84+
"status_msg": "Skipped test, since generating image.",
85+
"baseline_image": null,
86+
"diff_image": null,
87+
"rms": null,
88+
"tolerance": null,
89+
"result_image": null,
90+
"baseline_hash": null,
91+
"result_hash": null
92+
},
6793
"subtests.subtest.test_hdiff_imissing": {
6894
"status": "skipped",
6995
"image_status": "generated",
@@ -103,6 +129,19 @@
103129
"baseline_hash": null,
104130
"result_hash": null
105131
},
132+
"subtests.subtest.test_hmissing_idiffshape": {
133+
"status": "skipped",
134+
"image_status": "generated",
135+
"hash_status": null,
136+
"status_msg": "Skipped test, since generating image.",
137+
"baseline_image": null,
138+
"diff_image": null,
139+
"rms": null,
140+
"tolerance": null,
141+
"result_image": null,
142+
"baseline_hash": null,
143+
"result_hash": null
144+
},
106145
"subtests.subtest.test_hmissing_imissing": {
107146
"status": "skipped",
108147
"image_status": "generated",

Diff for: ‎tests/subtests/summaries/test_hash.json

+39
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,19 @@
2525
"baseline_hash": "8b5c00365e6f784d5c8947091f09a92fd7d222e790431f297b4848f173e64857",
2626
"result_hash": "8b5c00365e6f784d5c8947091f09a92fd7d222e790431f297b4848f173e64857"
2727
},
28+
"subtests.subtest.test_hmatch_idiffshape": {
29+
"status": "passed",
30+
"image_status": null,
31+
"hash_status": "match",
32+
"status_msg": "Test hash matches baseline hash.",
33+
"baseline_image": null,
34+
"diff_image": null,
35+
"rms": null,
36+
"tolerance": null,
37+
"result_image": null,
38+
"baseline_hash": "e3fed4ad2d22aff2cd771c5503dcb30c6161b21d154430ededa5faa1ec54366e",
39+
"result_hash": "e3fed4ad2d22aff2cd771c5503dcb30c6161b21d154430ededa5faa1ec54366e"
40+
},
2841
"subtests.subtest.test_hmatch_imissing": {
2942
"status": "passed",
3043
"image_status": null,
@@ -64,6 +77,19 @@
6477
"baseline_hash": "d1ff61bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43",
6578
"result_hash": "443361bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43"
6679
},
80+
"subtests.subtest.test_hdiff_idiffshape": {
81+
"status": "failed",
82+
"image_status": null,
83+
"hash_status": "diff",
84+
"status_msg": "REGEX:Hash 5d37745bbdf2aac6743dbd830afb1877c2ac25a5f926d4f6483c1e24d19a0580 doesn't match hash d1ff745bbdf2aac6743dbd830afb1877c2ac25a5f926d4f6483c1e24d19a0580 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiffshape\\.",
85+
"baseline_image": null,
86+
"diff_image": null,
87+
"rms": null,
88+
"tolerance": null,
89+
"result_image": "subtests.subtest.test_hdiff_idiffshape/result.png",
90+
"baseline_hash": "d1ff745bbdf2aac6743dbd830afb1877c2ac25a5f926d4f6483c1e24d19a0580",
91+
"result_hash": "5d37745bbdf2aac6743dbd830afb1877c2ac25a5f926d4f6483c1e24d19a0580"
92+
},
6793
"subtests.subtest.test_hdiff_imissing": {
6894
"status": "failed",
6995
"status_msg": "REGEX:Hash 301e63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e doesn't match hash d1ff63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imissing\\.",
@@ -103,6 +129,19 @@
103129
"baseline_hash": null,
104130
"result_hash": "e69570c4a70b2cc88ddee0f0a82312cae4f394b7f62e5760245feda1364c03ab"
105131
},
132+
"subtests.subtest.test_hmissing_idiffshape": {
133+
"status": "failed",
134+
"image_status": null,
135+
"hash_status": "missing",
136+
"status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_idiffshape' not found in .*\\.json\\. Generated hash is 204d43b643538cafafa2d0fb7dafac9cf7009dbb421d4b32c71da2e400ceb59d\\.",
137+
"baseline_image": null,
138+
"diff_image": null,
139+
"rms": null,
140+
"tolerance": null,
141+
"result_image": "subtests.subtest.test_hmissing_idiffshape/result.png",
142+
"baseline_hash": null,
143+
"result_hash": "204d43b643538cafafa2d0fb7dafac9cf7009dbb421d4b32c71da2e400ceb59d"
144+
},
106145
"subtests.subtest.test_hmissing_imissing": {
107146
"status": "failed",
108147
"status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_imissing' not found in .*\\.json\\. Generated hash is 5c8a9c7412e4e098f6f2683ee247c08bd50805a93df4d4b6d8fccf3579b4c56b\\.",

Diff for: ‎tests/subtests/summaries/test_html_generate.json

+39
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,19 @@
2525
"baseline_hash": "2b48790b0a2cee4b41cdb9820336acaf229ba811ae21c6a92b4b92838843adfa",
2626
"result_hash": null
2727
},
28+
"subtests.subtest.test_hmatch_idiffshape": {
29+
"status": "skipped",
30+
"image_status": "generated",
31+
"hash_status": "generated",
32+
"status_msg": "Skipped test, since generating image.",
33+
"baseline_image": "subtests.subtest.test_hmatch_idiffshape/baseline.png",
34+
"diff_image": null,
35+
"rms": null,
36+
"tolerance": null,
37+
"result_image": null,
38+
"baseline_hash": "e3fed4ad2d22aff2cd771c5503dcb30c6161b21d154430ededa5faa1ec54366e",
39+
"result_hash": null
40+
},
2841
"subtests.subtest.test_hmatch_imissing": {
2942
"status": "skipped",
3043
"image_status": "generated",
@@ -64,6 +77,19 @@
6477
"baseline_hash": "927521206ef454a25417e3ba0bd3235c84518cb202c2d1fa7afcfdfcde5fdcde",
6578
"result_hash": null
6679
},
80+
"subtests.subtest.test_hdiff_idiffshape": {
81+
"status": "skipped",
82+
"image_status": "generated",
83+
"hash_status": "generated",
84+
"status_msg": "Skipped test, since generating image.",
85+
"baseline_image": "subtests.subtest.test_hdiff_idiffshape/baseline.png",
86+
"diff_image": null,
87+
"rms": null,
88+
"tolerance": null,
89+
"result_image": null,
90+
"baseline_hash": "5d37745bbdf2aac6743dbd830afb1877c2ac25a5f926d4f6483c1e24d19a0580",
91+
"result_hash": null
92+
},
6793
"subtests.subtest.test_hdiff_imissing": {
6894
"status": "skipped",
6995
"image_status": "generated",
@@ -103,6 +129,19 @@
103129
"baseline_hash": "4eeda1d349f4b0f26df97df41ba5410dce2b1c7ed520062d58f3c5f0e3790ebd",
104130
"result_hash": null
105131
},
132+
"subtests.subtest.test_hmissing_idiffshape": {
133+
"status": "skipped",
134+
"image_status": "generated",
135+
"hash_status": "generated",
136+
"status_msg": "Skipped test, since generating image.",
137+
"baseline_image": "subtests.subtest.test_hmissing_idiffshape/baseline.png",
138+
"diff_image": null,
139+
"rms": null,
140+
"tolerance": null,
141+
"result_image": null,
142+
"baseline_hash": "204d43b643538cafafa2d0fb7dafac9cf7009dbb421d4b32c71da2e400ceb59d",
143+
"result_hash": null
144+
},
106145
"subtests.subtest.test_hmissing_imissing": {
107146
"status": "skipped",
108147
"image_status": "generated",

Diff for: ‎tests/subtests/summaries/test_html_generate_hashes_only.json

+39
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,19 @@
2525
"baseline_hash": "2b48790b0a2cee4b41cdb9820336acaf229ba811ae21c6a92b4b92838843adfa",
2626
"result_hash": null
2727
},
28+
"subtests.subtest.test_hmatch_idiffshape": {
29+
"status": "failed",
30+
"image_status": "diff",
31+
"hash_status": "generated",
32+
"status_msg": "REGEX:Error: Image dimensions did not match\\.\n Expected shape: \\(400, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png",
33+
"baseline_image": "subtests.subtest.test_hmatch_idiffshape/baseline.png",
34+
"diff_image": null,
35+
"rms": null,
36+
"tolerance": null,
37+
"result_image": "subtests.subtest.test_hmatch_idiffshape/result.png",
38+
"baseline_hash": "e3fed4ad2d22aff2cd771c5503dcb30c6161b21d154430ededa5faa1ec54366e",
39+
"result_hash": null
40+
},
2841
"subtests.subtest.test_hmatch_imissing": {
2942
"status": "failed",
3043
"image_status": "missing",
@@ -64,6 +77,19 @@
6477
"baseline_hash": "927521206ef454a25417e3ba0bd3235c84518cb202c2d1fa7afcfdfcde5fdcde",
6578
"result_hash": null
6679
},
80+
"subtests.subtest.test_hdiff_idiffshape": {
81+
"status": "failed",
82+
"image_status": "diff",
83+
"hash_status": "generated",
84+
"status_msg": "REGEX:Error: Image dimensions did not match\\.\n Expected shape: \\(400, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png",
85+
"baseline_image": "subtests.subtest.test_hdiff_idiffshape/baseline.png",
86+
"diff_image": null,
87+
"rms": null,
88+
"tolerance": null,
89+
"result_image": "subtests.subtest.test_hdiff_idiffshape/result.png",
90+
"baseline_hash": "5d37745bbdf2aac6743dbd830afb1877c2ac25a5f926d4f6483c1e24d19a0580",
91+
"result_hash": null
92+
},
6793
"subtests.subtest.test_hdiff_imissing": {
6894
"status": "failed",
6995
"image_status": "missing",
@@ -103,6 +129,19 @@
103129
"baseline_hash": "4eeda1d349f4b0f26df97df41ba5410dce2b1c7ed520062d58f3c5f0e3790ebd",
104130
"result_hash": null
105131
},
132+
"subtests.subtest.test_hmissing_idiffshape": {
133+
"status": "failed",
134+
"image_status": "diff",
135+
"hash_status": "generated",
136+
"status_msg": "REGEX:Error: Image dimensions did not match\\.\n Expected shape: \\(400, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png",
137+
"baseline_image": "subtests.subtest.test_hmissing_idiffshape/baseline.png",
138+
"diff_image": null,
139+
"rms": null,
140+
"tolerance": null,
141+
"result_image": "subtests.subtest.test_hmissing_idiffshape/result.png",
142+
"baseline_hash": "204d43b643538cafafa2d0fb7dafac9cf7009dbb421d4b32c71da2e400ceb59d",
143+
"result_hash": null
144+
},
106145
"subtests.subtest.test_hmissing_imissing": {
107146
"status": "failed",
108147
"image_status": "missing",

Diff for: ‎tests/subtests/summaries/test_html_generate_images_only.json

+39
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,19 @@
2525
"baseline_hash": null,
2626
"result_hash": null
2727
},
28+
"subtests.subtest.test_hmatch_idiffshape": {
29+
"status": "skipped",
30+
"image_status": "generated",
31+
"hash_status": null,
32+
"status_msg": "Skipped test, since generating image.",
33+
"baseline_image": "subtests.subtest.test_hmatch_idiffshape/baseline.png",
34+
"diff_image": null,
35+
"rms": null,
36+
"tolerance": null,
37+
"result_image": null,
38+
"baseline_hash": null,
39+
"result_hash": null
40+
},
2841
"subtests.subtest.test_hmatch_imissing": {
2942
"status": "skipped",
3043
"image_status": "generated",
@@ -64,6 +77,19 @@
6477
"baseline_hash": null,
6578
"result_hash": null
6679
},
80+
"subtests.subtest.test_hdiff_idiffshape": {
81+
"status": "skipped",
82+
"image_status": "generated",
83+
"hash_status": null,
84+
"status_msg": "Skipped test, since generating image.",
85+
"baseline_image": "subtests.subtest.test_hdiff_idiffshape/baseline.png",
86+
"diff_image": null,
87+
"rms": null,
88+
"tolerance": null,
89+
"result_image": null,
90+
"baseline_hash": null,
91+
"result_hash": null
92+
},
6793
"subtests.subtest.test_hdiff_imissing": {
6894
"status": "skipped",
6995
"image_status": "generated",
@@ -103,6 +129,19 @@
103129
"baseline_hash": null,
104130
"result_hash": null
105131
},
132+
"subtests.subtest.test_hmissing_idiffshape": {
133+
"status": "skipped",
134+
"image_status": "generated",
135+
"hash_status": null,
136+
"status_msg": "Skipped test, since generating image.",
137+
"baseline_image": "subtests.subtest.test_hmissing_idiffshape/baseline.png",
138+
"diff_image": null,
139+
"rms": null,
140+
"tolerance": null,
141+
"result_image": null,
142+
"baseline_hash": null,
143+
"result_hash": null
144+
},
106145
"subtests.subtest.test_hmissing_imissing": {
107146
"status": "skipped",
108147
"image_status": "generated",

Diff for: ‎tests/subtests/summaries/test_html_hashes_only.json

+39
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,19 @@
2525
"baseline_hash": "8b5c00365e6f784d5c8947091f09a92fd7d222e790431f297b4848f173e64857",
2626
"result_hash": "8b5c00365e6f784d5c8947091f09a92fd7d222e790431f297b4848f173e64857"
2727
},
28+
"subtests.subtest.test_hmatch_idiffshape": {
29+
"status": "passed",
30+
"image_status": null,
31+
"hash_status": "match",
32+
"status_msg": "Test hash matches baseline hash.",
33+
"baseline_image": null,
34+
"diff_image": null,
35+
"rms": null,
36+
"tolerance": null,
37+
"result_image": "subtests.subtest.test_hmatch_idiffshape/result.png",
38+
"baseline_hash": "e3fed4ad2d22aff2cd771c5503dcb30c6161b21d154430ededa5faa1ec54366e",
39+
"result_hash": "e3fed4ad2d22aff2cd771c5503dcb30c6161b21d154430ededa5faa1ec54366e"
40+
},
2841
"subtests.subtest.test_hmatch_imissing": {
2942
"status": "passed",
3043
"image_status": null,
@@ -64,6 +77,19 @@
6477
"baseline_hash": "d1ff61bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43",
6578
"result_hash": "443361bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43"
6679
},
80+
"subtests.subtest.test_hdiff_idiffshape": {
81+
"status": "failed",
82+
"image_status": null,
83+
"hash_status": "diff",
84+
"status_msg": "REGEX:Hash 5d37745bbdf2aac6743dbd830afb1877c2ac25a5f926d4f6483c1e24d19a0580 doesn't match hash d1ff745bbdf2aac6743dbd830afb1877c2ac25a5f926d4f6483c1e24d19a0580 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiffshape\\.",
85+
"baseline_image": null,
86+
"diff_image": null,
87+
"rms": null,
88+
"tolerance": null,
89+
"result_image": "subtests.subtest.test_hdiff_idiffshape/result.png",
90+
"baseline_hash": "d1ff745bbdf2aac6743dbd830afb1877c2ac25a5f926d4f6483c1e24d19a0580",
91+
"result_hash": "5d37745bbdf2aac6743dbd830afb1877c2ac25a5f926d4f6483c1e24d19a0580"
92+
},
6793
"subtests.subtest.test_hdiff_imissing": {
6894
"status": "failed",
6995
"status_msg": "REGEX:Hash 301e63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e doesn't match hash d1ff63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imissing\\.",
@@ -103,6 +129,19 @@
103129
"baseline_hash": null,
104130
"result_hash": "e69570c4a70b2cc88ddee0f0a82312cae4f394b7f62e5760245feda1364c03ab"
105131
},
132+
"subtests.subtest.test_hmissing_idiffshape": {
133+
"status": "failed",
134+
"image_status": null,
135+
"hash_status": "missing",
136+
"status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_idiffshape' not found in .*\\.json\\. Generated hash is 204d43b643538cafafa2d0fb7dafac9cf7009dbb421d4b32c71da2e400ceb59d\\.",
137+
"baseline_image": null,
138+
"diff_image": null,
139+
"rms": null,
140+
"tolerance": null,
141+
"result_image": "subtests.subtest.test_hmissing_idiffshape/result.png",
142+
"baseline_hash": null,
143+
"result_hash": "204d43b643538cafafa2d0fb7dafac9cf7009dbb421d4b32c71da2e400ceb59d"
144+
},
106145
"subtests.subtest.test_hmissing_imissing": {
107146
"status": "failed",
108147
"status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_imissing' not found in .*\\.json\\. Generated hash is 5c8a9c7412e4e098f6f2683ee247c08bd50805a93df4d4b6d8fccf3579b4c56b\\.",

Diff for: ‎tests/subtests/summaries/test_html_images_only.json

+39
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,19 @@
2525
"baseline_hash": null,
2626
"result_hash": null
2727
},
28+
"subtests.subtest.test_hmatch_idiffshape": {
29+
"status": "failed",
30+
"image_status": "diff",
31+
"hash_status": null,
32+
"status_msg": "REGEX:Error: Image dimensions did not match\\.\n Expected shape: \\(400, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png",
33+
"baseline_image": "subtests.subtest.test_hmatch_idiffshape/baseline.png",
34+
"diff_image": null,
35+
"rms": null,
36+
"tolerance": null,
37+
"result_image": "subtests.subtest.test_hmatch_idiffshape/result.png",
38+
"baseline_hash": null,
39+
"result_hash": null
40+
},
2841
"subtests.subtest.test_hmatch_imissing": {
2942
"status": "failed",
3043
"image_status": "missing",
@@ -64,6 +77,19 @@
6477
"baseline_hash": null,
6578
"result_hash": null
6679
},
80+
"subtests.subtest.test_hdiff_idiffshape": {
81+
"status": "failed",
82+
"image_status": "diff",
83+
"hash_status": null,
84+
"status_msg": "REGEX:Error: Image dimensions did not match\\.\n Expected shape: \\(400, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png",
85+
"baseline_image": "subtests.subtest.test_hdiff_idiffshape/baseline.png",
86+
"diff_image": null,
87+
"rms": null,
88+
"tolerance": null,
89+
"result_image": "subtests.subtest.test_hdiff_idiffshape/result.png",
90+
"baseline_hash": null,
91+
"result_hash": null
92+
},
6793
"subtests.subtest.test_hdiff_imissing": {
6894
"status": "failed",
6995
"image_status": "missing",
@@ -103,6 +129,19 @@
103129
"baseline_hash": null,
104130
"result_hash": null
105131
},
132+
"subtests.subtest.test_hmissing_idiffshape": {
133+
"status": "failed",
134+
"image_status": "diff",
135+
"hash_status": null,
136+
"status_msg": "REGEX:Error: Image dimensions did not match\\.\n Expected shape: \\(400, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png",
137+
"baseline_image": "subtests.subtest.test_hmissing_idiffshape/baseline.png",
138+
"diff_image": null,
139+
"rms": null,
140+
"tolerance": null,
141+
"result_image": "subtests.subtest.test_hmissing_idiffshape/result.png",
142+
"baseline_hash": null,
143+
"result_hash": null
144+
},
106145
"subtests.subtest.test_hmissing_imissing": {
107146
"status": "failed",
108147
"image_status": "missing",

Diff for: ‎tests/subtests/summaries/test_hybrid.json

+39
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,19 @@
2525
"baseline_hash": "8b5c00365e6f784d5c8947091f09a92fd7d222e790431f297b4848f173e64857",
2626
"result_hash": "8b5c00365e6f784d5c8947091f09a92fd7d222e790431f297b4848f173e64857"
2727
},
28+
"subtests.subtest.test_hmatch_idiffshape": {
29+
"status": "passed",
30+
"image_status": null,
31+
"hash_status": "match",
32+
"status_msg": "Test hash matches baseline hash.",
33+
"baseline_image": null,
34+
"diff_image": null,
35+
"rms": null,
36+
"tolerance": null,
37+
"result_image": null,
38+
"baseline_hash": "e3fed4ad2d22aff2cd771c5503dcb30c6161b21d154430ededa5faa1ec54366e",
39+
"result_hash": "e3fed4ad2d22aff2cd771c5503dcb30c6161b21d154430ededa5faa1ec54366e"
40+
},
2841
"subtests.subtest.test_hmatch_imissing": {
2942
"status": "passed",
3043
"image_status": null,
@@ -64,6 +77,19 @@
6477
"baseline_hash": "d1ff61bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43",
6578
"result_hash": "443361bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43"
6679
},
80+
"subtests.subtest.test_hdiff_idiffshape": {
81+
"status": "failed",
82+
"image_status": "diff",
83+
"hash_status": "diff",
84+
"status_msg": "REGEX:Hash 5d37745bbdf2aac6743dbd830afb1877c2ac25a5f926d4f6483c1e24d19a0580 doesn't match hash d1ff745bbdf2aac6743dbd830afb1877c2ac25a5f926d4f6483c1e24d19a0580 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiffshape\\.\n\nImage comparison test\n---------------------\nError: Image dimensions did not match\\.\n Expected shape: \\(400, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png",
85+
"baseline_image": "subtests.subtest.test_hdiff_idiffshape/baseline.png",
86+
"diff_image": null,
87+
"rms": null,
88+
"tolerance": null,
89+
"result_image": "subtests.subtest.test_hdiff_idiffshape/result.png",
90+
"baseline_hash": "d1ff745bbdf2aac6743dbd830afb1877c2ac25a5f926d4f6483c1e24d19a0580",
91+
"result_hash": "5d37745bbdf2aac6743dbd830afb1877c2ac25a5f926d4f6483c1e24d19a0580"
92+
},
6793
"subtests.subtest.test_hdiff_imissing": {
6894
"status": "failed",
6995
"status_msg": "REGEX:Hash 301e63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e doesn't match hash d1ff63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imissing\\.\n\nImage comparison test\n---------------------\nImage file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png",
@@ -103,6 +129,19 @@
103129
"baseline_hash": null,
104130
"result_hash": "e69570c4a70b2cc88ddee0f0a82312cae4f394b7f62e5760245feda1364c03ab"
105131
},
132+
"subtests.subtest.test_hmissing_idiffshape": {
133+
"status": "failed",
134+
"image_status": "diff",
135+
"hash_status": "missing",
136+
"status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_idiffshape' not found in .*\\.json\\. Generated hash is 204d43b643538cafafa2d0fb7dafac9cf7009dbb421d4b32c71da2e400ceb59d\\.\n\nImage comparison test\n---------------------\nError: Image dimensions did not match\\.\n Expected shape: \\(400, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png",
137+
"baseline_image": "subtests.subtest.test_hmissing_idiffshape/baseline.png",
138+
"diff_image": null,
139+
"rms": null,
140+
"tolerance": null,
141+
"result_image": "subtests.subtest.test_hmissing_idiffshape/result.png",
142+
"baseline_hash": null,
143+
"result_hash": "204d43b643538cafafa2d0fb7dafac9cf7009dbb421d4b32c71da2e400ceb59d"
144+
},
106145
"subtests.subtest.test_hmissing_imissing": {
107146
"status": "failed",
108147
"status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_imissing' not found in .*\\.json\\. Generated hash is 5c8a9c7412e4e098f6f2683ee247c08bd50805a93df4d4b6d8fccf3579b4c56b\\.\n\nImage comparison test\n---------------------\nImage file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png",

Diff for: ‎tests/subtests/summaries/test_results_always.json

+39
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,19 @@
2525
"baseline_hash": "8b5c00365e6f784d5c8947091f09a92fd7d222e790431f297b4848f173e64857",
2626
"result_hash": "8b5c00365e6f784d5c8947091f09a92fd7d222e790431f297b4848f173e64857"
2727
},
28+
"subtests.subtest.test_hmatch_idiffshape": {
29+
"status": "passed",
30+
"image_status": "diff",
31+
"hash_status": "match",
32+
"status_msg": "REGEX:Test hash matches baseline hash\\.\n\nImage comparison test\n---------------------\nError: Image dimensions did not match\\.\n Expected shape: \\(400, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png",
33+
"baseline_image": "subtests.subtest.test_hmatch_idiffshape/baseline.png",
34+
"diff_image": null,
35+
"rms": null,
36+
"tolerance": null,
37+
"result_image": "subtests.subtest.test_hmatch_idiffshape/result.png",
38+
"baseline_hash": "e3fed4ad2d22aff2cd771c5503dcb30c6161b21d154430ededa5faa1ec54366e",
39+
"result_hash": "e3fed4ad2d22aff2cd771c5503dcb30c6161b21d154430ededa5faa1ec54366e"
40+
},
2841
"subtests.subtest.test_hmatch_imissing": {
2942
"status": "passed",
3043
"image_status": "missing",
@@ -64,6 +77,19 @@
6477
"baseline_hash": "d1ff61bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43",
6578
"result_hash": "443361bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43"
6679
},
80+
"subtests.subtest.test_hdiff_idiffshape": {
81+
"status": "failed",
82+
"image_status": "diff",
83+
"hash_status": "diff",
84+
"status_msg": "REGEX:Hash 5d37745bbdf2aac6743dbd830afb1877c2ac25a5f926d4f6483c1e24d19a0580 doesn't match hash d1ff745bbdf2aac6743dbd830afb1877c2ac25a5f926d4f6483c1e24d19a0580 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiffshape\\.\n\nImage comparison test\n---------------------\nError: Image dimensions did not match\\.\n Expected shape: \\(400, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png",
85+
"baseline_image": "subtests.subtest.test_hdiff_idiffshape/baseline.png",
86+
"diff_image": null,
87+
"rms": null,
88+
"tolerance": null,
89+
"result_image": "subtests.subtest.test_hdiff_idiffshape/result.png",
90+
"baseline_hash": "d1ff745bbdf2aac6743dbd830afb1877c2ac25a5f926d4f6483c1e24d19a0580",
91+
"result_hash": "5d37745bbdf2aac6743dbd830afb1877c2ac25a5f926d4f6483c1e24d19a0580"
92+
},
6793
"subtests.subtest.test_hdiff_imissing": {
6894
"status": "failed",
6995
"status_msg": "REGEX:Hash 301e63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e doesn't match hash d1ff63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imissing\\.\n\nImage comparison test\n---------------------\nImage file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png",
@@ -103,6 +129,19 @@
103129
"baseline_hash": null,
104130
"result_hash": "e69570c4a70b2cc88ddee0f0a82312cae4f394b7f62e5760245feda1364c03ab"
105131
},
132+
"subtests.subtest.test_hmissing_idiffshape": {
133+
"status": "failed",
134+
"image_status": "diff",
135+
"hash_status": "missing",
136+
"status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_idiffshape' not found in .*\\.json\\. Generated hash is 204d43b643538cafafa2d0fb7dafac9cf7009dbb421d4b32c71da2e400ceb59d\\.\n\nImage comparison test\n---------------------\nError: Image dimensions did not match\\.\n Expected shape: \\(400, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png",
137+
"baseline_image": "subtests.subtest.test_hmissing_idiffshape/baseline.png",
138+
"diff_image": null,
139+
"rms": null,
140+
"tolerance": null,
141+
"result_image": "subtests.subtest.test_hmissing_idiffshape/result.png",
142+
"baseline_hash": null,
143+
"result_hash": "204d43b643538cafafa2d0fb7dafac9cf7009dbb421d4b32c71da2e400ceb59d"
144+
},
106145
"subtests.subtest.test_hmissing_imissing": {
107146
"status": "failed",
108147
"status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_imissing' not found in .*\\.json\\. Generated hash is 5c8a9c7412e4e098f6f2683ee247c08bd50805a93df4d4b6d8fccf3579b4c56b\\.\n\nImage comparison test\n---------------------\nImage file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png",

0 commit comments

Comments
 (0)
Please sign in to comment.