From cba4aa153125232b4a6461aade57699ee1399bd9 Mon Sep 17 00:00:00 2001 From: Gabriel Fouasnon Date: Tue, 10 Oct 2023 10:43:26 +0300 Subject: [PATCH 1/4] Use pytest-regressions with a11y tests to mark expected failures --- .pre-commit-config.yaml | 5 +- tests/test_a11y.py | 86 +++++++++++++++---- ...nk_admonitions_html__admonitions_dark_.yml | 1 + ...k_admonitions_html__admonitions_light_.yml | 1 + ...sink_api_html__api_documentation_dark_.yml | 1 + ...ink_api_html__api_documentation_light_.yml | 1 + ...kitchen_sink_blocks_html__blocks_dark_.yml | 1 + ...itchen_sink_blocks_html__blocks_light_.yml | 1 + ...sink_generic_html__generic_items_dark_.yml | 1 + ...ink_generic_html__generic_items_light_.yml | 1 + ...sink_images_html__images_figures_dark_.yml | 1 + ...ink_images_html__images_figures_light_.yml | 1 + ...s_kitchen_sink_lists_html__lists_dark_.yml | 1 + ..._kitchen_sink_lists_html__lists_light_.yml | 1 + ...ture_html__structural_elements_2_dark_.yml | 1 + ...ure_html__structural_elements_2_light_.yml | 1 + ...ucture_html__structural_elements_dark_.yml | 1 + ...cture_html__structural_elements_light_.yml | 1 + ...kitchen_sink_tables_html__tables_dark_.yml | 1 + ...itchen_sink_tables_html__tables_light_.yml | 1 + ...sink_typography_html__typography_dark_.yml | 1 + ...ink_typography_html__typography_light_.yml | 1 + ...data_html__pydata_library_styles_dark_.yml | 5 ++ ...ata_html__pydata_library_styles_light_.yml | 5 ++ ...ending_html__extending_the_theme_dark_.yml | 1 + ...nding_html__extending_the_theme_light_.yml | 1 + ...omium__user_guide_page_toc_html__dark_.yml | 1 + ...mium__user_guide_page_toc_html__light_.yml | 1 + ...ng_html__theme_variables_and_css_dark_.yml | 1 + ...g_html__theme_variables_and_css_light_.yml | 1 + ...ts_html__theme_specific_elements_dark_.yml | 1 + ...s_html__theme_specific_elements_light_.yml | 1 + ...s_html__sphinx_design_components_dark_.yml | 1 + ..._html__sphinx_design_components_light_.yml | 1 + tests/utils/pretty_axe_results.py | 47 ---------- 35 files changed, 112 insertions(+), 66 deletions(-) create mode 100644 tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_admonitions_html__admonitions_dark_.yml create mode 100644 tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_admonitions_html__admonitions_light_.yml create mode 100644 tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_api_html__api_documentation_dark_.yml create mode 100644 tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_api_html__api_documentation_light_.yml create mode 100644 tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_blocks_html__blocks_dark_.yml create mode 100644 tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_blocks_html__blocks_light_.yml create mode 100644 tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_generic_html__generic_items_dark_.yml create mode 100644 tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_generic_html__generic_items_light_.yml create mode 100644 tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_images_html__images_figures_dark_.yml create mode 100644 tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_images_html__images_figures_light_.yml create mode 100644 tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_lists_html__lists_dark_.yml create mode 100644 tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_lists_html__lists_light_.yml create mode 100644 tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_structure_html__structural_elements_2_dark_.yml create mode 100644 tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_structure_html__structural_elements_2_light_.yml create mode 100644 tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_structure_html__structural_elements_dark_.yml create mode 100644 tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_structure_html__structural_elements_light_.yml create mode 100644 tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_tables_html__tables_dark_.yml create mode 100644 tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_tables_html__tables_light_.yml create mode 100644 tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_typography_html__typography_dark_.yml create mode 100644 tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_typography_html__typography_light_.yml create mode 100644 tests/test_a11y/test_axe_core_chromium__examples_pydata_html__pydata_library_styles_dark_.yml create mode 100644 tests/test_a11y/test_axe_core_chromium__examples_pydata_html__pydata_library_styles_light_.yml create mode 100644 tests/test_a11y/test_axe_core_chromium__user_guide_extending_html__extending_the_theme_dark_.yml create mode 100644 tests/test_a11y/test_axe_core_chromium__user_guide_extending_html__extending_the_theme_light_.yml create mode 100644 tests/test_a11y/test_axe_core_chromium__user_guide_page_toc_html__dark_.yml create mode 100644 tests/test_a11y/test_axe_core_chromium__user_guide_page_toc_html__light_.yml create mode 100644 tests/test_a11y/test_axe_core_chromium__user_guide_styling_html__theme_variables_and_css_dark_.yml create mode 100644 tests/test_a11y/test_axe_core_chromium__user_guide_styling_html__theme_variables_and_css_light_.yml create mode 100644 tests/test_a11y/test_axe_core_chromium__user_guide_theme_elements_html__theme_specific_elements_dark_.yml create mode 100644 tests/test_a11y/test_axe_core_chromium__user_guide_theme_elements_html__theme_specific_elements_light_.yml create mode 100644 tests/test_a11y/test_axe_core_chromium__user_guide_web_components_html__sphinx_design_components_dark_.yml create mode 100644 tests/test_a11y/test_axe_core_chromium__user_guide_web_components_html__sphinx_design_components_light_.yml delete mode 100644 tests/utils/pretty_axe_results.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1b5e3aa41..f0100e807 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,8 +9,9 @@ repos: hooks: - id: prettier # Exclude the HTML, since it doesn't understand Jinja2 - # exclude also the webpack.congo.js file has it embed complete url dificult to prettify - exclude: .+\.html|webpack\.config\.js + # exclude also the webpack.config.js file has it embed complete url dificult to prettify + # exclude the pytest-regressions folder tests/test_ally + exclude: .+\.html|webpack\.config\.js|tests/test_a11y/ - repo: https://github.com/psf/black rev: 22.12.0 diff --git a/tests/test_a11y.py b/tests/test_a11y.py index b2710f38c..861ce9828 100644 --- a/tests/test_a11y.py +++ b/tests/test_a11y.py @@ -8,11 +8,9 @@ import pytest -from .utils.pretty_axe_results import pretty_axe_results - # Using importorskip to ensure these tests are only loaded if Playwright is installed. playwright = pytest.importorskip("playwright") -from playwright.sync_api import Page, expect # noqa: E402 +from playwright.sync_api import Page, expect # NOQA: E402 # Important note: automated accessibility scans can only find a fraction of # potential accessibility issues. @@ -72,26 +70,75 @@ def url_base(): process.wait() +def fingerprint_violations(accessibility_page_scan_violations): + """Create a fingerprint of the Axe violations array. + + https://playwright.dev/docs/accessibility-testing#using-snapshots-to-allow-specific-known-issues + """ + return [ + { + "id": violation["id"], + "help": violation["help"], + "helpUrl": violation["helpUrl"], + "targets": [node["target"] for node in violation["nodes"]], + } + for violation in accessibility_page_scan_violations + ] + + @pytest.mark.a11y @pytest.mark.parametrize("theme", ["light", "dark"]) @pytest.mark.parametrize( "url_pathname,selector", [ - ("/examples/kitchen-sink/admonitions.html", "#admonitions"), - ("/examples/kitchen-sink/api.html", "#api-documentation"), + ( + "/examples/kitchen-sink/admonitions.html", + "#admonitions", + ), + ( + "/examples/kitchen-sink/api.html", + "#api-documentation", + ), ("/examples/kitchen-sink/blocks.html", "#blocks"), - ("/examples/kitchen-sink/generic.html", "#generic-items"), - ("/examples/kitchen-sink/images.html", "#images-figures"), + ( + "/examples/kitchen-sink/generic.html", + "#generic-items", + ), + ( + "/examples/kitchen-sink/images.html", + "#images-figures", + ), ("/examples/kitchen-sink/lists.html", "#lists"), - ("/examples/kitchen-sink/structure.html", "#structural-elements"), - ("/examples/kitchen-sink/structure.html", "#structural-elements-2"), + ( + "/examples/kitchen-sink/structure.html", + "#structural-elements", + ), + ( + "/examples/kitchen-sink/structure.html", + "#structural-elements-2", + ), ("/examples/kitchen-sink/tables.html", "#tables"), - ("/examples/kitchen-sink/typography.html", "#typography"), + ( + "/examples/kitchen-sink/typography.html", + "#typography", + ), ("/examples/pydata.html", "#pydata-library-styles"), - ("/user_guide/theme-elements.html", "#theme-specific-elements"), - ("/user_guide/web-components.html", "#sphinx-design-components"), - ("/user_guide/extending.html", "#extending-the-theme"), - ("/user_guide/styling.html", "#theme-variables-and-css"), + ( + "/user_guide/theme-elements.html", + "#theme-specific-elements", + ), + ( + "/user_guide/web-components.html", + "#sphinx-design-components", + ), + ( + "/user_guide/extending.html", + "#extending-the-theme", + ), + ( + "/user_guide/styling.html", + "#theme-variables-and-css", + ), # Using one of the simplest pages on the site, select the whole page for # testing in order to effectively test repeated website elements like # nav, sidebars, breadcrumbs, footer @@ -99,7 +146,12 @@ def url_base(): ], ) def test_axe_core( - theme: str, url_base: str, url_pathname: str, selector: str, page: Page + data_regression, + theme: str, + url_base: str, + url_pathname: str, + selector: str, + page: Page, ): """Should have no Axe-core violations at the provided theme and page section.""" # Load the page at the provided path @@ -118,8 +170,8 @@ def test_axe_core( # the PyData Sphinx Theme documentation website.) results = page.evaluate("axe.run()" if selector == "" else f"axe.run('{selector}')") - # Expect Axe-core to have found 0 accessibility violations - assert len(results["violations"]) == 0, pretty_axe_results(results, selector) + # Check found violations against known violations that we do not plan to fix + data_regression.check(fingerprint_violations(results["violations"])) def test_version_switcher_highlighting(page: Page, url_base: str) -> None: diff --git a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_admonitions_html__admonitions_dark_.yml b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_admonitions_html__admonitions_dark_.yml new file mode 100644 index 000000000..fe51488c7 --- /dev/null +++ b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_admonitions_html__admonitions_dark_.yml @@ -0,0 +1 @@ +[] diff --git a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_admonitions_html__admonitions_light_.yml b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_admonitions_html__admonitions_light_.yml new file mode 100644 index 000000000..fe51488c7 --- /dev/null +++ b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_admonitions_html__admonitions_light_.yml @@ -0,0 +1 @@ +[] diff --git a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_api_html__api_documentation_dark_.yml b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_api_html__api_documentation_dark_.yml new file mode 100644 index 000000000..fe51488c7 --- /dev/null +++ b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_api_html__api_documentation_dark_.yml @@ -0,0 +1 @@ +[] diff --git a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_api_html__api_documentation_light_.yml b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_api_html__api_documentation_light_.yml new file mode 100644 index 000000000..fe51488c7 --- /dev/null +++ b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_api_html__api_documentation_light_.yml @@ -0,0 +1 @@ +[] diff --git a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_blocks_html__blocks_dark_.yml b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_blocks_html__blocks_dark_.yml new file mode 100644 index 000000000..fe51488c7 --- /dev/null +++ b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_blocks_html__blocks_dark_.yml @@ -0,0 +1 @@ +[] diff --git a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_blocks_html__blocks_light_.yml b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_blocks_html__blocks_light_.yml new file mode 100644 index 000000000..fe51488c7 --- /dev/null +++ b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_blocks_html__blocks_light_.yml @@ -0,0 +1 @@ +[] diff --git a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_generic_html__generic_items_dark_.yml b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_generic_html__generic_items_dark_.yml new file mode 100644 index 000000000..fe51488c7 --- /dev/null +++ b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_generic_html__generic_items_dark_.yml @@ -0,0 +1 @@ +[] diff --git a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_generic_html__generic_items_light_.yml b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_generic_html__generic_items_light_.yml new file mode 100644 index 000000000..fe51488c7 --- /dev/null +++ b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_generic_html__generic_items_light_.yml @@ -0,0 +1 @@ +[] diff --git a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_images_html__images_figures_dark_.yml b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_images_html__images_figures_dark_.yml new file mode 100644 index 000000000..fe51488c7 --- /dev/null +++ b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_images_html__images_figures_dark_.yml @@ -0,0 +1 @@ +[] diff --git a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_images_html__images_figures_light_.yml b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_images_html__images_figures_light_.yml new file mode 100644 index 000000000..fe51488c7 --- /dev/null +++ b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_images_html__images_figures_light_.yml @@ -0,0 +1 @@ +[] diff --git a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_lists_html__lists_dark_.yml b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_lists_html__lists_dark_.yml new file mode 100644 index 000000000..fe51488c7 --- /dev/null +++ b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_lists_html__lists_dark_.yml @@ -0,0 +1 @@ +[] diff --git a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_lists_html__lists_light_.yml b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_lists_html__lists_light_.yml new file mode 100644 index 000000000..fe51488c7 --- /dev/null +++ b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_lists_html__lists_light_.yml @@ -0,0 +1 @@ +[] diff --git a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_structure_html__structural_elements_2_dark_.yml b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_structure_html__structural_elements_2_dark_.yml new file mode 100644 index 000000000..fe51488c7 --- /dev/null +++ b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_structure_html__structural_elements_2_dark_.yml @@ -0,0 +1 @@ +[] diff --git a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_structure_html__structural_elements_2_light_.yml b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_structure_html__structural_elements_2_light_.yml new file mode 100644 index 000000000..fe51488c7 --- /dev/null +++ b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_structure_html__structural_elements_2_light_.yml @@ -0,0 +1 @@ +[] diff --git a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_structure_html__structural_elements_dark_.yml b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_structure_html__structural_elements_dark_.yml new file mode 100644 index 000000000..fe51488c7 --- /dev/null +++ b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_structure_html__structural_elements_dark_.yml @@ -0,0 +1 @@ +[] diff --git a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_structure_html__structural_elements_light_.yml b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_structure_html__structural_elements_light_.yml new file mode 100644 index 000000000..fe51488c7 --- /dev/null +++ b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_structure_html__structural_elements_light_.yml @@ -0,0 +1 @@ +[] diff --git a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_tables_html__tables_dark_.yml b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_tables_html__tables_dark_.yml new file mode 100644 index 000000000..fe51488c7 --- /dev/null +++ b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_tables_html__tables_dark_.yml @@ -0,0 +1 @@ +[] diff --git a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_tables_html__tables_light_.yml b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_tables_html__tables_light_.yml new file mode 100644 index 000000000..fe51488c7 --- /dev/null +++ b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_tables_html__tables_light_.yml @@ -0,0 +1 @@ +[] diff --git a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_typography_html__typography_dark_.yml b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_typography_html__typography_dark_.yml new file mode 100644 index 000000000..fe51488c7 --- /dev/null +++ b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_typography_html__typography_dark_.yml @@ -0,0 +1 @@ +[] diff --git a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_typography_html__typography_light_.yml b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_typography_html__typography_light_.yml new file mode 100644 index 000000000..fe51488c7 --- /dev/null +++ b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_typography_html__typography_light_.yml @@ -0,0 +1 @@ +[] diff --git a/tests/test_a11y/test_axe_core_chromium__examples_pydata_html__pydata_library_styles_dark_.yml b/tests/test_a11y/test_axe_core_chromium__examples_pydata_html__pydata_library_styles_dark_.yml new file mode 100644 index 000000000..b84d7335f --- /dev/null +++ b/tests/test_a11y/test_axe_core_chromium__examples_pydata_html__pydata_library_styles_dark_.yml @@ -0,0 +1,5 @@ +- help: Table header text should not be empty + helpUrl: https://dequeuniversity.com/rules/axe/4.6/empty-table-header?application=axeAPI + id: empty-table-header + targets: + - - thead > tr > th:nth-child(1) diff --git a/tests/test_a11y/test_axe_core_chromium__examples_pydata_html__pydata_library_styles_light_.yml b/tests/test_a11y/test_axe_core_chromium__examples_pydata_html__pydata_library_styles_light_.yml new file mode 100644 index 000000000..b84d7335f --- /dev/null +++ b/tests/test_a11y/test_axe_core_chromium__examples_pydata_html__pydata_library_styles_light_.yml @@ -0,0 +1,5 @@ +- help: Table header text should not be empty + helpUrl: https://dequeuniversity.com/rules/axe/4.6/empty-table-header?application=axeAPI + id: empty-table-header + targets: + - - thead > tr > th:nth-child(1) diff --git a/tests/test_a11y/test_axe_core_chromium__user_guide_extending_html__extending_the_theme_dark_.yml b/tests/test_a11y/test_axe_core_chromium__user_guide_extending_html__extending_the_theme_dark_.yml new file mode 100644 index 000000000..fe51488c7 --- /dev/null +++ b/tests/test_a11y/test_axe_core_chromium__user_guide_extending_html__extending_the_theme_dark_.yml @@ -0,0 +1 @@ +[] diff --git a/tests/test_a11y/test_axe_core_chromium__user_guide_extending_html__extending_the_theme_light_.yml b/tests/test_a11y/test_axe_core_chromium__user_guide_extending_html__extending_the_theme_light_.yml new file mode 100644 index 000000000..fe51488c7 --- /dev/null +++ b/tests/test_a11y/test_axe_core_chromium__user_guide_extending_html__extending_the_theme_light_.yml @@ -0,0 +1 @@ +[] diff --git a/tests/test_a11y/test_axe_core_chromium__user_guide_page_toc_html__dark_.yml b/tests/test_a11y/test_axe_core_chromium__user_guide_page_toc_html__dark_.yml new file mode 100644 index 000000000..fe51488c7 --- /dev/null +++ b/tests/test_a11y/test_axe_core_chromium__user_guide_page_toc_html__dark_.yml @@ -0,0 +1 @@ +[] diff --git a/tests/test_a11y/test_axe_core_chromium__user_guide_page_toc_html__light_.yml b/tests/test_a11y/test_axe_core_chromium__user_guide_page_toc_html__light_.yml new file mode 100644 index 000000000..fe51488c7 --- /dev/null +++ b/tests/test_a11y/test_axe_core_chromium__user_guide_page_toc_html__light_.yml @@ -0,0 +1 @@ +[] diff --git a/tests/test_a11y/test_axe_core_chromium__user_guide_styling_html__theme_variables_and_css_dark_.yml b/tests/test_a11y/test_axe_core_chromium__user_guide_styling_html__theme_variables_and_css_dark_.yml new file mode 100644 index 000000000..fe51488c7 --- /dev/null +++ b/tests/test_a11y/test_axe_core_chromium__user_guide_styling_html__theme_variables_and_css_dark_.yml @@ -0,0 +1 @@ +[] diff --git a/tests/test_a11y/test_axe_core_chromium__user_guide_styling_html__theme_variables_and_css_light_.yml b/tests/test_a11y/test_axe_core_chromium__user_guide_styling_html__theme_variables_and_css_light_.yml new file mode 100644 index 000000000..fe51488c7 --- /dev/null +++ b/tests/test_a11y/test_axe_core_chromium__user_guide_styling_html__theme_variables_and_css_light_.yml @@ -0,0 +1 @@ +[] diff --git a/tests/test_a11y/test_axe_core_chromium__user_guide_theme_elements_html__theme_specific_elements_dark_.yml b/tests/test_a11y/test_axe_core_chromium__user_guide_theme_elements_html__theme_specific_elements_dark_.yml new file mode 100644 index 000000000..fe51488c7 --- /dev/null +++ b/tests/test_a11y/test_axe_core_chromium__user_guide_theme_elements_html__theme_specific_elements_dark_.yml @@ -0,0 +1 @@ +[] diff --git a/tests/test_a11y/test_axe_core_chromium__user_guide_theme_elements_html__theme_specific_elements_light_.yml b/tests/test_a11y/test_axe_core_chromium__user_guide_theme_elements_html__theme_specific_elements_light_.yml new file mode 100644 index 000000000..fe51488c7 --- /dev/null +++ b/tests/test_a11y/test_axe_core_chromium__user_guide_theme_elements_html__theme_specific_elements_light_.yml @@ -0,0 +1 @@ +[] diff --git a/tests/test_a11y/test_axe_core_chromium__user_guide_web_components_html__sphinx_design_components_dark_.yml b/tests/test_a11y/test_axe_core_chromium__user_guide_web_components_html__sphinx_design_components_dark_.yml new file mode 100644 index 000000000..fe51488c7 --- /dev/null +++ b/tests/test_a11y/test_axe_core_chromium__user_guide_web_components_html__sphinx_design_components_dark_.yml @@ -0,0 +1 @@ +[] diff --git a/tests/test_a11y/test_axe_core_chromium__user_guide_web_components_html__sphinx_design_components_light_.yml b/tests/test_a11y/test_axe_core_chromium__user_guide_web_components_html__sphinx_design_components_light_.yml new file mode 100644 index 000000000..fe51488c7 --- /dev/null +++ b/tests/test_a11y/test_axe_core_chromium__user_guide_web_components_html__sphinx_design_components_light_.yml @@ -0,0 +1 @@ +[] diff --git a/tests/utils/pretty_axe_results.py b/tests/utils/pretty_axe_results.py deleted file mode 100644 index c5929e087..000000000 --- a/tests/utils/pretty_axe_results.py +++ /dev/null @@ -1,47 +0,0 @@ -"""Readable report of accessibility violations.""" - -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -# This code taken from: -# https://github.com/mozilla-services/axe-selenium-python/blob/3cfbdd67c9b40ab03f37b3ba2521f77c2071827b/axe_selenium_python/axe.py -# Typical usage: -# assert(len(results["violations"])) == 0, pretty_axe_results(results) - - -def pretty_axe_results(results: dict, selector: str) -> str: - """Create readable string that can be printed to console from the Axe-core results object. - - :param results: The object promised by `axe.run()`. - :type results: dict - :return report: Readable report of violations. - :rtype: string. - - """ - violations = results["violations"] - string = "" - string += f"Found {len(violations)} accessibility violations:" - for violation in violations: - string += ( - f"\n\n\nRule Violated:\n {violation['id']} - {violation['description']} \n\t" - f"URL: {violation['helpUrl']} \n\t" - f"Impact Level: {violation['impact']} \n\tTags:" - ) - for tag in violation["tags"]: - string += f" {tag}" - string += f"\n\tTested selector: {selector}" - string += "\n\tElements Affected:" - i = 1 - for node in violation["nodes"]: - for target in node["target"]: - string += f"\n\t {i} Target: {target}" - i += 1 - for item in node["all"]: - string += f"\n\t\t {item['message']}" - for item in node["any"]: - string += f"\n\t\t {item['message']}" - for item in node["none"]: - string += f"\n\t\t {item['message']}" - string += "\n\n\n" - return string From 9c8a700fa9a4b944ea1cc26dbd94a5a8ff7737a4 Mon Sep 17 00:00:00 2001 From: gabalafou Date: Fri, 8 Dec 2023 10:08:21 -0600 Subject: [PATCH 2/4] Update tests/test_a11y.py --- tests/test_a11y.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_a11y.py b/tests/test_a11y.py index 861ce9828..c462e43bf 100644 --- a/tests/test_a11y.py +++ b/tests/test_a11y.py @@ -10,7 +10,7 @@ # Using importorskip to ensure these tests are only loaded if Playwright is installed. playwright = pytest.importorskip("playwright") -from playwright.sync_api import Page, expect # NOQA: E402 +from playwright.sync_api import Page, expect # noqa: E402 # Important note: automated accessibility scans can only find a fraction of # potential accessibility issues. From 3b0c73e93d1f1b52f45b4918015c66fe5142c294 Mon Sep 17 00:00:00 2001 From: Gabriel Fouasnon Date: Fri, 8 Dec 2023 16:50:15 -0500 Subject: [PATCH 3/4] frozen data class --- tests/test_a11y.py | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/tests/test_a11y.py b/tests/test_a11y.py index c462e43bf..1d9ab6f5e 100644 --- a/tests/test_a11y.py +++ b/tests/test_a11y.py @@ -1,6 +1,7 @@ """Using Axe-core, scan the Kitchen Sink pages for accessibility violations.""" import time +from dataclasses import dataclass from http.client import HTTPConnection from pathlib import Path from subprocess import PIPE, Popen @@ -70,20 +71,30 @@ def url_base(): process.wait() -def fingerprint_violations(accessibility_page_scan_violations): +@dataclass(frozen=True) +class AxeViolationFingerprint: + """Fingerprint for single Axe violation.""" + + id: str + help: str + helpUrl: str + targets: tuple + + +def fingerprint(accessibility_page_scan_violations): """Create a fingerprint of the Axe violations array. https://playwright.dev/docs/accessibility-testing#using-snapshots-to-allow-specific-known-issues """ - return [ - { - "id": violation["id"], - "help": violation["help"], - "helpUrl": violation["helpUrl"], - "targets": [node["target"] for node in violation["nodes"]], - } + return tuple( + AxeViolationFingerprint( + violation["id"], + violation["help"], + violation["helpUrl"], + tuple(node["target"] for node in violation["nodes"]), + ) for violation in accessibility_page_scan_violations - ] + ) @pytest.mark.a11y @@ -171,7 +182,7 @@ def test_axe_core( results = page.evaluate("axe.run()" if selector == "" else f"axe.run('{selector}')") # Check found violations against known violations that we do not plan to fix - data_regression.check(fingerprint_violations(results["violations"])) + data_regression.check(fingerprint(results["violations"])) def test_version_switcher_highlighting(page: Page, url_base: str) -> None: From 1f81622a10ec7c32a2f7b7f2af5a58d3b76a0efc Mon Sep 17 00:00:00 2001 From: Daniel McCloy Date: Mon, 11 Dec 2023 10:51:05 -0600 Subject: [PATCH 4/4] Revert "frozen data class" This reverts commit 3b0c73e93d1f1b52f45b4918015c66fe5142c294. --- tests/test_a11y.py | 31 ++++++++++--------------------- 1 file changed, 10 insertions(+), 21 deletions(-) diff --git a/tests/test_a11y.py b/tests/test_a11y.py index 1d9ab6f5e..c462e43bf 100644 --- a/tests/test_a11y.py +++ b/tests/test_a11y.py @@ -1,7 +1,6 @@ """Using Axe-core, scan the Kitchen Sink pages for accessibility violations.""" import time -from dataclasses import dataclass from http.client import HTTPConnection from pathlib import Path from subprocess import PIPE, Popen @@ -71,30 +70,20 @@ def url_base(): process.wait() -@dataclass(frozen=True) -class AxeViolationFingerprint: - """Fingerprint for single Axe violation.""" - - id: str - help: str - helpUrl: str - targets: tuple - - -def fingerprint(accessibility_page_scan_violations): +def fingerprint_violations(accessibility_page_scan_violations): """Create a fingerprint of the Axe violations array. https://playwright.dev/docs/accessibility-testing#using-snapshots-to-allow-specific-known-issues """ - return tuple( - AxeViolationFingerprint( - violation["id"], - violation["help"], - violation["helpUrl"], - tuple(node["target"] for node in violation["nodes"]), - ) + return [ + { + "id": violation["id"], + "help": violation["help"], + "helpUrl": violation["helpUrl"], + "targets": [node["target"] for node in violation["nodes"]], + } for violation in accessibility_page_scan_violations - ) + ] @pytest.mark.a11y @@ -182,7 +171,7 @@ def test_axe_core( results = page.evaluate("axe.run()" if selector == "" else f"axe.run('{selector}')") # Check found violations against known violations that we do not plan to fix - data_regression.check(fingerprint(results["violations"])) + data_regression.check(fingerprint_violations(results["violations"])) def test_version_switcher_highlighting(page: Page, url_base: str) -> None: