From 2a389bc5e6317d0e6dd1fe5f917bd1c670d89ad3 Mon Sep 17 00:00:00 2001 From: Vasilis Vryniotis Date: Mon, 5 Sep 2022 15:13:28 +0100 Subject: [PATCH 1/3] Skip big models per platform/device --- test/test_models.py | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/test/test_models.py b/test/test_models.py index b70314f17c5..5f724a4907d 100644 --- a/test/test_models.py +++ b/test/test_models.py @@ -3,9 +3,11 @@ import operator import os import pkgutil +import platform import sys import warnings from collections import OrderedDict +from itertools import product from tempfile import TemporaryDirectory from typing import Any @@ -343,12 +345,27 @@ def _check_input_backprop(model, inputs): _model_params[m] = {"input_shape": (1, 3, 64, 64)} -# skip big models to reduce memory usage on CI test +# skip big models to reduce memory usage on CI test. We can exclude combinations of (platform-system, device). +_all_platforms = ("Darwin", "Linux", "Windows") +_all_devices = ("cpu", "cuda") skipped_big_models = { - "vit_h_14", - "regnet_y_128gf", + "vit_h_14": set(product(_all_platforms, _all_devices)), + "regnet_y_128gf": set(product(_all_platforms, _all_devices)), + "mvit_v1_b": {("Windows", "cuda")}, + "mvit_v2_s": {("Windows", "cuda")}, } + +def is_skippable(model_name, device): + if model_name not in skipped_big_models: + return False + + platform_system = platform.system() + device_name = str(device).split(":")[0] + + return (platform_system, device_name) in skipped_big_models[model_name] + + # The following contains configuration and expected values to be used tests that are model specific _model_tests_values = { "retinanet_resnet50_fpn": { @@ -612,7 +629,7 @@ def test_classification_model(model_fn, dev): "input_shape": (1, 3, 224, 224), } model_name = model_fn.__name__ - if SKIP_BIG_MODEL and model_name in skipped_big_models: + if SKIP_BIG_MODEL and is_skippable(model_name, dev): pytest.skip("Skipped to reduce memory usage. Set env var SKIP_BIG_MODEL=0 to enable test for this model") kwargs = {**defaults, **_model_params.get(model_name, {})} num_classes = kwargs.get("num_classes") @@ -841,7 +858,7 @@ def test_video_model(model_fn, dev): "num_classes": 50, } model_name = model_fn.__name__ - if SKIP_BIG_MODEL and model_name in skipped_big_models: + if SKIP_BIG_MODEL and is_skippable(model_name, dev): pytest.skip("Skipped to reduce memory usage. Set env var SKIP_BIG_MODEL=0 to enable test for this model") kwargs = {**defaults, **_model_params.get(model_name, {})} num_classes = kwargs.get("num_classes") From 5b9ceb9634f59a2d81accb4abe51b4ccb5345376 Mon Sep 17 00:00:00 2001 From: Vasilis Vryniotis Date: Mon, 5 Sep 2022 15:33:28 +0100 Subject: [PATCH 2/3] Specifying skips on Windows only. --- test/test_models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_models.py b/test/test_models.py index 5f724a4907d..28e19cc6f39 100644 --- a/test/test_models.py +++ b/test/test_models.py @@ -349,8 +349,8 @@ def _check_input_backprop(model, inputs): _all_platforms = ("Darwin", "Linux", "Windows") _all_devices = ("cpu", "cuda") skipped_big_models = { - "vit_h_14": set(product(_all_platforms, _all_devices)), - "regnet_y_128gf": set(product(_all_platforms, _all_devices)), + "vit_h_14": set(product(("Windows",), _all_devices)), + "regnet_y_128gf": set(product(("Windows",), _all_devices)), "mvit_v1_b": {("Windows", "cuda")}, "mvit_v2_s": {("Windows", "cuda")}, } From d1133a5eadf2ec3ed98c57f4cef5d9e0649c8fdf Mon Sep 17 00:00:00 2001 From: Vasilis Vryniotis Date: Mon, 5 Sep 2022 16:32:10 +0100 Subject: [PATCH 3/3] Simplify and clean up code. --- test/test_models.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/test/test_models.py b/test/test_models.py index 28e19cc6f39..7645dc419ff 100644 --- a/test/test_models.py +++ b/test/test_models.py @@ -7,7 +7,6 @@ import sys import warnings from collections import OrderedDict -from itertools import product from tempfile import TemporaryDirectory from typing import Any @@ -346,11 +345,9 @@ def _check_input_backprop(model, inputs): # skip big models to reduce memory usage on CI test. We can exclude combinations of (platform-system, device). -_all_platforms = ("Darwin", "Linux", "Windows") -_all_devices = ("cpu", "cuda") skipped_big_models = { - "vit_h_14": set(product(("Windows",), _all_devices)), - "regnet_y_128gf": set(product(("Windows",), _all_devices)), + "vit_h_14": {("Windows", "cpu"), ("Windows", "cuda")}, + "regnet_y_128gf": {("Windows", "cpu"), ("Windows", "cuda")}, "mvit_v1_b": {("Windows", "cuda")}, "mvit_v2_s": {("Windows", "cuda")}, }