Replace ConditionalFact usages with custom facts (#2402)

* Replace ConditionalFact usages with custom facts

* Refactor ConditionalTheory usages

Add AttributeUsage attributes

* Make LessThanNetCore30OrNotNetCoreFactAttribute require an explicit skip message

* Use correct target framework moniker for netcoreapp3.0

* Remove CORECLR constant

* Remove an unused fact attribute

Author: mareklinka

test/Microsoft.ML.Benchmarks.Tests/BenchmarksTest.cs

@@ -12,7 +12,7 @@
 using BenchmarkDotNet.Loggers;
 using BenchmarkDotNet.Running;
 using Microsoft.ML.Benchmarks.Harness;
-using Microsoft.ML.Internal.CpuMath;
+using Microsoft.ML.TestFramework.Attributes;
 using Xunit;
 using Xunit.Abstractions;
 
@@ -29,15 +29,6 @@ public class BenchmarksTest
 
         private ITestOutputHelper Output { get; }
 
-        public static bool CanExecute =>
-#if DEBUG
-            false; // BenchmarkDotNet does not allow running the benchmarks in Debug, so this test is disabled for DEBUG
-#elif NET461
-            false; // We are currently not running Benchmarks for FullFramework
-#else
-            Environment.Is64BitProcess; // we don't support 32 bit yet
-#endif
-
         public static TheoryData<Type> GetBenchmarks()
         {
             TheoryData<Type> benchmarks = new TheoryData<Type>();
@@ -54,7 +45,7 @@ where Attribute.IsDefined(type, typeof(CIBenchmark))
             return benchmarks;
         }
 
-        [ConditionalTheory(typeof(BenchmarksTest), nameof(CanExecute))]
+        [BenchmarkTheory]
         [MemberData(nameof(GetBenchmarks))]
         public void BenchmarksProjectIsNotBroken(Type type)
         {

test/Microsoft.ML.Benchmarks/README.md

@@ -94,7 +94,7 @@ you can debug this test locally by:
 build.cmd -release -buildNative
 
 2- Changing the configuration in Visual Studio from Debug -> Release
-3- Changing the annotation in the `BenchmarksProjectIsNotBroken` to replace `ConditionalTheory` with `Theory`, as below. 
+3- Changing the annotation in the `BenchmarksProjectIsNotBroken` to replace `BenchmarkTheory` with `Theory`, as below. 
 
 ```cs
 [Theory]

test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs

@@ -22,6 +22,7 @@
 using Microsoft.ML.Internal.Utilities;
 using Microsoft.ML.LightGBM;
 using Microsoft.ML.Model.Onnx;
+using Microsoft.ML.TestFramework.Attributes;
 using Microsoft.ML.TimeSeriesProcessing;
 using Microsoft.ML.Trainers;
 using Microsoft.ML.Trainers.FastTree;
@@ -1312,7 +1313,7 @@ public void EntryPointMulticlassPipelineEnsemble()
             }
         }
 
-        [ConditionalFact(typeof(BaseTestBaseline), nameof(BaseTestBaseline.LessThanNetCore30OrNotNetCore))]
+        [LessThanNetCore30OrNotNetCoreFact("netcoreapp3.0 output differs from Baseline")]
         public void EntryPointPipelineEnsembleGetSummary()
         {
             var dataPath = GetDataPath("breast-cancer-withheader.txt");
@@ -1916,14 +1917,14 @@ public void EntryPointEvaluateRanking()
             }
         }
 
-        [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // LightGBM is 64-bit only
+        [LightGBMFact]
         public void EntryPointLightGbmBinary()
         {
             Env.ComponentCatalog.RegisterAssembly(typeof(LightGbmBinaryModelParameters).Assembly);
             TestEntryPointRoutine("breast-cancer.txt", "Trainers.LightGbmBinaryClassifier");
         }
 
-        [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // LightGBM is 64-bit only
+        [LightGBMFact]
         public void EntryPointLightGbmMultiClass()
         {
             Env.ComponentCatalog.RegisterAssembly(typeof(LightGbmBinaryModelParameters).Assembly);
@@ -3649,7 +3650,7 @@ public void EntryPointWordEmbeddings()
             }
         }
 
-        [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // TensorFlow is 64-bit only
+        [TensorFlowFact]
         public void EntryPointTensorFlowTransform()
         {
             Env.ComponentCatalog.RegisterAssembly(typeof(TensorFlowTransformer).Assembly);
@@ -4069,7 +4070,7 @@ public void TestSimpleTrainExperiment()
             }
         }
 
-        [ConditionalFact(typeof(BaseTestBaseline), nameof(BaseTestBaseline.LessThanNetCore30OrNotNetCore))] // netcore3.0 output differs from Baseline
+        [LessThanNetCore30OrNotNetCoreFact("netcoreapp3.0 output differs from Baseline")]
         public void TestCrossValidationMacro()
         {
             var dataPath = GetDataPath(TestDatasets.generatedRegressionDatasetmacro.trainFilename);
@@ -5537,7 +5538,7 @@ public void TestOvaMacroWithUncalibratedLearner()
             }
         }
 
-        [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // TensorFlow is 64-bit only
+        [TensorFlowFact]
         public void TestTensorFlowEntryPoint()
         {
             var dataPath = GetDataPath("Train-Tiny-28x28.txt");

test/Microsoft.ML.OnnxTransformTest/DnnImageFeaturizerTest.cs

@@ -13,6 +13,7 @@
 using Microsoft.ML.Model;
 using Microsoft.ML.RunTests;
 using Microsoft.ML.StaticPipe;
+using Microsoft.ML.TestFramework.Attributes;
 using Microsoft.ML.Transforms;
 using Microsoft.ML.Transforms.StaticPipe;
 using Xunit;
@@ -57,16 +58,9 @@ public DnnImageFeaturizerTests(ITestOutputHelper helper) : base(helper)
         {
         }
 
-        // Onnx is only supported on x64 Windows
-        [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))]
+        [OnnxFact]
         void TestDnnImageFeaturizer()
         {
-            // Onnxruntime supports Ubuntu 16.04, but not CentOS
-            // Do not execute on CentOS image
-            if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
-                return;
-
-
             var samplevector = GetSampleArrayData();
 
             var dataView = DataViewConstructionUtils.CreateFromList(Env,
@@ -97,13 +91,9 @@ void TestDnnImageFeaturizer()
             catch (InvalidOperationException) { }
         }
 
-        // Onnx is only supported on x64 Windows
-        [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))]
+        [OnnxFact]
         public void OnnxStatic()
         {
-            if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
-                return;
-
             var env = new MLContext(null, 1);
             var imageHeight = 224;
             var imageWidth = 224;
@@ -141,13 +131,9 @@ public void OnnxStatic()
         }
 
         // Onnx is only supported on x64 Windows
-        [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))]
+        [OnnxFact]
         public void TestOldSavingAndLoading()
         {
-            if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
-                return;
-
-
             var samplevector = GetSampleArrayData();
 
             var dataView = ML.Data.ReadFromEnumerable(

test/Microsoft.ML.OnnxTransformTest/OnnxTransformTests.cs

@@ -13,6 +13,7 @@
 using Microsoft.ML.Model;
 using Microsoft.ML.RunTests;
 using Microsoft.ML.StaticPipe;
+using Microsoft.ML.TestFramework.Attributes;
 using Microsoft.ML.Tools;
 using Microsoft.ML.Transforms;
 using Microsoft.ML.Transforms.StaticPipe;
@@ -21,24 +22,6 @@
 
 namespace Microsoft.ML.Tests
 {
-
-    /// <summary>
-    /// A Fact attribute for Onnx unit tests. Onnxruntime only supported
-    /// on Windows, Linux (Ubuntu 16.04) and 64-bit platforms.
-    /// </summary>
-    public class OnnxFact : FactAttribute
-    {
-        public OnnxFact()
-        {
-            if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux) ||
-                RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ||
-                !Environment.Is64BitProcess)
-            {
-                Skip = "Require 64 bit and Windows or Linux (Ubuntu 16.04).";
-            }
-        }
-    }
-
     public class OnnxTransformTests : TestDataPipeBase
     {
         private const int inputSize = 150528;
@@ -137,16 +120,12 @@ void TestSimpleCase()
             catch (ArgumentOutOfRangeException) { }
             catch (InvalidOperationException) { }
         }
-
-        // x86 not supported
-        [ConditionalTheory(typeof(Environment), nameof(Environment.Is64BitProcess))]
+ 
+        [OnnxTheory]
         [InlineData(null, false)]
         [InlineData(null, true)]
         void TestOldSavingAndLoading(int? gpuDeviceId, bool fallbackToCpu)
         {
-            if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
-                return;
-
             var modelFile = "squeezenet/00000001/model.onnx";
             var samplevector = GetSampleArrayData();