diff --git a/test/Microsoft.ML.Benchmarks.Tests/BenchmarksTest.cs b/test/Microsoft.ML.Benchmarks.Tests/BenchmarksTest.cs index 940ca83dd6..9dc6854e8b 100644 --- a/test/Microsoft.ML.Benchmarks.Tests/BenchmarksTest.cs +++ b/test/Microsoft.ML.Benchmarks.Tests/BenchmarksTest.cs @@ -12,7 +12,7 @@ using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Running; using Microsoft.ML.Benchmarks.Harness; -using Microsoft.ML.Internal.CpuMath; +using Microsoft.ML.TestFramework.Attributes; using Xunit; using Xunit.Abstractions; @@ -29,15 +29,6 @@ public class BenchmarksTest private ITestOutputHelper Output { get; } - public static bool CanExecute => -#if DEBUG - false; // BenchmarkDotNet does not allow running the benchmarks in Debug, so this test is disabled for DEBUG -#elif NET461 - false; // We are currently not running Benchmarks for FullFramework -#else - Environment.Is64BitProcess; // we don't support 32 bit yet -#endif - public static TheoryData GetBenchmarks() { TheoryData benchmarks = new TheoryData(); @@ -54,7 +45,7 @@ where Attribute.IsDefined(type, typeof(CIBenchmark)) return benchmarks; } - [ConditionalTheory(typeof(BenchmarksTest), nameof(CanExecute))] + [BenchmarkTheory] [MemberData(nameof(GetBenchmarks))] public void BenchmarksProjectIsNotBroken(Type type) { diff --git a/test/Microsoft.ML.Benchmarks/README.md b/test/Microsoft.ML.Benchmarks/README.md index 4cefdfbaab..0f84b4ca46 100644 --- a/test/Microsoft.ML.Benchmarks/README.md +++ b/test/Microsoft.ML.Benchmarks/README.md @@ -94,7 +94,7 @@ you can debug this test locally by: build.cmd -release -buildNative 2- Changing the configuration in Visual Studio from Debug -> Release -3- Changing the annotation in the `BenchmarksProjectIsNotBroken` to replace `ConditionalTheory` with `Theory`, as below. +3- Changing the annotation in the `BenchmarksProjectIsNotBroken` to replace `BenchmarkTheory` with `Theory`, as below. ```cs [Theory] diff --git a/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs b/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs index c9ae3f9da3..4f5686c3d7 100644 --- a/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs +++ b/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs @@ -22,6 +22,7 @@ using Microsoft.ML.Internal.Utilities; using Microsoft.ML.LightGBM; using Microsoft.ML.Model.Onnx; +using Microsoft.ML.TestFramework.Attributes; using Microsoft.ML.TimeSeriesProcessing; using Microsoft.ML.Trainers; using Microsoft.ML.Trainers.FastTree; @@ -1312,7 +1313,7 @@ public void EntryPointMulticlassPipelineEnsemble() } } - [ConditionalFact(typeof(BaseTestBaseline), nameof(BaseTestBaseline.LessThanNetCore30OrNotNetCore))] + [LessThanNetCore30OrNotNetCoreFact("netcoreapp3.0 output differs from Baseline")] public void EntryPointPipelineEnsembleGetSummary() { var dataPath = GetDataPath("breast-cancer-withheader.txt"); @@ -1916,14 +1917,14 @@ public void EntryPointEvaluateRanking() } } - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // LightGBM is 64-bit only + [LightGBMFact] public void EntryPointLightGbmBinary() { Env.ComponentCatalog.RegisterAssembly(typeof(LightGbmBinaryModelParameters).Assembly); TestEntryPointRoutine("breast-cancer.txt", "Trainers.LightGbmBinaryClassifier"); } - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // LightGBM is 64-bit only + [LightGBMFact] public void EntryPointLightGbmMultiClass() { Env.ComponentCatalog.RegisterAssembly(typeof(LightGbmBinaryModelParameters).Assembly); @@ -3649,7 +3650,7 @@ public void EntryPointWordEmbeddings() } } - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // TensorFlow is 64-bit only + [TensorFlowFact] public void EntryPointTensorFlowTransform() { Env.ComponentCatalog.RegisterAssembly(typeof(TensorFlowTransformer).Assembly); @@ -4069,7 +4070,7 @@ public void TestSimpleTrainExperiment() } } - [ConditionalFact(typeof(BaseTestBaseline), nameof(BaseTestBaseline.LessThanNetCore30OrNotNetCore))] // netcore3.0 output differs from Baseline + [LessThanNetCore30OrNotNetCoreFact("netcoreapp3.0 output differs from Baseline")] public void TestCrossValidationMacro() { var dataPath = GetDataPath(TestDatasets.generatedRegressionDatasetmacro.trainFilename); @@ -5537,7 +5538,7 @@ public void TestOvaMacroWithUncalibratedLearner() } } - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // TensorFlow is 64-bit only + [TensorFlowFact] public void TestTensorFlowEntryPoint() { var dataPath = GetDataPath("Train-Tiny-28x28.txt"); diff --git a/test/Microsoft.ML.OnnxTransformTest/DnnImageFeaturizerTest.cs b/test/Microsoft.ML.OnnxTransformTest/DnnImageFeaturizerTest.cs index 70ec3dffc8..2ebb55ee4d 100644 --- a/test/Microsoft.ML.OnnxTransformTest/DnnImageFeaturizerTest.cs +++ b/test/Microsoft.ML.OnnxTransformTest/DnnImageFeaturizerTest.cs @@ -13,6 +13,7 @@ using Microsoft.ML.Model; using Microsoft.ML.RunTests; using Microsoft.ML.StaticPipe; +using Microsoft.ML.TestFramework.Attributes; using Microsoft.ML.Transforms; using Microsoft.ML.Transforms.StaticPipe; using Xunit; @@ -57,16 +58,9 @@ public DnnImageFeaturizerTests(ITestOutputHelper helper) : base(helper) { } - // Onnx is only supported on x64 Windows - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] + [OnnxFact] void TestDnnImageFeaturizer() { - // Onnxruntime supports Ubuntu 16.04, but not CentOS - // Do not execute on CentOS image - if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) - return; - - var samplevector = GetSampleArrayData(); var dataView = DataViewConstructionUtils.CreateFromList(Env, @@ -97,13 +91,9 @@ void TestDnnImageFeaturizer() catch (InvalidOperationException) { } } - // Onnx is only supported on x64 Windows - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] + [OnnxFact] public void OnnxStatic() { - if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) - return; - var env = new MLContext(null, 1); var imageHeight = 224; var imageWidth = 224; @@ -141,13 +131,9 @@ public void OnnxStatic() } // Onnx is only supported on x64 Windows - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] + [OnnxFact] public void TestOldSavingAndLoading() { - if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) - return; - - var samplevector = GetSampleArrayData(); var dataView = ML.Data.ReadFromEnumerable( diff --git a/test/Microsoft.ML.OnnxTransformTest/OnnxTransformTests.cs b/test/Microsoft.ML.OnnxTransformTest/OnnxTransformTests.cs index 904282a7a2..6b95e217db 100644 --- a/test/Microsoft.ML.OnnxTransformTest/OnnxTransformTests.cs +++ b/test/Microsoft.ML.OnnxTransformTest/OnnxTransformTests.cs @@ -13,6 +13,7 @@ using Microsoft.ML.Model; using Microsoft.ML.RunTests; using Microsoft.ML.StaticPipe; +using Microsoft.ML.TestFramework.Attributes; using Microsoft.ML.Tools; using Microsoft.ML.Transforms; using Microsoft.ML.Transforms.StaticPipe; @@ -21,24 +22,6 @@ namespace Microsoft.ML.Tests { - - /// - /// A Fact attribute for Onnx unit tests. Onnxruntime only supported - /// on Windows, Linux (Ubuntu 16.04) and 64-bit platforms. - /// - public class OnnxFact : FactAttribute - { - public OnnxFact() - { - if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux) || - RuntimeInformation.IsOSPlatform(OSPlatform.OSX) || - !Environment.Is64BitProcess) - { - Skip = "Require 64 bit and Windows or Linux (Ubuntu 16.04)."; - } - } - } - public class OnnxTransformTests : TestDataPipeBase { private const int inputSize = 150528; @@ -137,16 +120,12 @@ void TestSimpleCase() catch (ArgumentOutOfRangeException) { } catch (InvalidOperationException) { } } - - // x86 not supported - [ConditionalTheory(typeof(Environment), nameof(Environment.Is64BitProcess))] + + [OnnxTheory] [InlineData(null, false)] [InlineData(null, true)] void TestOldSavingAndLoading(int? gpuDeviceId, bool fallbackToCpu) { - if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) - return; - var modelFile = "squeezenet/00000001/model.onnx"; var samplevector = GetSampleArrayData(); diff --git a/test/Microsoft.ML.Predictor.Tests/TestPredictors.cs b/test/Microsoft.ML.Predictor.Tests/TestPredictors.cs index 4ecd744c36..0af5ad72b5 100644 --- a/test/Microsoft.ML.Predictor.Tests/TestPredictors.cs +++ b/test/Microsoft.ML.Predictor.Tests/TestPredictors.cs @@ -5,6 +5,7 @@ using System; using System.Collections.Generic; using System.IO; +using Microsoft.ML.TestFramework.Attributes; using Float = System.Single; namespace Microsoft.ML.RunTests @@ -160,7 +161,7 @@ public void EarlyStoppingTest() /// /// Multiclass Logistic Regression test. /// - [ConditionalFact(typeof(BaseTestBaseline), nameof(BaseTestBaseline.LessThanNetCore30OrNotNetCore))] // netcore3.0 output differs from Baseline + [LessThanNetCore30OrNotNetCoreFact("netcoreapp3.0 output differs from Baseline")] [TestCategory("Multiclass")] [TestCategory("Logistic Regression")] public void MulticlassLRTest() @@ -172,7 +173,7 @@ public void MulticlassLRTest() /// /// Multiclass Logistic Regression with non-negative coefficients test. /// - [ConditionalFact(typeof(BaseTestBaseline), nameof(BaseTestBaseline.LessThanNetCore30OrNotNetCore))] // netcore3.0 output differs from Baseline + [LessThanNetCore30OrNotNetCoreFact("netcoreapp3.0 output differs from Baseline")] [TestCategory("Multiclass")] [TestCategory("Logistic Regression")] public void MulticlassLRNonNegativeTest() @@ -196,7 +197,7 @@ public void MulticlassSdcaTest() /// /// Multiclass Logistic Regression test with a tree featurizer. /// - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // x86 output differs from Baseline + [X64Fact("x86 output differs from Baseline")] [TestCategory("Multiclass")] [TestCategory("Logistic Regression")] [TestCategory("FastTree")] @@ -254,7 +255,7 @@ public void KMeansClusteringTest() Done(); } - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // x86 output differs from Baseline + [X64Fact("x86 output differs from Baseline")] [TestCategory("Binary")] [TestCategory("SDCA")] public void LinearClassifierTest() @@ -275,7 +276,7 @@ public void LinearClassifierTest() /// ///A test for binary classifiers /// - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // x86 output differs from Baseline + [X64Fact("x86 output differs from Baseline")] [TestCategory("Binary")] public void BinaryClassifierLogisticRegressionTest() { @@ -285,7 +286,7 @@ public void BinaryClassifierLogisticRegressionTest() Done(); } - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // x86 output differs from Baseline + [X64Fact("x86 output differs from Baseline")] [TestCategory("Binary")] public void BinaryClassifierSymSgdTest() { @@ -297,7 +298,7 @@ public void BinaryClassifierSymSgdTest() Done(); } - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // x86 output differs from Baseline + [X64Fact("x86 output differs from Baseline")] [TestCategory("Binary")] public void BinaryClassifierTesterThresholdingTest() { @@ -323,7 +324,7 @@ public void BinaryClassifierLogisticRegressionNormTest() /// ///A test for binary classifiers with non-negative coefficients /// - [ConditionalFact(typeof(BaseTestBaseline), nameof(BaseTestBaseline.LessThanNetCore30OrNotNetCoreAnd64BitProcess))] // netcore3.0 and x86 output differs from Baseline + [LessThanNetCore30OrNotNetCoreAndX64Fact("netcoreapp3.0 and x86 output differs from Baseline")] [TestCategory("Binary")] public void BinaryClassifierLogisticRegressionNonNegativeTest() { @@ -336,7 +337,7 @@ public void BinaryClassifierLogisticRegressionNonNegativeTest() /// ///A test for binary classifiers /// - [ConditionalFact(typeof(BaseTestBaseline), nameof(BaseTestBaseline.LessThanNetCore30OrNotNetCore))] // netcore3.0 output differs from Baseline + [LessThanNetCore30OrNotNetCoreFact("netcoreapp3.0 output differs from Baseline")] [TestCategory("Binary")] public void BinaryClassifierLogisticRegressionBinNormTest() { @@ -349,7 +350,7 @@ public void BinaryClassifierLogisticRegressionBinNormTest() /// ///A test for binary classifiers /// - [ConditionalFact(typeof(BaseTestBaseline), nameof(BaseTestBaseline.LessThanNetCore30OrNotNetCoreAnd64BitProcess))] // x86 output differs from Baseline and flaky on netcore 3.0 + [LessThanNetCore30OrNotNetCoreAndX64Fact("x86 output differs from Baseline and flaky on netcore 3.0")] [TestCategory("Binary")] public void BinaryClassifierLogisticRegressionGaussianNormTest() { @@ -386,7 +387,7 @@ public void BinaryClassifierFastRankClassificationTest() /// ///A test for binary classifiers /// - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // x86 output differs from Baseline + [X64Fact("x86 output differs from Baseline")] [TestCategory("Binary")] [TestCategory("FastForest")] public void FastForestClassificationTest() @@ -453,7 +454,7 @@ public void WeightingFastForestRegressionPredictorsTest() Done(); } - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // x86 output differs from Baseline + [X64Fact("x86 output differs from Baseline")] [TestCategory("Binary")] [TestCategory("FastTree")] public void FastTreeBinaryClassificationTest() @@ -472,7 +473,7 @@ public void FastTreeBinaryClassificationTest() Done(); } - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // LightGBM is 64-bit only + [LightGBMFact] [TestCategory("Binary")] [TestCategory("LightGBM")] public void LightGBMClassificationTest() @@ -488,7 +489,7 @@ public void LightGBMClassificationTest() Done(); } - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // LightGBM is 64-bit only + [LightGBMFact] [TestCategory("Binary"), TestCategory("LightGBM")] public void GossLightGBMTest() { @@ -498,7 +499,7 @@ public void GossLightGBMTest() Done(); } - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // LightGBM is 64-bit only + [LightGBMFact] [TestCategory("Binary")] [TestCategory("LightGBM")] public void DartLightGBMTest() @@ -512,7 +513,7 @@ public void DartLightGBMTest() /// /// A test for multi class classifiers. /// - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // LightGBM is 64-bit only + [LightGBMFact] [TestCategory("Multiclass")] [TestCategory("LightGBM")] public void MultiClassifierLightGBMKeyLabelTest() @@ -526,7 +527,7 @@ public void MultiClassifierLightGBMKeyLabelTest() /// /// A test for multi class classifiers. /// - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // LightGBM is 64-bit only + [LightGBMFact] [TestCategory("Multiclass")] [TestCategory("LightGBM")] public void MultiClassifierLightGBMKeyLabelU404Test() @@ -540,7 +541,7 @@ public void MultiClassifierLightGBMKeyLabelU404Test() /// /// A test for regression. /// - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // LightGBM is 64-bit only + [LightGBMFact] [TestCategory("Regression")] [TestCategory("LightGBM")] public void RegressorLightGBMTest() @@ -554,7 +555,7 @@ public void RegressorLightGBMTest() /// /// A test for regression. /// - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // LightGBM is 64-bit only + [LightGBMFact] [TestCategory("Regression")] [TestCategory("LightGBM")] public void RegressorLightGBMMAETest() @@ -568,7 +569,7 @@ public void RegressorLightGBMMAETest() /// /// A test for regression. /// - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // LightGBM is 64-bit only + [LightGBMFact] [TestCategory("Regression")] [TestCategory("LightGBM")] public void RegressorLightGBMRMSETest() @@ -600,8 +601,7 @@ public void RankingLightGBMTest() Done(); } - // x86 fails. Associated GitHubIssue: https://github.com/dotnet/machinelearning/issues/1216 - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] + [X64Fact("x86 fails. Associated GitHubIssue: https://github.com/dotnet/machinelearning/issues/1216")] public void TestTreeEnsembleCombiner() { var dataPath = GetDataPath("breast-cancer.txt"); @@ -622,8 +622,7 @@ public void TestTreeEnsembleCombiner() CombineAndTestTreeEnsembles(dataView, fastTrees); } - // x86 fails. Associated GitHubIssue: https://github.com/dotnet/machinelearning/issues/1216 - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] + [X64Fact("x86 fails. Associated GitHubIssue: https://github.com/dotnet/machinelearning/issues/1216")] public void TestTreeEnsembleCombinerWithCategoricalSplits() { var dataPath = GetDataPath("adult.tiny.with-schema.txt"); @@ -725,8 +724,7 @@ private void CombineAndTestTreeEnsembles(IDataView idv, PredictorModel[] fastTre } } - // x86 fails. Associated GitHubIssue: https://github.com/dotnet/machinelearning/issues/1216 - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] + [X64Fact("x86 fails. Associated GitHubIssue: https://github.com/dotnet/machinelearning/issues/1216")] public void TestEnsembleCombiner() { var dataPath = GetDataPath("breast-cancer.txt"); @@ -771,8 +769,7 @@ public void TestEnsembleCombiner() CombineAndTestEnsembles(dataView, "pe", "oc=average", PredictionKind.BinaryClassification, predictors); } - // x86 fails. Associated GitHubIssue: https://github.com/dotnet/machinelearning/issues/1216 - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] + [X64Fact("x86 fails. Associated GitHubIssue: https://github.com/dotnet/machinelearning/issues/1216")] public void TestMultiClassEnsembleCombiner() { var dataPath = GetDataPath("breast-cancer.txt"); @@ -941,7 +938,7 @@ private void CombineAndTestEnsembles(IDataView idv, string name, string options, } - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // x86 output differs from Baseline + [X64Fact("x86 output differs from Baseline")] [TestCategory("Binary")] [TestCategory("FastTree")] public void FastTreeBinaryClassificationCategoricalSplitTest() @@ -980,7 +977,7 @@ public void FastTreeRegressionCategoricalSplitTest() Done(); } - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // x86 output differs from Baseline + [X64Fact("x86 output differs from Baseline")] [TestCategory("Binary")] [TestCategory("FastTree")] public void FastTreeBinaryClassificationNoOpGroupIdTest() @@ -1000,7 +997,7 @@ public void FastTreeBinaryClassificationNoOpGroupIdTest() Done(); } - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // x86 output differs from Baseline + [X64Fact("x86 output differs from Baseline")] [TestCategory("Binary")] [TestCategory("FastTree")] public void FastTreeHighMinDocsTest() @@ -1575,7 +1572,7 @@ public IList GetDatasetsForCalibratorTest() /// ///A test for no calibrators /// - [ConditionalFact(typeof(BaseTestBaseline), nameof(BaseTestBaseline.LessThanNetCore30OrNotNetCore))] // netcore3.0 output differs from Baseline + [LessThanNetCore30OrNotNetCoreFact("netcoreapp3.0 output differs from Baseline")] [TestCategory("Calibrator")] public void DefaultCalibratorPerceptronTest() { @@ -1587,7 +1584,7 @@ public void DefaultCalibratorPerceptronTest() /// ///A test for PAV calibrators /// - [ConditionalFact(typeof(BaseTestBaseline), nameof(BaseTestBaseline.LessThanNetCore30OrNotNetCore))] // netcore3.0 output differs from Baseline + [LessThanNetCore30OrNotNetCoreFact("netcoreapp3.0 output differs from Baseline")] [TestCategory("Calibrator")] public void PAVCalibratorPerceptronTest() { @@ -1599,7 +1596,7 @@ public void PAVCalibratorPerceptronTest() /// ///A test for random calibrators /// - [ConditionalFact(typeof(BaseTestBaseline), nameof(BaseTestBaseline.LessThanNetCore30OrNotNetCoreAnd64BitProcess))] // netcore3.0 and x86 output differs from Baseline + [LessThanNetCore30OrNotNetCoreAndX64Fact("netcoreapp3.0 and x86 output differs from Baseline")] [TestCategory("Calibrator")] public void RandomCalibratorPerceptronTest() { diff --git a/test/Microsoft.ML.StaticPipelineTesting/Training.cs b/test/Microsoft.ML.StaticPipelineTesting/Training.cs index c14b5071d2..5f76129a89 100644 --- a/test/Microsoft.ML.StaticPipelineTesting/Training.cs +++ b/test/Microsoft.ML.StaticPipelineTesting/Training.cs @@ -14,6 +14,7 @@ using Microsoft.ML.LightGBM.StaticPipe; using Microsoft.ML.RunTests; using Microsoft.ML.StaticPipe; +using Microsoft.ML.TestFramework.Attributes; using Microsoft.ML.Trainers; using Microsoft.ML.Trainers.FastTree; using Microsoft.ML.Trainers.KMeans; @@ -448,7 +449,7 @@ public void FastTreeRegression() Assert.InRange(metrics.LossFn, 0, double.PositiveInfinity); } - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // LightGBM is 64-bit only + [LightGBMFact] public void LightGbmBinaryClassification() { var env = new MLContext(seed: 0); @@ -488,7 +489,7 @@ public void LightGbmBinaryClassification() Assert.InRange(metrics.Auprc, 0, 1); } - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // LightGBM is 64-bit only + [LightGBMFact] public void LightGbmRegression() { var env = new MLContext(seed: 0); @@ -804,7 +805,7 @@ public void FastTreeRanking() Assert.InRange(metrics.Ndcg[2], 36.5, 37); } - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // LightGBM is 64-bit only + [LightGBMFact] public void LightGBMRanking() { var env = new MLContext(seed: 0); @@ -845,7 +846,7 @@ public void LightGBMRanking() Assert.InRange(metrics.Ndcg[2], 36.5, 37); } - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // LightGBM is 64-bit only + [LightGBMFact] public void MultiClassLightGBM() { var env = new MLContext(seed: 0); @@ -966,7 +967,7 @@ public void HogwildSGDBinaryClassification() Assert.InRange(metrics.Auprc, 0, 1); } - [ConditionalFact(typeof(BaseTestBaseline), nameof(BaseTestBaseline.LessThanNetCore30OrNotNetCoreAnd64BitProcess))] // netcore3.0 and x86 output differs from Baseline. This test is being fixed as part of issue #1441. + [LessThanNetCore30OrNotNetCoreAndX64Fact("netcoreapp3.0 and x86 output differs from Baseline. Being tracked as part of https://github.com/dotnet/machinelearning/issues/1441")] public void MatrixFactorization() { // Create a new context for ML.NET operations. It can be used for exception tracking and logging, @@ -1017,7 +1018,7 @@ public void MatrixFactorization() Assert.InRange(metrics.L2, 0, 0.5); } - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // LightGBM is 64-bit only + [LightGBMFact] public void MultiClassLightGbmStaticPipelineWithInMemoryData() { // Create a general context for ML.NET operations. It can be used for exception tracking and logging, diff --git a/test/Microsoft.ML.TestFramework/Attributes/BenchmarkTheoryAttribute.cs b/test/Microsoft.ML.TestFramework/Attributes/BenchmarkTheoryAttribute.cs new file mode 100644 index 0000000000..fae467176e --- /dev/null +++ b/test/Microsoft.ML.TestFramework/Attributes/BenchmarkTheoryAttribute.cs @@ -0,0 +1,29 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace Microsoft.ML.TestFramework.Attributes +{ + /// + /// A theory for BenchmarkDotNet tests. + /// + public sealed class BenchmarkTheoryAttribute : EnvironmentSpecificTheoryAttribute + { +#if DEBUG + private const string SkipMessage = "BenchmarkDotNet does not allow running the benchmarks in Debug, so this test is disabled for DEBUG"; + private readonly bool _isEnvironmentSupported = false; +#elif NET461 + private const string SkipMessage = "We are currently not running Benchmarks for FullFramework"; + private readonly bool _isEnvironmentSupported = false; +#else + private const string SkipMessage = "We don't support 32 bit yet"; + private readonly bool _isEnvironmentSupported = System.Environment.Is64BitProcess; +#endif + + public BenchmarkTheoryAttribute() : base(SkipMessage) + { + } + + protected override bool IsEnvironmentSupported() => _isEnvironmentSupported; + } +} \ No newline at end of file diff --git a/test/Microsoft.ML.TestFramework/Attributes/EnvironmentSpecificFactAttribute.cs b/test/Microsoft.ML.TestFramework/Attributes/EnvironmentSpecificFactAttribute.cs new file mode 100644 index 0000000000..fe754b6eb2 --- /dev/null +++ b/test/Microsoft.ML.TestFramework/Attributes/EnvironmentSpecificFactAttribute.cs @@ -0,0 +1,34 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using Xunit; + +namespace Microsoft.ML.TestFramework.Attributes +{ + /// + /// A base class for environment-specific fact attributes. + /// + [AttributeUsage(AttributeTargets.Method, AllowMultiple = false, Inherited = true)] + public abstract class EnvironmentSpecificFactAttribute : FactAttribute + { + private readonly string _skipMessage; + + /// + /// Creates a new instance of the class. + /// + /// The message to be used when skipping the test marked with this attribute. + protected EnvironmentSpecificFactAttribute(string skipMessage) + { + _skipMessage = skipMessage ?? throw new ArgumentNullException(nameof(skipMessage)); + } + + public sealed override string Skip => IsEnvironmentSupported() ? null : _skipMessage; + + /// + /// A method used to evaluate whether to skip a test marked with this attribute. Skips iff this method evaluates to false. + /// + protected abstract bool IsEnvironmentSupported(); + } +} \ No newline at end of file diff --git a/test/Microsoft.ML.TestFramework/Attributes/EnvironmentSpecificTheoryAttribute.cs b/test/Microsoft.ML.TestFramework/Attributes/EnvironmentSpecificTheoryAttribute.cs new file mode 100644 index 0000000000..c8f4b44c56 --- /dev/null +++ b/test/Microsoft.ML.TestFramework/Attributes/EnvironmentSpecificTheoryAttribute.cs @@ -0,0 +1,34 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using Xunit; + +namespace Microsoft.ML.TestFramework.Attributes +{ + /// + /// A base class for environment-specific fact attributes. + /// + [AttributeUsage(AttributeTargets.Method, AllowMultiple = false, Inherited = true)] + public abstract class EnvironmentSpecificTheoryAttribute : TheoryAttribute + { + private readonly string _skipMessage; + + /// + /// Creates a new instance of the class. + /// + /// The message to be used when skipping the test marked with this attribute. + protected EnvironmentSpecificTheoryAttribute(string skipMessage) + { + _skipMessage = skipMessage ?? throw new ArgumentNullException(nameof(skipMessage)); + } + + public sealed override string Skip => IsEnvironmentSupported() ? null : _skipMessage; + + /// + /// A method used to evaluate whether to skip a test marked with this attribute. Skips iff this method evaluates to false. + /// + protected abstract bool IsEnvironmentSupported(); + } +} \ No newline at end of file diff --git a/test/Microsoft.ML.TestFramework/Attributes/LessThanNetCore30OrNotNetCoreAndX64FactAttribute.cs b/test/Microsoft.ML.TestFramework/Attributes/LessThanNetCore30OrNotNetCoreAndX64FactAttribute.cs new file mode 100644 index 0000000000..4d8814b5d6 --- /dev/null +++ b/test/Microsoft.ML.TestFramework/Attributes/LessThanNetCore30OrNotNetCoreAndX64FactAttribute.cs @@ -0,0 +1,24 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; + +namespace Microsoft.ML.TestFramework.Attributes +{ + /// + /// A fact for tests requiring x64 environment and either .NET Core version lower than 3.0 or framework other than .NET Core. + /// + public sealed class LessThanNetCore30OrNotNetCoreAndX64FactAttribute : EnvironmentSpecificFactAttribute + { + public LessThanNetCore30OrNotNetCoreAndX64FactAttribute(string skipMessage) : base(skipMessage) + { + } + + /// + protected override bool IsEnvironmentSupported() + { + return Environment.Is64BitProcess && AppDomain.CurrentDomain.GetData("FX_PRODUCT_VERSION") == null; + } + } +} \ No newline at end of file diff --git a/test/Microsoft.ML.TestFramework/Attributes/LessThanNetCore30OrNotNetCoreFactAttribute.cs b/test/Microsoft.ML.TestFramework/Attributes/LessThanNetCore30OrNotNetCoreFactAttribute.cs new file mode 100644 index 0000000000..7fdfdf2708 --- /dev/null +++ b/test/Microsoft.ML.TestFramework/Attributes/LessThanNetCore30OrNotNetCoreFactAttribute.cs @@ -0,0 +1,24 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; + +namespace Microsoft.ML.TestFramework.Attributes +{ + /// + /// A fact for tests requiring either .NET Core version lower than 3.0 or framework other than .NET Core. + /// + public sealed class LessThanNetCore30OrNotNetCoreFactAttribute : EnvironmentSpecificFactAttribute + { + public LessThanNetCore30OrNotNetCoreFactAttribute(string skipMessage) : base(skipMessage) + { + } + + /// + protected override bool IsEnvironmentSupported() + { + return AppDomain.CurrentDomain.GetData("FX_PRODUCT_VERSION") == null; + } + } +} \ No newline at end of file diff --git a/test/Microsoft.ML.TestFramework/Attributes/LightGBMFactAttribute.cs b/test/Microsoft.ML.TestFramework/Attributes/LightGBMFactAttribute.cs new file mode 100644 index 0000000000..726d50e970 --- /dev/null +++ b/test/Microsoft.ML.TestFramework/Attributes/LightGBMFactAttribute.cs @@ -0,0 +1,24 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; + +namespace Microsoft.ML.TestFramework.Attributes +{ + /// + /// A fact for tests requiring LightGBM. + /// + public sealed class LightGBMFactAttribute : EnvironmentSpecificFactAttribute + { + public LightGBMFactAttribute() : base("LightGBM is 64-bit only") + { + } + + /// + protected override bool IsEnvironmentSupported() + { + return Environment.Is64BitProcess; + } + } +} \ No newline at end of file diff --git a/test/Microsoft.ML.TestFramework/Attributes/MatrixFactorizationFactAttribute.cs b/test/Microsoft.ML.TestFramework/Attributes/MatrixFactorizationFactAttribute.cs new file mode 100644 index 0000000000..8ba0bba3e1 --- /dev/null +++ b/test/Microsoft.ML.TestFramework/Attributes/MatrixFactorizationFactAttribute.cs @@ -0,0 +1,24 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; + +namespace Microsoft.ML.TestFramework.Attributes +{ + /// + /// A fact for tests requiring matrix factorization. + /// + public sealed class MatrixFactorizationFactAttribute : EnvironmentSpecificFactAttribute + { + public MatrixFactorizationFactAttribute() : base("Disabled - this test is being fixed as part of https://github.com/dotnet/machinelearning/issues/1441") + { + } + + /// + protected override bool IsEnvironmentSupported() + { + return Environment.Is64BitProcess; + } + } +} \ No newline at end of file diff --git a/test/Microsoft.ML.TestFramework/Attributes/OnnxFactAttribute.cs b/test/Microsoft.ML.TestFramework/Attributes/OnnxFactAttribute.cs new file mode 100644 index 0000000000..d2ed6763d6 --- /dev/null +++ b/test/Microsoft.ML.TestFramework/Attributes/OnnxFactAttribute.cs @@ -0,0 +1,25 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Runtime.InteropServices; + +namespace Microsoft.ML.TestFramework.Attributes +{ + /// + /// A fact for tests requiring Onnx. + /// + public sealed class OnnxFactAttribute : EnvironmentSpecificFactAttribute + { + public OnnxFactAttribute() : base("Onnx is 64-bit Windows only") + { + } + + /// + protected override bool IsEnvironmentSupported() + { + return Environment.Is64BitProcess && RuntimeInformation.IsOSPlatform(OSPlatform.Windows); + } + } +} \ No newline at end of file diff --git a/test/Microsoft.ML.TestFramework/Attributes/OnnxTheoryAttribute.cs b/test/Microsoft.ML.TestFramework/Attributes/OnnxTheoryAttribute.cs new file mode 100644 index 0000000000..979654c9c3 --- /dev/null +++ b/test/Microsoft.ML.TestFramework/Attributes/OnnxTheoryAttribute.cs @@ -0,0 +1,25 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Runtime.InteropServices; + +namespace Microsoft.ML.TestFramework.Attributes +{ + /// + /// A fact for tests requiring Onnx. + /// + public sealed class OnnxTheoryAttribute : EnvironmentSpecificTheoryAttribute + { + public OnnxTheoryAttribute() : base("Onnx is 64-bit Windows only") + { + } + + /// + protected override bool IsEnvironmentSupported() + { + return Environment.Is64BitProcess && RuntimeInformation.IsOSPlatform(OSPlatform.Windows); + } + } +} \ No newline at end of file diff --git a/test/Microsoft.ML.TestFramework/Attributes/TensorflowFactAttribute.cs b/test/Microsoft.ML.TestFramework/Attributes/TensorflowFactAttribute.cs new file mode 100644 index 0000000000..7711e4d05f --- /dev/null +++ b/test/Microsoft.ML.TestFramework/Attributes/TensorflowFactAttribute.cs @@ -0,0 +1,24 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; + +namespace Microsoft.ML.TestFramework.Attributes +{ + /// + /// A fact for tests requiring TensorFlow. + /// + public sealed class TensorFlowFactAttribute : EnvironmentSpecificFactAttribute + { + public TensorFlowFactAttribute() : base("TensorFlow is 64-bit only") + { + } + + /// + protected override bool IsEnvironmentSupported() + { + return Environment.Is64BitProcess; + } + } +} diff --git a/test/Microsoft.ML.TestFramework/Attributes/X64FactAttribute.cs b/test/Microsoft.ML.TestFramework/Attributes/X64FactAttribute.cs new file mode 100644 index 0000000000..dcf8814bb8 --- /dev/null +++ b/test/Microsoft.ML.TestFramework/Attributes/X64FactAttribute.cs @@ -0,0 +1,24 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; + +namespace Microsoft.ML.TestFramework.Attributes +{ + /// + /// A fact for tests requiring X64 environment. + /// + public sealed class X64FactAttribute : EnvironmentSpecificFactAttribute + { + public X64FactAttribute(string skipMessage) : base(skipMessage) + { + } + + /// + protected override bool IsEnvironmentSupported() + { + return Environment.Is64BitProcess; + } + } +} \ No newline at end of file diff --git a/test/Microsoft.ML.TestFramework/BaseTestBaseline.cs b/test/Microsoft.ML.TestFramework/BaseTestBaseline.cs index b3c2faf3be..43d89d9982 100644 --- a/test/Microsoft.ML.TestFramework/BaseTestBaseline.cs +++ b/test/Microsoft.ML.TestFramework/BaseTestBaseline.cs @@ -24,20 +24,8 @@ public abstract partial class BaseTestBaseline : BaseTestClass { public const int DigitsOfPrecision = 7; - public static bool NotFullFramework { get; set;} = true; - - public static bool LessThanNetCore30OrNotNetCore { get; } = AppDomain.CurrentDomain.GetData("FX_PRODUCT_VERSION") == null ? true : false; - - public static bool LessThanNetCore30AndNotFullFramework { get; set; } = LessThanNetCore30OrNotNetCore; - - public static bool LessThanNetCore30OrNotNetCoreAnd64BitProcess { get; } = LessThanNetCore30OrNotNetCore && Environment.Is64BitProcess; - protected BaseTestBaseline(ITestOutputHelper output) : base(output) { -#if NETFRAMEWORK - NotFullFramework = false; - LessThanNetCore30AndNotFullFramework = false; -#endif } internal const string RawSuffix = ".raw"; @@ -777,9 +765,6 @@ public void RunMTAThread(ThreadStart fn) } }); t.IsBackground = true; -#if !CORECLR // CoreCLR does not support apartment state settings for threads. - t.SetApartmentState(ApartmentState.MTA); -#endif t.Start(); t.Join(); if (inner != null) @@ -802,11 +787,8 @@ public void RunMTAThread(ThreadStart fn) protected static StreamWriter OpenWriter(string path, bool append = false, Encoding encoding = null, int bufferSize = 1024) { Contracts.CheckNonWhiteSpace(path, nameof(path)); -#if CORECLR + return Utils.OpenWriter(File.Open(path, append ? FileMode.Append : FileMode.OpenOrCreate), encoding, bufferSize, false); -#else - return new StreamWriter(path, append); -#endif } /// @@ -817,11 +799,8 @@ protected static StreamWriter OpenWriter(string path, bool append = false, Encod protected static StreamReader OpenReader(string path) { Contracts.CheckNonWhiteSpace(path, nameof(path)); -#if CORECLR + return new StreamReader(File.Open(path, FileMode.Open, FileAccess.Read, FileShare.Read)); -#else - return new StreamReader(path); -#endif } /// diff --git a/test/Microsoft.ML.TestFramework/DataPipe/TestDataPipeBase.cs b/test/Microsoft.ML.TestFramework/DataPipe/TestDataPipeBase.cs index 61149b7022..fa7c4c8ea0 100644 --- a/test/Microsoft.ML.TestFramework/DataPipe/TestDataPipeBase.cs +++ b/test/Microsoft.ML.TestFramework/DataPipe/TestDataPipeBase.cs @@ -1096,23 +1096,6 @@ protected Func GetColumnComparer(Row r1, Row r2, int col, ColumnType type, return GetComparerVec(r1, r2, col, size, (x, y) => x.Equals(y)); } -#if !CORECLR // REVIEW: Port Picture type to CoreTLC. - if (type is PictureType) - { - var g1 = r1.GetGetter(col); - var g2 = r2.GetGetter(col); - Picture v1 = null; - Picture v2 = null; - return - () => - { - g1(ref v1); - g2(ref v2); - return ComparePicture(v1, v2); - }; - } -#endif - throw Contracts.Except("Unknown type in GetColumnComparer: '{0}'", type); } @@ -1250,36 +1233,5 @@ protected void VerifyVecEquality(ValueGetter> vecGetter, ValueGett vecNGetter(ref fvn); Assert.True(CompareVec(in fv, in fvn, size, compare)); } - -#if !CORECLR - // REVIEW: Port Picture type to Core TLC. - protected bool ComparePicture(Picture v1, Picture v2) - { - if (v1 == null || v2 == null) - return v1 == v2; - - var p1 = v1.Contents.Pixels; - var p2 = v2.Contents.Pixels; - - if (p1.Width != p2.Width) - return false; - if (p1.Height != p2.Height) - return false; - if (p1.PixelFormat != p2.PixelFormat) - return false; - - for (int y = 0; y < p1.Height; y++) - { - for (int x = 0; x < p1.Width; x++) - { - var x1 = p1.GetPixel(x, y); - var x2 = p2.GetPixel(x, y); - if (x1 != x2) - return false; - } - } - return true; - } -#endif } } diff --git a/test/Microsoft.ML.TestFramework/Microsoft.ML.TestFramework.csproj b/test/Microsoft.ML.TestFramework/Microsoft.ML.TestFramework.csproj index a8a0ad7000..d24892a1a8 100644 --- a/test/Microsoft.ML.TestFramework/Microsoft.ML.TestFramework.csproj +++ b/test/Microsoft.ML.TestFramework/Microsoft.ML.TestFramework.csproj @@ -1,8 +1,4 @@  - - CORECLR - - diff --git a/test/Microsoft.ML.TestFramework/TestCommandBase.cs b/test/Microsoft.ML.TestFramework/TestCommandBase.cs index cb3a88782f..4475568a6b 100644 --- a/test/Microsoft.ML.TestFramework/TestCommandBase.cs +++ b/test/Microsoft.ML.TestFramework/TestCommandBase.cs @@ -15,6 +15,7 @@ using Microsoft.ML.Internal.Utilities; using Microsoft.ML.Model; using Microsoft.ML.TestFramework; +using Microsoft.ML.TestFramework.Attributes; using Microsoft.ML.Tools; using Xunit; using Xunit.Abstractions; @@ -838,7 +839,7 @@ public void CommandCrossValidation() Done(); } - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // x86 output differs from Baseline + [X64Fact("x86 output differs from Baseline")] public void CommandCrossValidationKeyLabelWithFloatKeyValues() { RunMTAThread(() => @@ -1170,7 +1171,7 @@ public void CommandTrainMlrWithLabelNames() Done(); } - [ConditionalFact(typeof(BaseTestBaseline), nameof(BaseTestBaseline.LessThanNetCore30OrNotNetCore))] // netcore3.0 output differs from Baseline + [LessThanNetCore30OrNotNetCoreFact("netcoreapp3.0 output differs from Baseline")] [TestCategory(Cat), TestCategory("Multiclass"), TestCategory("Logistic Regression")] public void CommandTrainMlrWithStats() { diff --git a/test/Microsoft.ML.Tests/FeatureContributionTests.cs b/test/Microsoft.ML.Tests/FeatureContributionTests.cs index 446aed03a0..7833b5afae 100644 --- a/test/Microsoft.ML.Tests/FeatureContributionTests.cs +++ b/test/Microsoft.ML.Tests/FeatureContributionTests.cs @@ -10,6 +10,7 @@ using Microsoft.ML.Internal.Internallearn; using Microsoft.ML.Internal.Utilities; using Microsoft.ML.RunTests; +using Microsoft.ML.TestFramework.Attributes; using Microsoft.ML.Trainers; using Microsoft.ML.Training; using Microsoft.ML.Transforms; @@ -47,7 +48,7 @@ public void TestOrdinaryLeastSquaresRegression() TestFeatureContribution(ML.Regression.Trainers.OrdinaryLeastSquares(), GetSparseDataset(numberOfInstances: 100), "LeastSquaresRegression"); } - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // LightGBM is 64-bit only + [LightGBMFact] public void TestLightGbmRegression() { TestFeatureContribution(ML.Regression.Trainers.LightGbm(), GetSparseDataset(numberOfInstances: 100), "LightGbmRegression"); @@ -104,7 +105,7 @@ public void TestFastTreeRanking() TestFeatureContribution(ML.Ranking.Trainers.FastTree(), GetSparseDataset(TaskType.Ranking, 100), "FastTreeRanking"); } - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // LightGBM is 64-bit only + [LightGBMFact] public void TestLightGbmRanking() { TestFeatureContribution(ML.Ranking.Trainers.LightGbm(), GetSparseDataset(TaskType.Ranking, 100), "LightGbmRanking"); @@ -141,7 +142,7 @@ public void TestFastTreeBinary() TestFeatureContribution(ML.BinaryClassification.Trainers.FastTree(), GetSparseDataset(TaskType.BinaryClassification, 100), "FastTreeBinary"); } - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // LightGBM is 64-bit only + [LightGBMFact] public void TestLightGbmBinary() { TestFeatureContribution(ML.BinaryClassification.Trainers.LightGbm(), GetSparseDataset(TaskType.BinaryClassification, 100), "LightGbmBinary"); diff --git a/test/Microsoft.ML.Tests/OnnxConversionTest.cs b/test/Microsoft.ML.Tests/OnnxConversionTest.cs index 00b7435d91..4d8a06972f 100644 --- a/test/Microsoft.ML.Tests/OnnxConversionTest.cs +++ b/test/Microsoft.ML.Tests/OnnxConversionTest.cs @@ -13,6 +13,7 @@ using Microsoft.ML.Data; using Microsoft.ML.Model.Onnx; using Microsoft.ML.RunTests; +using Microsoft.ML.TestFramework.Attributes; using Microsoft.ML.Tools; using Microsoft.ML.Trainers; using Microsoft.ML.Transforms; @@ -118,7 +119,7 @@ private class BreastCancerMulticlassExample public float[] Features; } - [ConditionalFact(typeof(BaseTestBaseline), nameof(BaseTestBaseline.LessThanNetCore30OrNotNetCore))] // Tracked by https://github.com/dotnet/machinelearning/issues/2087 + [LessThanNetCore30OrNotNetCoreFact("netcoreapp3.0 output differs from Baseline. Tracked by https://github.com/dotnet/machinelearning/issues/2087")] public void KmeansOnnxConversionTest() { // Create a new context for ML.NET operations. It can be used for exception tracking and logging, @@ -330,7 +331,7 @@ public void LogisticRegressionOnnxConversionTest() Done(); } - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // LightGBM is 64-bit only + [LightGBMFact] public void LightGbmBinaryClassificationOnnxConversionTest() { // Step 1: Create and train a ML.NET pipeline. diff --git a/test/Microsoft.ML.Tests/Scenarios/TensorflowTests.cs b/test/Microsoft.ML.Tests/Scenarios/TensorflowTests.cs index c36fb058a5..8d615d9330 100644 --- a/test/Microsoft.ML.Tests/Scenarios/TensorflowTests.cs +++ b/test/Microsoft.ML.Tests/Scenarios/TensorflowTests.cs @@ -2,11 +2,10 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. -using System; using System.IO; using Microsoft.ML.Data; using Microsoft.ML.ImageAnalytics; -using Microsoft.ML.Trainers; +using Microsoft.ML.TestFramework.Attributes; using Microsoft.ML.Transforms; using Microsoft.ML.Transforms.Conversions; using Xunit; @@ -15,7 +14,7 @@ namespace Microsoft.ML.Scenarios { public partial class ScenariosTests { - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // TensorFlow is 64-bit only + [TensorFlowFact] public void TensorFlowTransforCifarEndToEndTest() { var imageHeight = 32; diff --git a/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs b/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs index 398d426684..f006c5a7f0 100644 --- a/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs +++ b/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs @@ -10,6 +10,7 @@ using Microsoft.ML.Data; using Microsoft.ML.ImageAnalytics; using Microsoft.ML.RunTests; +using Microsoft.ML.TestFramework.Attributes; using Microsoft.ML.Transforms; using Microsoft.ML.Transforms.Conversions; using Microsoft.ML.Transforms.Normalizers; @@ -29,7 +30,7 @@ private class TestData public float[] b; } - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // TensorFlow is 64-bit only + [TensorFlowFact] public void TensorFlowTransformMatrixMultiplicationTest() { var modelLocation = "model_matmul/frozen_saved_model.pb"; @@ -214,7 +215,7 @@ private class TypesData /// /// Test to ensure the supported datatypes can passed to TensorFlow . /// - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // TensorFlow is 64-bit only + [TensorFlowFact] public void TensorFlowTransformInputOutputTypesTest() { // This an identity model which returns the same output as input. @@ -408,7 +409,7 @@ public void TensorFlowTransformInceptionTest() } } - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // TensorFlow is 64-bit only + [TensorFlowFact] public void TensorFlowInputsOutputsSchemaTest() { var mlContext = new MLContext(seed: 1, conc: 1); @@ -485,7 +486,7 @@ public void TensorFlowInputsOutputsSchemaTest() } } - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // TensorFlow is 64-bit only + [TensorFlowFact] public void TensorFlowTransformMNISTConvTest() { var mlContext = new MLContext(seed: 1, conc: 1); @@ -523,7 +524,7 @@ public void TensorFlowTransformMNISTConvTest() Assert.Equal(5, GetMaxIndexForOnePrediction(onePrediction)); } - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // TensorFlow is 64-bit only + [TensorFlowFact] public void TensorFlowTransformMNISTLRTrainingTest() { const double expectedMicroAccuracy = 0.72173913043478266; @@ -608,7 +609,7 @@ private void CleanUp(string model_location) } } - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // TensorFlow is 64-bit only + [TensorFlowFact] public void TensorFlowTransformMNISTConvTrainingTest() { ExecuteTFTransformMNISTConvTrainingTest(false, null, 0.74782608695652175, 0.608843537414966); @@ -703,7 +704,7 @@ private void ExecuteTFTransformMNISTConvTrainingTest(bool shuffle, int? shuffleS } } - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // TensorFlow is 64-bit only + [TensorFlowFact] public void TensorFlowTransformMNISTConvSavedModelTest() { // This test trains a multi-class classifier pipeline where a pre-trained Tenroflow model is used for featurization. @@ -826,7 +827,7 @@ public class MNISTPrediction public float[] PredictedLabels; } - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // TensorFlow is 64-bit only + [TensorFlowFact] public void TensorFlowTransformCifar() { var modelLocation = "cifar_model/frozen_model.pb"; @@ -872,7 +873,7 @@ public void TensorFlowTransformCifar() } } - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // TensorFlow is 64-bit only + [TensorFlowFact] public void TensorFlowTransformCifarSavedModel() { var modelLocation = "cifar_saved_model"; @@ -914,7 +915,7 @@ public void TensorFlowTransformCifarSavedModel() } // This test has been created as result of https://github.com/dotnet/machinelearning/issues/2156. - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // TensorFlow is 64-bit only + [TensorFlowFact] public void TensorFlowGettingSchemaMultipleTimes() { var modelLocation = "cifar_saved_model"; @@ -927,7 +928,7 @@ public void TensorFlowGettingSchemaMultipleTimes() } - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] + [TensorFlowFact] public void TensorFlowTransformCifarInvalidShape() { var modelLocation = "cifar_model/frozen_model.pb"; @@ -972,7 +973,7 @@ public class TensorFlowSentiment public float[] Prediction; } - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] + [TensorFlowFact] public void TensorFlowSentimentClassificationTest() { var mlContext = new MLContext(seed: 1, conc: 1); diff --git a/test/Microsoft.ML.Tests/TensorFlowEstimatorTests.cs b/test/Microsoft.ML.Tests/TensorFlowEstimatorTests.cs index a7d75f86cb..9a05f95bbf 100644 --- a/test/Microsoft.ML.Tests/TensorFlowEstimatorTests.cs +++ b/test/Microsoft.ML.Tests/TensorFlowEstimatorTests.cs @@ -12,6 +12,7 @@ using Microsoft.ML.Model; using Microsoft.ML.RunTests; using Microsoft.ML.StaticPipe; +using Microsoft.ML.TestFramework.Attributes; using Microsoft.ML.Tools; using Microsoft.ML.Transforms; using Microsoft.ML.Transforms.StaticPipe; @@ -56,7 +57,7 @@ public TensorFlowEstimatorTests(ITestOutputHelper output) : base(output) { } - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // TensorFlow is 64-bit only + [TensorFlowFact] void TestSimpleCase() { var modelFile = "model_matmul/frozen_saved_model.pb"; @@ -96,7 +97,7 @@ void TestSimpleCase() catch (InvalidOperationException) { } } - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // TensorFlow is 64-bit only + [TensorFlowFact] void TestOldSavingAndLoading() { var modelFile = "model_matmul/frozen_saved_model.pb"; @@ -132,7 +133,7 @@ void TestOldSavingAndLoading() } } - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // x86 output differs from Baseline + [TensorFlowFact] void TestCommandLine() { // typeof helps to load the TensorFlowTransformer type. @@ -140,7 +141,7 @@ void TestCommandLine() Assert.Equal(Maml.Main(new[] { @"showschema loader=Text{col=a:R4:0-3 col=b:R4:0-3} xf=TFTransform{inputs=a inputs=b outputs=c modellocation={model_matmul/frozen_saved_model.pb}}" }), (int)0); } - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // TensorFlow is 64-bit only + [TensorFlowFact] public void TestTensorFlowStatic() { var modelLocation = "cifar_model/frozen_model.pb"; @@ -182,7 +183,7 @@ public void TestTensorFlowStatic() } } - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] + [TensorFlowFact] public void TestTensorFlowStaticWithSchema() { const string modelLocation = "cifar_model/frozen_model.pb"; diff --git a/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs b/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs index 64179fb039..2749961baf 100644 --- a/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs +++ b/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs @@ -10,6 +10,7 @@ using Microsoft.ML.Core.Data; using Microsoft.ML.Data; using Microsoft.ML.RunTests; +using Microsoft.ML.TestFramework.Attributes; using Microsoft.ML.Trainers; using Xunit; @@ -17,8 +18,8 @@ namespace Microsoft.ML.Tests.TrainerEstimators { public partial class TrainerEstimators : TestDataPipeBase { - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // This test is being fixed as part of issue #1441. - public void MatrixFactorizationEstimator() + [MatrixFactorizationFact] + public void MatrixFactorization_Estimator() { string labelColumnName = "Label"; string matrixColumnIndexColumnName = "Col"; @@ -50,7 +51,7 @@ public void MatrixFactorizationEstimator() Done(); } - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // This test is being fixed as part of issue #1441. + [MatrixFactorizationFact] public void MatrixFactorizationSimpleTrainAndPredict() { var mlContext = new MLContext(seed: 1, conc: 1); @@ -189,7 +190,7 @@ internal class MatrixElementForScore public float Score; } - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // This test is being fixed as part of issue #1441. + [MatrixFactorizationFact] public void MatrixFactorizationInMemoryData() { // Create an in-memory matrix as a list of tuples (column index, row index, value). @@ -278,7 +279,7 @@ internal class MatrixElementZeroBasedForScore public float Score; } - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // This test is being fixed as part of issue #1441. + [MatrixFactorizationFact] public void MatrixFactorizationInMemoryDataZeroBaseIndex() { // Create an in-memory matrix as a list of tuples (column index, row index, value). @@ -392,7 +393,7 @@ private class OneClassMatrixElementZeroBasedForScore public float Score; } - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // This test is being fixed as part of issue #1441. + [MatrixFactorizationFact] public void OneClassMatrixFactorizationInMemoryDataZeroBaseIndex() { // Create an in-memory matrix as a list of tuples (column index, row index, value). For one-class matrix @@ -464,7 +465,7 @@ public void OneClassMatrixFactorizationInMemoryDataZeroBaseIndex() CompareNumbersWithTolerance(0.141411, testResults[1].Score, digitsOfPrecision: 5); } - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // This test is being fixed as part of issue #1441. + [MatrixFactorizationFact] public void MatrixFactorizationBackCompat() { // This test is meant to check backwards compatibility after the change that removed Min and Contiguous from KeyType. diff --git a/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs b/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs index 9f3dfe20ee..9e995ceee0 100644 --- a/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs +++ b/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs @@ -10,6 +10,7 @@ using Microsoft.ML.Internal.Utilities; using Microsoft.ML.LightGBM; using Microsoft.ML.RunTests; +using Microsoft.ML.TestFramework.Attributes; using Microsoft.ML.Trainers.FastTree; using Microsoft.ML.Transforms.Conversions; using Xunit; @@ -42,7 +43,7 @@ public void FastTreeBinaryEstimator() Done(); } - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // LightGBM is 64-bit only + [LightGBMFact] public void LightGBMBinaryEstimator() { var (pipe, dataView) = GetBinaryClassificationPipeline(); @@ -128,7 +129,7 @@ public void FastTreeRankerEstimator() /// /// LightGbmRankingTrainer TrainerEstimator test /// - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // LightGBM is 64-bit only + [LightGBMFact] public void LightGBMRankerEstimator() { var (pipe, dataView) = GetRankingPipeline(); @@ -161,7 +162,7 @@ public void FastTreeRegressorEstimator() /// /// LightGbmRegressorTrainer TrainerEstimator test /// - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // LightGBM is 64-bit only + [LightGBMFact] public void LightGBMRegressorEstimator() { var dataView = GetRegressionPipeline(); @@ -237,7 +238,7 @@ public void FastForestRegressorEstimator() /// /// LightGbmMulticlass TrainerEstimator test /// - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // LightGBM is 64-bit only + [LightGBMFact] public void LightGbmMultiClassEstimator() { var (pipeline, dataView) = GetMultiClassPipeline(); @@ -369,7 +370,7 @@ private void LightGbmHelper(bool useSoftmax, out string modelString, out List