diff --git a/test/Microsoft.ML.Benchmarks.Tests/BenchmarksTest.cs b/test/Microsoft.ML.Benchmarks.Tests/BenchmarksTest.cs index 30dc7f3adb..88f975927a 100644 --- a/test/Microsoft.ML.Benchmarks.Tests/BenchmarksTest.cs +++ b/test/Microsoft.ML.Benchmarks.Tests/BenchmarksTest.cs @@ -52,13 +52,6 @@ where Attribute.IsDefined(type, typeof(CIBenchmark)) [MemberData(nameof(GetBenchmarks))] public void BenchmarksProjectIsNotBroken(Type type) { - // TODO: [TEST_STABILITY]: Benchmark test sometime hangs on windows of dotnet core 3.1 - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && - AppDomain.CurrentDomain.GetData("FX_PRODUCT_VERSION") != null) - { - return; - } - var summary = BenchmarkRunner.Run(type, new TestConfig().With(new OutputLogger(output))); Assert.False(summary.HasCriticalValidationErrors, "The \"Summary\" should have NOT \"HasCriticalValidationErrors\""); diff --git a/test/Microsoft.ML.Benchmarks/BenchmarkBase.cs b/test/Microsoft.ML.Benchmarks/BenchmarkBase.cs new file mode 100644 index 0000000000..2ebd1cc9c7 --- /dev/null +++ b/test/Microsoft.ML.Benchmarks/BenchmarkBase.cs @@ -0,0 +1,36 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.IO; +using Microsoft.ML.TestFrameworkCommon; + +namespace Microsoft.ML.Benchmarks +{ + public class BenchmarkBase + { + // Make sure DataDir is initialized before benchmark running. + static BenchmarkBase() + { + RootDir = TestCommon.GetRepoRoot(); + DataDir = Path.Combine(RootDir, "test", "data"); + } + + protected static string RootDir { get; } + protected static string DataDir { get; } + + // Don't use BaseTestClass's GetDataPath method instead for benchmark. + // BaseTestClass's static constructor is not guaranteed to be called before + // benchmark running (depending on CLR version this has different behaviour). + // The problem with executing BaseTestClass's static constructor when benchmark + // is running is it sometime cause process hanging when the constructor trying + // to load MKL, this is related to below issue: + // https://github.com/dotnet/machinelearning/issues/1073 + public static string GetBenchmarkDataPath(string name) + { + if (string.IsNullOrWhiteSpace(name)) + return null; + return Path.GetFullPath(Path.Combine(DataDir, name)); + } + } +} diff --git a/test/Microsoft.ML.Benchmarks/CacheDataViewBench.cs b/test/Microsoft.ML.Benchmarks/CacheDataViewBench.cs index 015bb8d43b..bc9a56cfeb 100644 --- a/test/Microsoft.ML.Benchmarks/CacheDataViewBench.cs +++ b/test/Microsoft.ML.Benchmarks/CacheDataViewBench.cs @@ -10,7 +10,7 @@ namespace Microsoft.ML.Benchmarks { [CIBenchmark] - public class CacheDataViewBench + public class CacheDataViewBench : BenchmarkBase { private const int Length = 100000; diff --git a/test/Microsoft.ML.Benchmarks/FeaturizeTextBench.cs b/test/Microsoft.ML.Benchmarks/FeaturizeTextBench.cs index 06d50906c3..39aa0d6feb 100644 --- a/test/Microsoft.ML.Benchmarks/FeaturizeTextBench.cs +++ b/test/Microsoft.ML.Benchmarks/FeaturizeTextBench.cs @@ -14,7 +14,7 @@ namespace Microsoft.ML.Benchmarks { [Config(typeof(TrainConfig))] - public class FeaturizeTextBench + public class FeaturizeTextBench : BenchmarkBase { private MLContext _mlContext; private IDataView _dataset; diff --git a/test/Microsoft.ML.Benchmarks/Harness/Metrics.cs b/test/Microsoft.ML.Benchmarks/Harness/Metrics.cs index 6efb237a80..5a2da65ec6 100644 --- a/test/Microsoft.ML.Benchmarks/Harness/Metrics.cs +++ b/test/Microsoft.ML.Benchmarks/Harness/Metrics.cs @@ -13,7 +13,7 @@ namespace Microsoft.ML.Benchmarks { - public abstract class WithExtraMetrics + public abstract class WithExtraMetrics : BenchmarkBase { protected abstract IEnumerable GetMetrics(); diff --git a/test/Microsoft.ML.Benchmarks/HashBench.cs b/test/Microsoft.ML.Benchmarks/HashBench.cs index 991cef06ee..36e339ebf1 100644 --- a/test/Microsoft.ML.Benchmarks/HashBench.cs +++ b/test/Microsoft.ML.Benchmarks/HashBench.cs @@ -13,7 +13,7 @@ namespace Microsoft.ML.Benchmarks { [CIBenchmark] - public class HashBench + public class HashBench : BenchmarkBase { private sealed class RowImpl : DataViewRow { diff --git a/test/Microsoft.ML.Benchmarks/ImageClassificationBench.cs b/test/Microsoft.ML.Benchmarks/ImageClassificationBench.cs index 1f6b57cbd9..fd7fe0ea44 100644 --- a/test/Microsoft.ML.Benchmarks/ImageClassificationBench.cs +++ b/test/Microsoft.ML.Benchmarks/ImageClassificationBench.cs @@ -18,7 +18,7 @@ namespace Microsoft.ML.Benchmarks { [Config(typeof(TrainConfig))] - public class ImageClassificationBench + public class ImageClassificationBench : BenchmarkBase { private MLContext _mlContext; private IDataView _trainDataset; diff --git a/test/Microsoft.ML.Benchmarks/KMeansAndLogisticRegressionBench.cs b/test/Microsoft.ML.Benchmarks/KMeansAndLogisticRegressionBench.cs index 371207cccd..f510724937 100644 --- a/test/Microsoft.ML.Benchmarks/KMeansAndLogisticRegressionBench.cs +++ b/test/Microsoft.ML.Benchmarks/KMeansAndLogisticRegressionBench.cs @@ -6,15 +6,14 @@ using Microsoft.ML.Benchmarks.Harness; using Microsoft.ML.Calibrators; using Microsoft.ML.Data; -using Microsoft.ML.TestFramework; using Microsoft.ML.Trainers; namespace Microsoft.ML.Benchmarks { [CIBenchmark] - public class KMeansAndLogisticRegressionBench + public class KMeansAndLogisticRegressionBench : BenchmarkBase { - private readonly string _dataPath = BaseTestClass.GetDataPath("adult.tiny.with-schema.txt"); + private readonly string _dataPath = GetBenchmarkDataPath("adult.tiny.with-schema.txt"); [Benchmark] public CalibratedModelParametersBase TrainKMeansAndLR() diff --git a/test/Microsoft.ML.Benchmarks/Microsoft.ML.Benchmarks.csproj b/test/Microsoft.ML.Benchmarks/Microsoft.ML.Benchmarks.csproj index 7a646050c8..af434d9820 100644 --- a/test/Microsoft.ML.Benchmarks/Microsoft.ML.Benchmarks.csproj +++ b/test/Microsoft.ML.Benchmarks/Microsoft.ML.Benchmarks.csproj @@ -22,6 +22,7 @@ + diff --git a/test/Microsoft.ML.Benchmarks/Numeric/Ranking.cs b/test/Microsoft.ML.Benchmarks/Numeric/Ranking.cs index 2a08d80a4b..a1c4672328 100644 --- a/test/Microsoft.ML.Benchmarks/Numeric/Ranking.cs +++ b/test/Microsoft.ML.Benchmarks/Numeric/Ranking.cs @@ -6,7 +6,6 @@ using BenchmarkDotNet.Attributes; using Microsoft.ML.Data; using Microsoft.ML.Trainers.LightGbm; -using Microsoft.ML.TestFramework; using Microsoft.ML.Trainers; using Microsoft.ML.Trainers.FastTree; using Microsoft.ML.Transforms; @@ -15,7 +14,7 @@ namespace Microsoft.ML.Benchmarks { [Config(typeof(TrainConfig))] - public class RankingTrain + public class RankingTrain : BenchmarkBase { private string _mslrWeb10kValidate; private string _mslrWeb10kTrain; @@ -23,8 +22,8 @@ public class RankingTrain [GlobalSetup] public void SetupTrainingSpeedTests() { - _mslrWeb10kValidate = BaseTestClass.GetDataPath(TestDatasets.MSLRWeb.validFilename); - _mslrWeb10kTrain = BaseTestClass.GetDataPath(TestDatasets.MSLRWeb.trainFilename); + _mslrWeb10kValidate = GetBenchmarkDataPath(TestDatasets.MSLRWeb.validFilename); + _mslrWeb10kTrain = GetBenchmarkDataPath(TestDatasets.MSLRWeb.trainFilename); if (!File.Exists(_mslrWeb10kValidate)) throw new FileNotFoundException(string.Format(Errors.DatasetNotFound, _mslrWeb10kValidate)); @@ -63,7 +62,7 @@ public void TrainTest_Ranking_MSLRWeb10K_RawNumericFeatures_LightGBMRanking() } } - public class RankingTest + public class RankingTest : BenchmarkBase { private string _mslrWeb10kValidate; private string _mslrWeb10kTrain; @@ -73,9 +72,9 @@ public class RankingTest [GlobalSetup] public void SetupScoringSpeedTests() { - _mslrWeb10kTest = BaseTestClass.GetDataPath(TestDatasets.MSLRWeb.testFilename); - _mslrWeb10kValidate = BaseTestClass.GetDataPath(TestDatasets.MSLRWeb.validFilename); - _mslrWeb10kTrain = BaseTestClass.GetDataPath(TestDatasets.MSLRWeb.trainFilename); + _mslrWeb10kTest = GetBenchmarkDataPath(TestDatasets.MSLRWeb.testFilename); + _mslrWeb10kValidate = GetBenchmarkDataPath(TestDatasets.MSLRWeb.validFilename); + _mslrWeb10kTrain = GetBenchmarkDataPath(TestDatasets.MSLRWeb.trainFilename); if (!File.Exists(_mslrWeb10kTest)) throw new FileNotFoundException(string.Format(Errors.DatasetNotFound, _mslrWeb10kTest)); diff --git a/test/Microsoft.ML.Benchmarks/PredictionEngineBench.cs b/test/Microsoft.ML.Benchmarks/PredictionEngineBench.cs index 743bd15f6a..3762ead4db 100644 --- a/test/Microsoft.ML.Benchmarks/PredictionEngineBench.cs +++ b/test/Microsoft.ML.Benchmarks/PredictionEngineBench.cs @@ -5,14 +5,13 @@ using BenchmarkDotNet.Attributes; using Microsoft.ML.Benchmarks.Harness; using Microsoft.ML.Data; -using Microsoft.ML.TestFramework; using Microsoft.ML.Trainers; using Microsoft.ML.Transforms; namespace Microsoft.ML.Benchmarks { [CIBenchmark] - public class PredictionEngineBench + public class PredictionEngineBench : BenchmarkBase { private IrisData _irisExample; private PredictionEngine _irisModel; @@ -34,7 +33,7 @@ public void SetupIrisPipeline() PetalWidth = 5.1f, }; - string irisDataPath = BaseTestClass.GetDataPath("iris.txt"); + string irisDataPath = GetBenchmarkDataPath("iris.txt"); var env = new MLContext(seed: 1); @@ -73,7 +72,7 @@ public void SetupSentimentPipeline() SentimentText = "Not a big fan of this." }; - string sentimentDataPath = BaseTestClass.GetDataPath("wikipedia-detox-250-line-data.tsv"); + string sentimentDataPath = GetBenchmarkDataPath("wikipedia-detox-250-line-data.tsv"); var mlContext = new MLContext(seed: 1); @@ -108,7 +107,7 @@ public void SetupBreastCancerPipeline() Features = new[] { 5f, 1f, 1f, 1f, 2f, 1f, 3f, 1f, 1f } }; - string breastCancerDataPath = BaseTestClass.GetDataPath("breast-cancer.txt"); + string breastCancerDataPath = GetBenchmarkDataPath("breast-cancer.txt"); var env = new MLContext(seed: 1); diff --git a/test/Microsoft.ML.Benchmarks/RffTransform.cs b/test/Microsoft.ML.Benchmarks/RffTransform.cs index 54a576411a..16b60c2a7c 100644 --- a/test/Microsoft.ML.Benchmarks/RffTransform.cs +++ b/test/Microsoft.ML.Benchmarks/RffTransform.cs @@ -6,22 +6,20 @@ using BenchmarkDotNet.Attributes; using Microsoft.ML.Benchmarks.Harness; using Microsoft.ML.Data; -using Microsoft.ML.RunTests; -using Microsoft.ML.TestFramework; using Microsoft.ML.TestFrameworkCommon; using Microsoft.ML.Transforms; namespace Microsoft.ML.Benchmarks { [CIBenchmark] - public class RffTransformTrain + public class RffTransformTrain : BenchmarkBase { private string _dataPathDigits; [GlobalSetup] public void SetupTrainingSpeedTests() { - _dataPathDigits = BaseTestClass.GetDataPath(TestDatasets.Digits.trainFilename); + _dataPathDigits = GetBenchmarkDataPath(TestDatasets.Digits.trainFilename); if (!File.Exists(_dataPathDigits)) throw new FileNotFoundException(string.Format(Errors.DatasetNotFound, _dataPathDigits)); diff --git a/test/Microsoft.ML.Benchmarks/StochasticDualCoordinateAscentClassifierBench.cs b/test/Microsoft.ML.Benchmarks/StochasticDualCoordinateAscentClassifierBench.cs index 64e2ca2e86..9a47f3dff5 100644 --- a/test/Microsoft.ML.Benchmarks/StochasticDualCoordinateAscentClassifierBench.cs +++ b/test/Microsoft.ML.Benchmarks/StochasticDualCoordinateAscentClassifierBench.cs @@ -8,7 +8,6 @@ using BenchmarkDotNet.Engines; using Microsoft.ML.Benchmarks.Harness; using Microsoft.ML.Data; -using Microsoft.ML.TestFramework; using Microsoft.ML.Trainers; using Microsoft.ML.Transforms; using Microsoft.ML.Transforms.Text; @@ -18,8 +17,8 @@ namespace Microsoft.ML.Benchmarks [CIBenchmark] public class StochasticDualCoordinateAscentClassifierBench : WithExtraMetrics { - private readonly string _dataPath = BaseTestClass.GetDataPath("iris.txt"); - private readonly string _sentimentDataPath = BaseTestClass.GetDataPath("wikipedia-detox-250-line-data.tsv"); + private readonly string _dataPath = GetBenchmarkDataPath("iris.txt"); + private readonly string _sentimentDataPath = GetBenchmarkDataPath("wikipedia-detox-250-line-data.tsv"); private readonly Consumer _consumer = new Consumer(); // BenchmarkDotNet utility type used to prevent dead code elimination private readonly MLContext _mlContext = new MLContext(seed: 1); diff --git a/test/Microsoft.ML.Benchmarks/Text/MultiClassClassification.cs b/test/Microsoft.ML.Benchmarks/Text/MultiClassClassification.cs index f37c76cf90..ba1c6ae510 100644 --- a/test/Microsoft.ML.Benchmarks/Text/MultiClassClassification.cs +++ b/test/Microsoft.ML.Benchmarks/Text/MultiClassClassification.cs @@ -6,8 +6,6 @@ using BenchmarkDotNet.Attributes; using Microsoft.ML.Data; using Microsoft.ML.Trainers.LightGbm; -using Microsoft.ML.RunTests; -using Microsoft.ML.TestFramework; using Microsoft.ML.Trainers; using Microsoft.ML.Transforms; using Microsoft.ML.TestFrameworkCommon; @@ -15,14 +13,14 @@ namespace Microsoft.ML.Benchmarks { [Config(typeof(TrainConfig))] - public class MulticlassClassificationTrain + public class MulticlassClassificationTrain : BenchmarkBase { private string _dataPathWiki; [GlobalSetup] public void SetupTrainingSpeedTests() { - _dataPathWiki = BaseTestClass.GetDataPath(TestDatasets.WikiDetox.trainFilename); + _dataPathWiki = GetBenchmarkDataPath(TestDatasets.WikiDetox.trainFilename); if (!File.Exists(_dataPathWiki)) throw new FileNotFoundException(string.Format(Errors.DatasetNotFound, _dataPathWiki)); @@ -91,7 +89,7 @@ public void CV_Multiclass_WikiDetox_WordEmbeddings_SDCAMC() } } - public class MulticlassClassificationTest + public class MulticlassClassificationTest : BenchmarkBase { private string _dataPathWiki; private string _modelPathWiki; @@ -99,7 +97,7 @@ public class MulticlassClassificationTest [GlobalSetup] public void SetupScoringSpeedTests() { - _dataPathWiki = BaseTestClass.GetDataPath(TestDatasets.WikiDetox.trainFilename); + _dataPathWiki = GetBenchmarkDataPath(TestDatasets.WikiDetox.trainFilename); if (!File.Exists(_dataPathWiki)) throw new FileNotFoundException(string.Format(Errors.DatasetNotFound, _dataPathWiki)); diff --git a/test/Microsoft.ML.Benchmarks/TextPredictionEngineCreation.cs b/test/Microsoft.ML.Benchmarks/TextPredictionEngineCreation.cs index 49014eeb1a..cb685dfb8d 100644 --- a/test/Microsoft.ML.Benchmarks/TextPredictionEngineCreation.cs +++ b/test/Microsoft.ML.Benchmarks/TextPredictionEngineCreation.cs @@ -6,13 +6,12 @@ using BenchmarkDotNet.Attributes; using Microsoft.ML; using Microsoft.ML.Benchmarks; -using Microsoft.ML.TestFramework; using Microsoft.ML.Trainers; namespace micro { [SimpleJob] - public class TextPredictionEngineCreationBenchmark + public class TextPredictionEngineCreationBenchmark : BenchmarkBase { private MLContext _context; private ITransformer _trainedModel; @@ -22,7 +21,8 @@ public class TextPredictionEngineCreationBenchmark public void Setup() { _context = new MLContext(1); - var data = _context.Data.LoadFromTextFile(BaseTestClass.GetDataPath("wikipedia-detox-250-line-data.tsv"), hasHeader: true); + var data = _context.Data.LoadFromTextFile( + GetBenchmarkDataPath("wikipedia-detox-250-line-data.tsv"), hasHeader: true); // Pipeline. var pipeline = _context.Transforms.Text.FeaturizeText("Features", "SentimentText")