fix benchmark test hanging issue (#4985)

Author: frank-dong-ms-zz

test/Microsoft.ML.Benchmarks.Tests/BenchmarksTest.cs

@@ -52,13 +52,6 @@ where Attribute.IsDefined(type, typeof(CIBenchmark))
         [MemberData(nameof(GetBenchmarks))]
         public void BenchmarksProjectIsNotBroken(Type type)
         {
-            // TODO: [TEST_STABILITY]: Benchmark test sometime hangs on windows of dotnet core 3.1
-            if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) &&
-                AppDomain.CurrentDomain.GetData("FX_PRODUCT_VERSION") != null)
-            {
-                return;
-            }
-
             var summary = BenchmarkRunner.Run(type, new TestConfig().With(new OutputLogger(output)));
 
             Assert.False(summary.HasCriticalValidationErrors, "The \"Summary\" should have NOT \"HasCriticalValidationErrors\"");

test/Microsoft.ML.Benchmarks/BenchmarkBase.cs

@@ -0,0 +1,36 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.IO;
+using Microsoft.ML.TestFrameworkCommon;
+
+namespace Microsoft.ML.Benchmarks
+{
+    public class BenchmarkBase
+    {
+        // Make sure DataDir is initialized before benchmark running.
+        static BenchmarkBase()
+        {
+            RootDir = TestCommon.GetRepoRoot();
+            DataDir = Path.Combine(RootDir, "test", "data");
+        }
+
+        protected static string RootDir { get; }
+        protected static string DataDir { get; }
+
+        // Don't use BaseTestClass's GetDataPath method instead for benchmark.
+        // BaseTestClass's static constructor is not guaranteed to be called before
+        // benchmark running (depending on CLR version this has different behaviour).
+        // The problem with executing BaseTestClass's static constructor when benchmark
+        // is running is it sometime cause process hanging when the constructor trying 
+        // to load MKL, this is related to below issue:
+        // https://github.com/dotnet/machinelearning/issues/1073
+        public static string GetBenchmarkDataPath(string name)
+        {
+            if (string.IsNullOrWhiteSpace(name))
+                return null;
+            return Path.GetFullPath(Path.Combine(DataDir, name));
+        }
+    }
+}

test/Microsoft.ML.Benchmarks/CacheDataViewBench.cs

@@ -10,7 +10,7 @@
 namespace Microsoft.ML.Benchmarks
 {
     [CIBenchmark]
-    public class CacheDataViewBench
+    public class CacheDataViewBench : BenchmarkBase
     {
         private const int Length = 100000;
 

test/Microsoft.ML.Benchmarks/FeaturizeTextBench.cs

@@ -14,7 +14,7 @@
 namespace Microsoft.ML.Benchmarks
 {
     [Config(typeof(TrainConfig))]
-    public class FeaturizeTextBench
+    public class FeaturizeTextBench : BenchmarkBase
     {
         private MLContext _mlContext;
         private IDataView _dataset;

test/Microsoft.ML.Benchmarks/Harness/Metrics.cs

@@ -13,7 +13,7 @@
 
 namespace Microsoft.ML.Benchmarks
 {
-    public abstract class WithExtraMetrics
+    public abstract class WithExtraMetrics : BenchmarkBase
     {
         protected abstract IEnumerable<Metric> GetMetrics();
 

test/Microsoft.ML.Benchmarks/HashBench.cs

@@ -13,7 +13,7 @@
 namespace Microsoft.ML.Benchmarks
 {
     [CIBenchmark]
-    public class HashBench
+    public class HashBench : BenchmarkBase
     {
         private sealed class RowImpl : DataViewRow
         {

test/Microsoft.ML.Benchmarks/ImageClassificationBench.cs

@@ -18,7 +18,7 @@
 namespace Microsoft.ML.Benchmarks
 {
     [Config(typeof(TrainConfig))]
-    public class ImageClassificationBench
+    public class ImageClassificationBench : BenchmarkBase
     {
         private MLContext _mlContext;
         private IDataView _trainDataset;

test/Microsoft.ML.Benchmarks/KMeansAndLogisticRegressionBench.cs

@@ -6,15 +6,14 @@
 using Microsoft.ML.Benchmarks.Harness;
 using Microsoft.ML.Calibrators;
 using Microsoft.ML.Data;
-using Microsoft.ML.TestFramework;
 using Microsoft.ML.Trainers;
 
 namespace Microsoft.ML.Benchmarks
 {
     [CIBenchmark]
-    public class KMeansAndLogisticRegressionBench
+    public class KMeansAndLogisticRegressionBench : BenchmarkBase
     {
-        private readonly string _dataPath = BaseTestClass.GetDataPath("adult.tiny.with-schema.txt");
+        private readonly string _dataPath = GetBenchmarkDataPath("adult.tiny.with-schema.txt");
 
         [Benchmark]
         public CalibratedModelParametersBase<LinearBinaryModelParameters, PlattCalibrator> TrainKMeansAndLR()

test/Microsoft.ML.Benchmarks/Microsoft.ML.Benchmarks.csproj

@@ -22,6 +22,7 @@
     <ProjectReference Include="..\..\src\Microsoft.ML.ImageAnalytics\Microsoft.ML.ImageAnalytics.csproj" />
     <ProjectReference Include="..\..\src\Microsoft.ML.StandardTrainers\Microsoft.ML.StandardTrainers.csproj" />
     <ProjectReference Include="..\..\test\Microsoft.ML.TestFramework\Microsoft.ML.TestFramework.csproj" />
+    <ProjectReference Include="..\..\test\Microsoft.ML.TestFrameworkCommon\Microsoft.ML.TestFrameworkCommon.csproj" />
   </ItemGroup>
   <ItemGroup>
     <NativeAssemblyReference Include="CpuMathNative" />

test/Microsoft.ML.Benchmarks/Numeric/Ranking.cs

@@ -6,7 +6,6 @@
 using BenchmarkDotNet.Attributes;
 using Microsoft.ML.Data;
 using Microsoft.ML.Trainers.LightGbm;
-using Microsoft.ML.TestFramework;
 using Microsoft.ML.Trainers;
 using Microsoft.ML.Trainers.FastTree;
 using Microsoft.ML.Transforms;
@@ -15,16 +14,16 @@
 namespace Microsoft.ML.Benchmarks
 {
     [Config(typeof(TrainConfig))]
-    public class RankingTrain
+    public class RankingTrain : BenchmarkBase
     {
         private string _mslrWeb10kValidate;
         private string _mslrWeb10kTrain;
 
         [GlobalSetup]
         public void SetupTrainingSpeedTests()
         {
-            _mslrWeb10kValidate = BaseTestClass.GetDataPath(TestDatasets.MSLRWeb.validFilename);
-            _mslrWeb10kTrain = BaseTestClass.GetDataPath(TestDatasets.MSLRWeb.trainFilename);
+            _mslrWeb10kValidate = GetBenchmarkDataPath(TestDatasets.MSLRWeb.validFilename);
+            _mslrWeb10kTrain = GetBenchmarkDataPath(TestDatasets.MSLRWeb.trainFilename);
 
             if (!File.Exists(_mslrWeb10kValidate))
                 throw new FileNotFoundException(string.Format(Errors.DatasetNotFound, _mslrWeb10kValidate));
@@ -63,7 +62,7 @@ public void TrainTest_Ranking_MSLRWeb10K_RawNumericFeatures_LightGBMRanking()
         }
     }
 
-    public class RankingTest
+    public class RankingTest : BenchmarkBase
     {
         private string _mslrWeb10kValidate;
         private string _mslrWeb10kTrain;
@@ -73,9 +72,9 @@ public class RankingTest
         [GlobalSetup]
         public void SetupScoringSpeedTests()
         {
-            _mslrWeb10kTest = BaseTestClass.GetDataPath(TestDatasets.MSLRWeb.testFilename);
-            _mslrWeb10kValidate = BaseTestClass.GetDataPath(TestDatasets.MSLRWeb.validFilename);
-            _mslrWeb10kTrain = BaseTestClass.GetDataPath(TestDatasets.MSLRWeb.trainFilename);
+            _mslrWeb10kTest = GetBenchmarkDataPath(TestDatasets.MSLRWeb.testFilename);
+            _mslrWeb10kValidate = GetBenchmarkDataPath(TestDatasets.MSLRWeb.validFilename);
+            _mslrWeb10kTrain = GetBenchmarkDataPath(TestDatasets.MSLRWeb.trainFilename);
 
             if (!File.Exists(_mslrWeb10kTest))
                 throw new FileNotFoundException(string.Format(Errors.DatasetNotFound, _mslrWeb10kTest));

test/Microsoft.ML.Benchmarks/PredictionEngineBench.cs

@@ -5,14 +5,13 @@
 using BenchmarkDotNet.Attributes;
 using Microsoft.ML.Benchmarks.Harness;
 using Microsoft.ML.Data;
-using Microsoft.ML.TestFramework;
 using Microsoft.ML.Trainers;
 using Microsoft.ML.Transforms;
 
 namespace Microsoft.ML.Benchmarks
 {
     [CIBenchmark]
-    public class PredictionEngineBench
+    public class PredictionEngineBench : BenchmarkBase
     {
         private IrisData _irisExample;
         private PredictionEngine<IrisData, IrisPrediction> _irisModel;
@@ -34,7 +33,7 @@ public void SetupIrisPipeline()
                 PetalWidth = 5.1f,
             };
 
-            string irisDataPath = BaseTestClass.GetDataPath("iris.txt");
+            string irisDataPath = GetBenchmarkDataPath("iris.txt");
 
             var env = new MLContext(seed: 1);
 
@@ -73,7 +72,7 @@ public void SetupSentimentPipeline()
                 SentimentText = "Not a big fan of this."
             };
 
-            string sentimentDataPath = BaseTestClass.GetDataPath("wikipedia-detox-250-line-data.tsv");
+            string sentimentDataPath = GetBenchmarkDataPath("wikipedia-detox-250-line-data.tsv");
 
             var mlContext = new MLContext(seed: 1);
 
@@ -108,7 +107,7 @@ public void SetupBreastCancerPipeline()
                 Features = new[] { 5f, 1f, 1f, 1f, 2f, 1f, 3f, 1f, 1f }
             };
 
-            string breastCancerDataPath = BaseTestClass.GetDataPath("breast-cancer.txt");
+            string breastCancerDataPath = GetBenchmarkDataPath("breast-cancer.txt");
 
             var env = new MLContext(seed: 1);
 

test/Microsoft.ML.Benchmarks/RffTransform.cs

@@ -6,22 +6,20 @@
 using BenchmarkDotNet.Attributes;
 using Microsoft.ML.Benchmarks.Harness;
 using Microsoft.ML.Data;
-using Microsoft.ML.RunTests;
-using Microsoft.ML.TestFramework;
 using Microsoft.ML.TestFrameworkCommon;
 using Microsoft.ML.Transforms;
 
 namespace Microsoft.ML.Benchmarks
 {
     [CIBenchmark]
-    public class RffTransformTrain
+    public class RffTransformTrain : BenchmarkBase
     {
         private string _dataPathDigits;
 
         [GlobalSetup]
         public void SetupTrainingSpeedTests()
         {
-            _dataPathDigits = BaseTestClass.GetDataPath(TestDatasets.Digits.trainFilename);
+            _dataPathDigits = GetBenchmarkDataPath(TestDatasets.Digits.trainFilename);
 
             if (!File.Exists(_dataPathDigits))
                 throw new FileNotFoundException(string.Format(Errors.DatasetNotFound, _dataPathDigits));

test/Microsoft.ML.Benchmarks/StochasticDualCoordinateAscentClassifierBench.cs

@@ -8,7 +8,6 @@
 using BenchmarkDotNet.Engines;
 using Microsoft.ML.Benchmarks.Harness;
 using Microsoft.ML.Data;
-using Microsoft.ML.TestFramework;
 using Microsoft.ML.Trainers;
 using Microsoft.ML.Transforms;
 using Microsoft.ML.Transforms.Text;
@@ -18,8 +17,8 @@ namespace Microsoft.ML.Benchmarks
     [CIBenchmark]
     public class StochasticDualCoordinateAscentClassifierBench : WithExtraMetrics
     {
-        private readonly string _dataPath = BaseTestClass.GetDataPath("iris.txt");
-        private readonly string _sentimentDataPath = BaseTestClass.GetDataPath("wikipedia-detox-250-line-data.tsv");
+        private readonly string _dataPath = GetBenchmarkDataPath("iris.txt");
+        private readonly string _sentimentDataPath = GetBenchmarkDataPath("wikipedia-detox-250-line-data.tsv");
         private readonly Consumer _consumer = new Consumer(); // BenchmarkDotNet utility type used to prevent dead code elimination
 
         private readonly MLContext _mlContext = new MLContext(seed: 1);

test/Microsoft.ML.Benchmarks/Text/MultiClassClassification.cs

@@ -6,23 +6,21 @@
 using BenchmarkDotNet.Attributes;
 using Microsoft.ML.Data;
 using Microsoft.ML.Trainers.LightGbm;
-using Microsoft.ML.RunTests;
-using Microsoft.ML.TestFramework;
 using Microsoft.ML.Trainers;
 using Microsoft.ML.Transforms;
 using Microsoft.ML.TestFrameworkCommon;
 
 namespace Microsoft.ML.Benchmarks
 {
     [Config(typeof(TrainConfig))]
-    public class MulticlassClassificationTrain
+    public class MulticlassClassificationTrain : BenchmarkBase
     {
         private string _dataPathWiki;
 
         [GlobalSetup]
         public void SetupTrainingSpeedTests()
         {
-            _dataPathWiki = BaseTestClass.GetDataPath(TestDatasets.WikiDetox.trainFilename);
+            _dataPathWiki = GetBenchmarkDataPath(TestDatasets.WikiDetox.trainFilename);
 
             if (!File.Exists(_dataPathWiki))
                 throw new FileNotFoundException(string.Format(Errors.DatasetNotFound, _dataPathWiki));
@@ -91,15 +89,15 @@ public void CV_Multiclass_WikiDetox_WordEmbeddings_SDCAMC()
         }
     }
 
-    public class MulticlassClassificationTest
+    public class MulticlassClassificationTest : BenchmarkBase
     {
         private string _dataPathWiki;
         private string _modelPathWiki;
 
         [GlobalSetup]
         public void SetupScoringSpeedTests()
         {
-            _dataPathWiki = BaseTestClass.GetDataPath(TestDatasets.WikiDetox.trainFilename);
+            _dataPathWiki = GetBenchmarkDataPath(TestDatasets.WikiDetox.trainFilename);
 
             if (!File.Exists(_dataPathWiki))
                 throw new FileNotFoundException(string.Format(Errors.DatasetNotFound, _dataPathWiki));

test/Microsoft.ML.Benchmarks/TextPredictionEngineCreation.cs

@@ -6,13 +6,12 @@
 using BenchmarkDotNet.Attributes;
 using Microsoft.ML;
 using Microsoft.ML.Benchmarks;
-using Microsoft.ML.TestFramework;
 using Microsoft.ML.Trainers;
 
 namespace micro
 {
     [SimpleJob]
-    public class TextPredictionEngineCreationBenchmark
+    public class TextPredictionEngineCreationBenchmark : BenchmarkBase
     {
         private MLContext _context;
         private ITransformer _trainedModel;
@@ -22,7 +21,8 @@ public class TextPredictionEngineCreationBenchmark
         public void Setup()
         {
             _context = new MLContext(1);
-            var data = _context.Data.LoadFromTextFile<SentimentData>(BaseTestClass.GetDataPath("wikipedia-detox-250-line-data.tsv"), hasHeader: true);
+            var data = _context.Data.LoadFromTextFile<SentimentData>(
+                GetBenchmarkDataPath("wikipedia-detox-250-line-data.tsv"), hasHeader: true);
 
             // Pipeline.
             var pipeline = _context.Transforms.Text.FeaturizeText("Features", "SentimentText")