Fix the benchmarks

nuget.config

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="utf-8"?>
+<configuration>
+ <packageSources>
+    <!--To inherit the global NuGet package sources remove the <clear/> line below -->
+    <clear />
+    <add key="nuget" value="https://api.nuget.org/v3/index.json" />
+    <add key="net core" value=" https://dotnet.myget.org/F/dotnet-core/api/v3/index.json" />
+ </packageSources>
+</configuration>

test/Microsoft.ML.Benchmarks/Harness/Configs.cs

@@ -0,0 +1,50 @@
+using BenchmarkDotNet.Configs;
+using BenchmarkDotNet.Diagnosers;
+using BenchmarkDotNet.Jobs;
+using BenchmarkDotNet.Toolchains;
+using BenchmarkDotNet.Toolchains.CsProj;
+using BenchmarkDotNet.Toolchains.DotNetCli;
+using Microsoft.ML.Benchmarks.Harness;
+
+namespace Microsoft.ML.Benchmarks
+{
+    public class RecommendedConfig : ManualConfig
+    {
+        public RecommendedConfig()
+        {
+            Add(DefaultConfig.Instance
+                .With(GetJobDefinition().With(CreateToolchain()))
+                .With(new ExtraMetricColumn())
+                .With(MemoryDiagnoser.Default));
+
+            UnionRule = ConfigUnionRule.AlwaysUseLocal;
+        }
+
+        protected virtual Job GetJobDefinition()
+            => Job.Default
+                .WithWarmupCount(1) // ML.NET benchmarks are typically CPU-heavy benchmarks, 1 warmup is usually enough
+                .WithMaxIterationCount(20);
+
+        /// <summary>
+        /// we need our own toolchain because MSBuild by default does not copy recursive native dependencies to the output
+        /// </summary>
+        private IToolchain CreateToolchain()
+        {
+            var csProj = CsProjCoreToolchain.Current.Value;
+            var tfm = NetCoreAppSettings.Current.Value.TargetFrameworkMoniker;
+
+            return new Toolchain(
+                tfm,
+                new ProjectGenerator(tfm),
+                csProj.Builder,
+                csProj.Executor);
+        }
+    }
+
+    public class TrainConfig : RecommendedConfig
+    {
+        protected override Job GetJobDefinition()
+            => Job.Dry // the "Dry" job runs the benchmark exactly once, without any warmup to mimic real-world scenario
+                  .WithLaunchCount(3); // BDN will run 3 dedicated processes, sequentially
+    }
+}

test/Microsoft.ML.Benchmarks/Numeric/Ranking.cs

@@ -64,7 +64,6 @@ public void TrainTest_Ranking_MSLRWeb10K_RawNumericFeatures_LightGBMRanking()
         }
     }
 
-    [Config(typeof(PredictConfig))]
     public class RankingTest
     {
         private string _mslrWeb10k_Validate;

test/Microsoft.ML.Benchmarks/Program.cs

@@ -3,10 +3,6 @@
 // See the LICENSE file in the project root for more information.
 
 using BenchmarkDotNet.Running;
-using BenchmarkDotNet.Toolchains;
-using BenchmarkDotNet.Toolchains.CsProj;
-using BenchmarkDotNet.Toolchains.DotNetCli;
-using Microsoft.ML.Benchmarks.Harness;
 using System.Globalization;
 using System.IO;
 using System.Threading;
@@ -22,22 +18,7 @@ class Program
         static void Main(string[] args) 
             => BenchmarkSwitcher
                 .FromAssembly(typeof(Program).Assembly)
-                .Run(args);
-
-        /// <summary>
-        /// we need our own toolchain because MSBuild by default does not copy recursive native dependencies to the output
-        /// </summary>
-        internal static IToolchain CreateToolchain()
-        {
-            var csProj = CsProjCoreToolchain.Current.Value;
-            var tfm = NetCoreAppSettings.Current.Value.TargetFrameworkMoniker;
-
-            return new Toolchain(
-                tfm, 
-                new ProjectGenerator(tfm), 
-                csProj.Builder, 
-                csProj.Executor);
-        }
+                .Run(args, new RecommendedConfig());
 
         internal static string GetInvariantCultureDataPath(string name)
         {

test/Microsoft.ML.Benchmarks/README.md

@@ -7,6 +7,10 @@ This project contains performance benchmarks.
 **Pre-requisite:** On a clean repo, `build.cmd` at the root installs the right version of dotnet.exe and builds the solution. You need to build the solution in `Release` with native dependencies. 
 
     build.cmd -release -buildNative
+
+Moreover, to run some of the benchmarks you have to download external dependencies.
+
+    build.cmd -- /t:DownloadExternalTestFiles /p:IncludeBenchmarkData=true
 
 1. Navigate to the benchmarks directory (machinelearning\test\Microsoft.ML.Benchmarks)
 

test/Microsoft.ML.Benchmarks/Text/MultiClassClassification.cs

@@ -96,7 +96,6 @@ public void CV_Multiclass_WikiDetox_WordEmbeddings_SDCAMC()
         }
     }
 
-    [Config(typeof(PredictConfig))]
     public class MultiClassClassificationTest
     {
         private string _dataPath_Wiki;