Fix the benchmarks

Microsoft.ML.sln

@@ -121,6 +121,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.DnnAnalyzer",
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.ML.OnnxTransformTest", "test\Microsoft.ML.OnnxTransformTest\Microsoft.ML.OnnxTransformTest.csproj", "{49D03292-8AFE-4B82-823C-D047BF8420F7}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.Benchmarks.Tests", "test\Microsoft.ML.Benchmarks.Tests\Microsoft.ML.Benchmarks.Tests.csproj", "{B6C83F04-A04B-4F00-9E68-1EC411F9317C}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -449,6 +451,14 @@ Global
 		{49D03292-8AFE-4B82-823C-D047BF8420F7}.Release|Any CPU.Build.0 = Release|Any CPU
 		{49D03292-8AFE-4B82-823C-D047BF8420F7}.Release-Intrinsics|Any CPU.ActiveCfg = Release|Any CPU
 		{49D03292-8AFE-4B82-823C-D047BF8420F7}.Release-Intrinsics|Any CPU.Build.0 = Release|Any CPU
+		{B6C83F04-A04B-4F00-9E68-1EC411F9317C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{B6C83F04-A04B-4F00-9E68-1EC411F9317C}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{B6C83F04-A04B-4F00-9E68-1EC411F9317C}.Debug-Intrinsics|Any CPU.ActiveCfg = Debug|Any CPU
+		{B6C83F04-A04B-4F00-9E68-1EC411F9317C}.Debug-Intrinsics|Any CPU.Build.0 = Debug|Any CPU
+		{B6C83F04-A04B-4F00-9E68-1EC411F9317C}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{B6C83F04-A04B-4F00-9E68-1EC411F9317C}.Release|Any CPU.Build.0 = Release|Any CPU
+		{B6C83F04-A04B-4F00-9E68-1EC411F9317C}.Release-Intrinsics|Any CPU.ActiveCfg = Release|Any CPU
+		{B6C83F04-A04B-4F00-9E68-1EC411F9317C}.Release-Intrinsics|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -499,6 +509,7 @@ Global
 		{8C05642D-C3AA-4972-B02C-93681161A6BC} = {09EADF06-BE25-4228-AB53-95AE3E15B530}
 		{73DAAC82-D308-48CC-8FFE-3B037F8BBCCA} = {09EADF06-BE25-4228-AB53-95AE3E15B530}
 		{49D03292-8AFE-4B82-823C-D047BF8420F7} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
+		{B6C83F04-A04B-4F00-9E68-1EC411F9317C} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {41165AF1-35BB-4832-A189-73060F82B01D}

test/Microsoft.ML.Benchmarks.Tests/BenchmarksTest.cs

@@ -0,0 +1,97 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Configs;
+using BenchmarkDotNet.Jobs;
+using BenchmarkDotNet.Loggers;
+using BenchmarkDotNet.Running;
+using Microsoft.ML.Runtime.Internal.CpuMath;
+using System;
+using System.Linq;
+using Xunit;
+using Xunit.Abstractions;
+
+namespace Microsoft.ML.Benchmarks.Tests
+{
+    public class TestConfig : RecommendedConfig
+    {
+        protected override Job GetJobDefinition() => Job.Dry; // Job.Dry runs the benchmark just once
+    }
+
+    public class BenchmarkTouchingNativeDependency
+    {
+        [Benchmark]
+        public float Simple() => CpuMathUtils.Sum(Enumerable.Range(0, 1024).Select(Convert.ToSingle).ToArray(), 1024);
+    }
+
+    public class BenchmarksTest
+    {
+        private const string SkipTheDebug =
+#if DEBUG
+            "BenchmarkDotNet does not allow running the benchmarks in Debug, so this test is disabled for DEBUG";
+#else
+            "";
+#endif
+
+        public BenchmarksTest(ITestOutputHelper output) => Output = output;
+
+        private ITestOutputHelper Output { get; }
+
+        [Fact(Skip = SkipTheDebug)]
+        public void BenchmarksProjectIsNotBroken()
+        {
+            var summary = BenchmarkRunner.Run<BenchmarkTouchingNativeDependency>(new TestConfig().With(new OutputLogger(Output)));
+
+            Assert.False(summary.HasCriticalValidationErrors, "The \"Summary\" should have NOT \"HasCriticalValidationErrors\"");
+
+            Assert.True(summary.Reports.Any(), "The \"Summary\" should contain at least one \"BenchmarkReport\" in the \"Reports\" collection");
+
+            Assert.True(summary.Reports.All(r => r.BuildResult.IsBuildSuccess),
+                "The following benchmarks are failed to build: " +
+                string.Join(", ", summary.Reports.Where(r => !r.BuildResult.IsBuildSuccess).Select(r => r.BenchmarkCase.DisplayInfo)));
+
+            Assert.True(summary.Reports.All(r => r.ExecuteResults != null),
+                "The following benchmarks don't have any execution results: " +
+                string.Join(", ", summary.Reports.Where(r => r.ExecuteResults == null).Select(r => r.BenchmarkCase.DisplayInfo)));
+
+            Assert.True(summary.Reports.All(r => r.ExecuteResults.Any(er => er.FoundExecutable && er.Data.Any())),
+                "All reports should have at least one \"ExecuteResult\" with \"FoundExecutable\" = true and at least one \"Data\" item");
+
+            Assert.True(summary.Reports.All(report => report.AllMeasurements.Any()),
+                "All reports should have at least one \"Measurement\" in the \"AllMeasurements\" collection");
+        }
+    }
+
+    public class OutputLogger : AccumulationLogger
+    {
+        private readonly ITestOutputHelper testOutputHelper;
+        private string currentLine = "";
+
+        public OutputLogger(ITestOutputHelper testOutputHelper)
+        {
+            this.testOutputHelper = testOutputHelper ?? throw new ArgumentNullException(nameof(testOutputHelper));
+        }
+
+        public override void Write(LogKind logKind, string text)
+        {
+            currentLine += text;
+            base.Write(logKind, text);
+        }
+
+        public override void WriteLine()
+        {
+            testOutputHelper.WriteLine(currentLine);
+            currentLine = "";
+            base.WriteLine();
+        }
+
+        public override void WriteLine(LogKind logKind, string text)
+        {
+            testOutputHelper.WriteLine(currentLine + text);
+            currentLine = "";
+            base.WriteLine(logKind, text);
+        }
+    }
+}

test/Microsoft.ML.Benchmarks.Tests/Microsoft.ML.Benchmarks.Tests.csproj

@@ -0,0 +1,8 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <ItemGroup>
+    <ProjectReference Include="..\Microsoft.ML.Benchmarks\Microsoft.ML.Benchmarks.csproj" />
+
+    <NativeAssemblyReference Include="CpuMathNative" />
+  </ItemGroup>
+</Project>

test/Microsoft.ML.Benchmarks/Harness/Configs.cs

@@ -0,0 +1,51 @@
+using BenchmarkDotNet.Configs;
+using BenchmarkDotNet.Jobs;
+using BenchmarkDotNet.Toolchains;
+using BenchmarkDotNet.Toolchains.CsProj;
+using BenchmarkDotNet.Toolchains.DotNetCli;
+using Microsoft.ML.Benchmarks.Harness;
+
+namespace Microsoft.ML.Benchmarks
+{
+    public class RecommendedConfig : ManualConfig
+    {
+        public RecommendedConfig()
+        {
+            Add(DefaultConfig.Instance); // this config contains all of the basic settings (exporters, columns etc)
+
+            Add(GetJobDefinition() // job defines how many times given benchmark should be executed
+                .With(CreateToolchain())); // toolchain is responsible for generating, building and running dedicated executable per benchmark
+
+            Add(new ExtraMetricColumn()); // an extra colum that can display additional metric reported by the benchmarks
+
+            UnionRule = ConfigUnionRule.AlwaysUseLocal; // global config can be overwritten with local (the one set via [ConfigAttribute])
+        }
+
+        protected virtual Job GetJobDefinition()
+            => Job.Default
+                .WithWarmupCount(1) // ML.NET benchmarks are typically CPU-heavy benchmarks, 1 warmup is usually enough
+                .WithMaxIterationCount(20);
+
+        /// <summary>
+        /// we need our own toolchain because MSBuild by default does not copy recursive native dependencies to the output
+        /// </summary>
+        private IToolchain CreateToolchain()
+        {
+            var csProj = CsProjCoreToolchain.Current.Value;
+            var tfm = NetCoreAppSettings.Current.Value.TargetFrameworkMoniker;
+
+            return new Toolchain(
+                tfm,
+                new ProjectGenerator(tfm), // custom generator that copies native dependencies
+                csProj.Builder,
+                csProj.Executor);
+        }
+    }
+
+    public class TrainConfig : RecommendedConfig
+    {
+        protected override Job GetJobDefinition()
+            => Job.Dry // the "Dry" job runs the benchmark exactly once, without any warmup to mimic real-world scenario
+                  .WithLaunchCount(3); // BDN will run 3 dedicated processes, sequentially
+    }
+}

test/Microsoft.ML.Benchmarks/Harness/ProjectGenerator.cs

@@ -3,11 +3,14 @@
 // See the LICENSE file in the project root for more information.
 
 using BenchmarkDotNet.Extensions;
+using BenchmarkDotNet.Loggers;
+using BenchmarkDotNet.Running;
 using BenchmarkDotNet.Toolchains;
 using BenchmarkDotNet.Toolchains.CsProj;
 using System;
 using System.IO;
 using System.Linq;
+using System.Text;
 
 namespace Microsoft.ML.Benchmarks.Harness
 {
@@ -18,37 +21,45 @@ namespace Microsoft.ML.Benchmarks.Harness
     /// the problem with ML.NET is that it has native dependencies, which are NOT copied by MSBuild to the output folder
     /// in case where A has native dependency and B references A
     /// 
-    /// this is why this class exists: to copy the native dependencies to folder with .exe
+    /// this is why this class exists: 
+    ///  1. to tell MSBuild to copy the native dependencies to folder with .exe (NativeAssemblyReference)
+    ///  2. to generate a .csproj file that does not exclude Directory.Build.props (default BDN behaviour) which contains custom NuGet feeds that are required for restore step
     /// </summary>
     public class ProjectGenerator : CsProjGenerator
     {
         public ProjectGenerator(string targetFrameworkMoniker) : base(targetFrameworkMoniker, platform => platform.ToConfig(), null)
         {
         }
 
-        protected override void CopyAllRequiredFiles(ArtifactsPaths artifactsPaths)
-        {
-            base.CopyAllRequiredFiles(artifactsPaths);
-
-            CopyMissingNativeDependencies(artifactsPaths);
-        }
+        protected override void GenerateProject(BuildPartition buildPartition, ArtifactsPaths artifactsPaths, ILogger logger)
+            => File.WriteAllText(artifactsPaths.ProjectFilePath, $@"
+<Project Sdk=""Microsoft.NET.Sdk"">
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <OutputPath>bin\{buildPartition.BuildConfiguration}</OutputPath>
+    <TargetFramework>{TargetFrameworkMoniker}</TargetFramework>
+    <AssemblyName>{artifactsPaths.ProgramName}</AssemblyName>
+    <AssemblyTitle>{artifactsPaths.ProgramName}</AssemblyTitle>
+    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+    <TreatWarningsAsErrors>False</TreatWarningsAsErrors>
+    <DebugType>pdbonly</DebugType>
+    <DebugSymbols>true</DebugSymbols>
+  </PropertyGroup>
+  {GetRuntimeSettings(buildPartition.RepresentativeBenchmarkCase.Job.Environment.Gc, buildPartition.Resolver)}
+  <ItemGroup>
+    <Compile Include=""{Path.GetFileName(artifactsPaths.ProgramCodePath)}"" Exclude=""bin\**;obj\**;**\*.xproj;packages\**"" />
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include=""{GetProjectFilePath(buildPartition.RepresentativeBenchmarkCase.Descriptor.Type, logger).FullName}"" />
+    {GenerateNativeReferences(buildPartition, logger)}
+  </ItemGroup>
+</Project>");
 
-        private void CopyMissingNativeDependencies(ArtifactsPaths artifactsPaths)
+        private string GenerateNativeReferences(BuildPartition buildPartition, ILogger logger)
         {
-            var foldeWithAutogeneratedExe = Path.GetDirectoryName(artifactsPaths.ExecutablePath);
-            var folderWithNativeDependencies = Path.GetDirectoryName(typeof(ProjectGenerator).Assembly.Location);
+            var csproj = GetProjectFilePath(buildPartition.RepresentativeBenchmarkCase.Descriptor.Type, logger);
 
-            foreach (var nativeDependency in Directory
-                .EnumerateFiles(folderWithNativeDependencies)
-                .Where(fileName => ContainsWithIgnoreCase(fileName, "native")))
-            {
-                File.Copy(
-                    sourceFileName: nativeDependency,
-                    destFileName: Path.Combine(foldeWithAutogeneratedExe, Path.GetFileName(nativeDependency)), 
-                    overwrite: true);
-            }
+            return string.Join(Environment.NewLine, File.ReadAllLines(csproj.FullName).Where(line => line.Contains("<NativeAssemblyReference")));
         }
-
-        bool ContainsWithIgnoreCase(string text, string word) => text != null && text.IndexOf(word, StringComparison.InvariantCultureIgnoreCase) >= 0;
     }
 }

test/Microsoft.ML.Benchmarks/Helpers.cs → test/Microsoft.ML.Benchmarks/Helpers/EmptyWriter.cs

@@ -7,16 +7,12 @@
 
 namespace Microsoft.ML.Benchmarks
 {
-    internal class Helpers
-    {
-        public static string DatasetNotFound = "Could not find {0} Please ensure you have run 'build.cmd -- /t:DownloadExternalTestFiles /p:IncludeBenchmarkData=true' from the root";
-    }
-
     // Adding this class to not print anything to the console.
     // This is required for the current version of BenchmarkDotNet
     internal class EmptyWriter : TextWriter
     {
         internal static readonly EmptyWriter Instance = new EmptyWriter();
+
         public override Encoding Encoding => null;
     }
 }

test/Microsoft.ML.Benchmarks/Helpers/EnvironmentFactory.cs

@@ -0,0 +1,45 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using Microsoft.ML.Core.Data;
+using Microsoft.ML.Runtime;
+using Microsoft.ML.Runtime.Data;
+
+namespace Microsoft.ML.Benchmarks
+{
+    internal static class EnvironmentFactory
+    {
+        internal static ConsoleEnvironment CreateClassificationEnvironment<TLoader, TTransformer, TTrainer>()
+           where TLoader : IDataReader<IMultiStreamSource>
+           where TTransformer : ITransformer
+           where TTrainer : ITrainer
+        {
+            var environment = new ConsoleEnvironment(verbose: false, sensitivity: MessageSensitivity.None, outWriter: EmptyWriter.Instance);
+
+            environment.ComponentCatalog.RegisterAssembly(typeof(TLoader).Assembly);
+            environment.ComponentCatalog.RegisterAssembly(typeof(TTransformer).Assembly);
+            environment.ComponentCatalog.RegisterAssembly(typeof(TTrainer).Assembly);
+
+            return environment;
+        }
+
+        internal static ConsoleEnvironment CreateRankingEnvironment<TEvaluator, TLoader, TTransformer, TTrainer>()
+            where TEvaluator : IEvaluator
+            where TLoader : IDataReader<IMultiStreamSource>
+            where TTransformer : ITransformer
+            where TTrainer : ITrainer
+        {
+            var environment = new ConsoleEnvironment(verbose: false, sensitivity: MessageSensitivity.None, outWriter: EmptyWriter.Instance);
+
+            environment.ComponentCatalog.RegisterAssembly(typeof(TEvaluator).Assembly);
+            environment.ComponentCatalog.RegisterAssembly(typeof(TLoader).Assembly);
+            environment.ComponentCatalog.RegisterAssembly(typeof(TTransformer).Assembly);
+            environment.ComponentCatalog.RegisterAssembly(typeof(TTrainer).Assembly);
+
+            environment.ComponentCatalog.RegisterAssembly(typeof(NAHandleTransform).Assembly);
+
+            return environment;
+        }
+    }
+}

test/Microsoft.ML.Benchmarks/Helpers/Errors.cs

@@ -0,0 +1,11 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+namespace Microsoft.ML.Benchmarks
+{
+    internal class Errors
+    {
+        public static string DatasetNotFound = "Could not find {0} Please ensure you have run 'build.cmd -- /t:DownloadExternalTestFiles /p:IncludeBenchmarkData=true' from the root";
+    }
+}