Skip to content

Commit 17ee205

Browse files
adamsitnikadamsitnik
authored
Fix the benchmarks (#1001)
* move Harness-related code to Harness folder * make sure that we always use recommended config * mention the external dependencies in the README * add nuget.config file so BDN can restore all packages * add comments to the config so I am not the only person who understands it * don't enable MemoryDiagnoser by default, it requires one extra iteration which is expensive for long running benchmarks * don't add nuget.config file, generate it on the fly when needed by BDN * generate a .csproj file that will handle both native dependencies and nuget.config file issue * describe authoring new benchmarks in the docs * add some integration tests that make sure that the benchmarks are not broken * register the right assemblies after recent change of assembly loading, makes all benchmark work again ;) * make Ranking benchmarks work * code review: split Helpers.cs into multiple files, cleanup the code, don't hardcode the dependencies
1 parent 8592d96 commit 17ee205

15 files changed

+317
-118
lines changed

‎Microsoft.ML.sln

+11
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.DnnAnalyzer",
121121
EndProject
122122
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.ML.OnnxTransformTest", "test\Microsoft.ML.OnnxTransformTest\Microsoft.ML.OnnxTransformTest.csproj", "{49D03292-8AFE-4B82-823C-D047BF8420F7}"
123123
EndProject
124+
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.Benchmarks.Tests", "test\Microsoft.ML.Benchmarks.Tests\Microsoft.ML.Benchmarks.Tests.csproj", "{B6C83F04-A04B-4F00-9E68-1EC411F9317C}"
125+
EndProject
124126
Global
125127
GlobalSection(SolutionConfigurationPlatforms) = preSolution
126128
Debug|Any CPU = Debug|Any CPU
@@ -449,6 +451,14 @@ Global
449451
{49D03292-8AFE-4B82-823C-D047BF8420F7}.Release|Any CPU.Build.0 = Release|Any CPU
450452
{49D03292-8AFE-4B82-823C-D047BF8420F7}.Release-Intrinsics|Any CPU.ActiveCfg = Release|Any CPU
451453
{49D03292-8AFE-4B82-823C-D047BF8420F7}.Release-Intrinsics|Any CPU.Build.0 = Release|Any CPU
454+
{B6C83F04-A04B-4F00-9E68-1EC411F9317C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
455+
{B6C83F04-A04B-4F00-9E68-1EC411F9317C}.Debug|Any CPU.Build.0 = Debug|Any CPU
456+
{B6C83F04-A04B-4F00-9E68-1EC411F9317C}.Debug-Intrinsics|Any CPU.ActiveCfg = Debug|Any CPU
457+
{B6C83F04-A04B-4F00-9E68-1EC411F9317C}.Debug-Intrinsics|Any CPU.Build.0 = Debug|Any CPU
458+
{B6C83F04-A04B-4F00-9E68-1EC411F9317C}.Release|Any CPU.ActiveCfg = Release|Any CPU
459+
{B6C83F04-A04B-4F00-9E68-1EC411F9317C}.Release|Any CPU.Build.0 = Release|Any CPU
460+
{B6C83F04-A04B-4F00-9E68-1EC411F9317C}.Release-Intrinsics|Any CPU.ActiveCfg = Release|Any CPU
461+
{B6C83F04-A04B-4F00-9E68-1EC411F9317C}.Release-Intrinsics|Any CPU.Build.0 = Release|Any CPU
452462
EndGlobalSection
453463
GlobalSection(SolutionProperties) = preSolution
454464
HideSolutionNode = FALSE
@@ -499,6 +509,7 @@ Global
499509
{8C05642D-C3AA-4972-B02C-93681161A6BC} = {09EADF06-BE25-4228-AB53-95AE3E15B530}
500510
{73DAAC82-D308-48CC-8FFE-3B037F8BBCCA} = {09EADF06-BE25-4228-AB53-95AE3E15B530}
501511
{49D03292-8AFE-4B82-823C-D047BF8420F7} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
512+
{B6C83F04-A04B-4F00-9E68-1EC411F9317C} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
502513
EndGlobalSection
503514
GlobalSection(ExtensibilityGlobals) = postSolution
504515
SolutionGuid = {41165AF1-35BB-4832-A189-73060F82B01D}

‎test/Microsoft.ML.Benchmarks.Tests/BenchmarksTest.cs

+97
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
// See the LICENSE file in the project root for more information.
4+
5+
using BenchmarkDotNet.Attributes;
6+
using BenchmarkDotNet.Configs;
7+
using BenchmarkDotNet.Jobs;
8+
using BenchmarkDotNet.Loggers;
9+
using BenchmarkDotNet.Running;
10+
using Microsoft.ML.Runtime.Internal.CpuMath;
11+
using System;
12+
using System.Linq;
13+
using Xunit;
14+
using Xunit.Abstractions;
15+
16+
namespace Microsoft.ML.Benchmarks.Tests
17+
{
18+
public class TestConfig : RecommendedConfig
19+
{
20+
protected override Job GetJobDefinition() => Job.Dry; // Job.Dry runs the benchmark just once
21+
}
22+
23+
public class BenchmarkTouchingNativeDependency
24+
{
25+
[Benchmark]
26+
public float Simple() => CpuMathUtils.Sum(Enumerable.Range(0, 1024).Select(Convert.ToSingle).ToArray(), 1024);
27+
}
28+
29+
public class BenchmarksTest
30+
{
31+
private const string SkipTheDebug =
32+
#if DEBUG
33+
"BenchmarkDotNet does not allow running the benchmarks in Debug, so this test is disabled for DEBUG";
34+
#else
35+
"";
36+
#endif
37+
38+
public BenchmarksTest(ITestOutputHelper output) => Output = output;
39+
40+
private ITestOutputHelper Output { get; }
41+
42+
[Fact(Skip = SkipTheDebug)]
43+
public void BenchmarksProjectIsNotBroken()
44+
{
45+
var summary = BenchmarkRunner.Run<BenchmarkTouchingNativeDependency>(new TestConfig().With(new OutputLogger(Output)));
46+
47+
Assert.False(summary.HasCriticalValidationErrors, "The \"Summary\" should have NOT \"HasCriticalValidationErrors\"");
48+
49+
Assert.True(summary.Reports.Any(), "The \"Summary\" should contain at least one \"BenchmarkReport\" in the \"Reports\" collection");
50+
51+
Assert.True(summary.Reports.All(r => r.BuildResult.IsBuildSuccess),
52+
"The following benchmarks are failed to build: " +
53+
string.Join(", ", summary.Reports.Where(r => !r.BuildResult.IsBuildSuccess).Select(r => r.BenchmarkCase.DisplayInfo)));
54+
55+
Assert.True(summary.Reports.All(r => r.ExecuteResults != null),
56+
"The following benchmarks don't have any execution results: " +
57+
string.Join(", ", summary.Reports.Where(r => r.ExecuteResults == null).Select(r => r.BenchmarkCase.DisplayInfo)));
58+
59+
Assert.True(summary.Reports.All(r => r.ExecuteResults.Any(er => er.FoundExecutable && er.Data.Any())),
60+
"All reports should have at least one \"ExecuteResult\" with \"FoundExecutable\" = true and at least one \"Data\" item");
61+
62+
Assert.True(summary.Reports.All(report => report.AllMeasurements.Any()),
63+
"All reports should have at least one \"Measurement\" in the \"AllMeasurements\" collection");
64+
}
65+
}
66+
67+
public class OutputLogger : AccumulationLogger
68+
{
69+
private readonly ITestOutputHelper testOutputHelper;
70+
private string currentLine = "";
71+
72+
public OutputLogger(ITestOutputHelper testOutputHelper)
73+
{
74+
this.testOutputHelper = testOutputHelper ?? throw new ArgumentNullException(nameof(testOutputHelper));
75+
}
76+
77+
public override void Write(LogKind logKind, string text)
78+
{
79+
currentLine += text;
80+
base.Write(logKind, text);
81+
}
82+
83+
public override void WriteLine()
84+
{
85+
testOutputHelper.WriteLine(currentLine);
86+
currentLine = "";
87+
base.WriteLine();
88+
}
89+
90+
public override void WriteLine(LogKind logKind, string text)
91+
{
92+
testOutputHelper.WriteLine(currentLine + text);
93+
currentLine = "";
94+
base.WriteLine(logKind, text);
95+
}
96+
}
97+
}

‎test/Microsoft.ML.Benchmarks.Tests/Microsoft.ML.Benchmarks.Tests.csproj

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
3+
<ItemGroup>
4+
<ProjectReference Include="..\Microsoft.ML.Benchmarks\Microsoft.ML.Benchmarks.csproj" />
5+
6+
<NativeAssemblyReference Include="CpuMathNative" />
7+
</ItemGroup>
8+
</Project>

‎test/Microsoft.ML.Benchmarks/Harness/Configs.cs

+51
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
using BenchmarkDotNet.Configs;
2+
using BenchmarkDotNet.Jobs;
3+
using BenchmarkDotNet.Toolchains;
4+
using BenchmarkDotNet.Toolchains.CsProj;
5+
using BenchmarkDotNet.Toolchains.DotNetCli;
6+
using Microsoft.ML.Benchmarks.Harness;
7+
8+
namespace Microsoft.ML.Benchmarks
9+
{
10+
public class RecommendedConfig : ManualConfig
11+
{
12+
public RecommendedConfig()
13+
{
14+
Add(DefaultConfig.Instance); // this config contains all of the basic settings (exporters, columns etc)
15+
16+
Add(GetJobDefinition() // job defines how many times given benchmark should be executed
17+
.With(CreateToolchain())); // toolchain is responsible for generating, building and running dedicated executable per benchmark
18+
19+
Add(new ExtraMetricColumn()); // an extra colum that can display additional metric reported by the benchmarks
20+
21+
UnionRule = ConfigUnionRule.AlwaysUseLocal; // global config can be overwritten with local (the one set via [ConfigAttribute])
22+
}
23+
24+
protected virtual Job GetJobDefinition()
25+
=> Job.Default
26+
.WithWarmupCount(1) // ML.NET benchmarks are typically CPU-heavy benchmarks, 1 warmup is usually enough
27+
.WithMaxIterationCount(20);
28+
29+
/// <summary>
30+
/// we need our own toolchain because MSBuild by default does not copy recursive native dependencies to the output
31+
/// </summary>
32+
private IToolchain CreateToolchain()
33+
{
34+
var csProj = CsProjCoreToolchain.Current.Value;
35+
var tfm = NetCoreAppSettings.Current.Value.TargetFrameworkMoniker;
36+
37+
return new Toolchain(
38+
tfm,
39+
new ProjectGenerator(tfm), // custom generator that copies native dependencies
40+
csProj.Builder,
41+
csProj.Executor);
42+
}
43+
}
44+
45+
public class TrainConfig : RecommendedConfig
46+
{
47+
protected override Job GetJobDefinition()
48+
=> Job.Dry // the "Dry" job runs the benchmark exactly once, without any warmup to mimic real-world scenario
49+
.WithLaunchCount(3); // BDN will run 3 dedicated processes, sequentially
50+
}
51+
}

‎test/Microsoft.ML.Benchmarks/Harness/ProjectGenerator.cs

+32-21
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,14 @@
33
// See the LICENSE file in the project root for more information.
44

55
using BenchmarkDotNet.Extensions;
6+
using BenchmarkDotNet.Loggers;
7+
using BenchmarkDotNet.Running;
68
using BenchmarkDotNet.Toolchains;
79
using BenchmarkDotNet.Toolchains.CsProj;
810
using System;
911
using System.IO;
1012
using System.Linq;
13+
using System.Text;
1114

1215
namespace Microsoft.ML.Benchmarks.Harness
1316
{
@@ -18,37 +21,45 @@ namespace Microsoft.ML.Benchmarks.Harness
1821
/// the problem with ML.NET is that it has native dependencies, which are NOT copied by MSBuild to the output folder
1922
/// in case where A has native dependency and B references A
2023
///
21-
/// this is why this class exists: to copy the native dependencies to folder with .exe
24+
/// this is why this class exists:
25+
/// 1. to tell MSBuild to copy the native dependencies to folder with .exe (NativeAssemblyReference)
26+
/// 2. to generate a .csproj file that does not exclude Directory.Build.props (default BDN behaviour) which contains custom NuGet feeds that are required for restore step
2227
/// </summary>
2328
public class ProjectGenerator : CsProjGenerator
2429
{
2530
public ProjectGenerator(string targetFrameworkMoniker) : base(targetFrameworkMoniker, platform => platform.ToConfig(), null)
2631
{
2732
}
2833

29-
protected override void CopyAllRequiredFiles(ArtifactsPaths artifactsPaths)
30-
{
31-
base.CopyAllRequiredFiles(artifactsPaths);
32-
33-
CopyMissingNativeDependencies(artifactsPaths);
34-
}
34+
protected override void GenerateProject(BuildPartition buildPartition, ArtifactsPaths artifactsPaths, ILogger logger)
35+
=> File.WriteAllText(artifactsPaths.ProjectFilePath, $@"
36+
<Project Sdk=""Microsoft.NET.Sdk"">
37+
<PropertyGroup>
38+
<OutputType>Exe</OutputType>
39+
<OutputPath>bin\{buildPartition.BuildConfiguration}</OutputPath>
40+
<TargetFramework>{TargetFrameworkMoniker}</TargetFramework>
41+
<AssemblyName>{artifactsPaths.ProgramName}</AssemblyName>
42+
<AssemblyTitle>{artifactsPaths.ProgramName}</AssemblyTitle>
43+
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
44+
<TreatWarningsAsErrors>False</TreatWarningsAsErrors>
45+
<DebugType>pdbonly</DebugType>
46+
<DebugSymbols>true</DebugSymbols>
47+
</PropertyGroup>
48+
{GetRuntimeSettings(buildPartition.RepresentativeBenchmarkCase.Job.Environment.Gc, buildPartition.Resolver)}
49+
<ItemGroup>
50+
<Compile Include=""{Path.GetFileName(artifactsPaths.ProgramCodePath)}"" Exclude=""bin\**;obj\**;**\*.xproj;packages\**"" />
51+
</ItemGroup>
52+
<ItemGroup>
53+
<ProjectReference Include=""{GetProjectFilePath(buildPartition.RepresentativeBenchmarkCase.Descriptor.Type, logger).FullName}"" />
54+
{GenerateNativeReferences(buildPartition, logger)}
55+
</ItemGroup>
56+
</Project>");
3557

36-
private void CopyMissingNativeDependencies(ArtifactsPaths artifactsPaths)
58+
private string GenerateNativeReferences(BuildPartition buildPartition, ILogger logger)
3759
{
38-
var foldeWithAutogeneratedExe = Path.GetDirectoryName(artifactsPaths.ExecutablePath);
39-
var folderWithNativeDependencies = Path.GetDirectoryName(typeof(ProjectGenerator).Assembly.Location);
60+
var csproj = GetProjectFilePath(buildPartition.RepresentativeBenchmarkCase.Descriptor.Type, logger);
4061

41-
foreach (var nativeDependency in Directory
42-
.EnumerateFiles(folderWithNativeDependencies)
43-
.Where(fileName => ContainsWithIgnoreCase(fileName, "native")))
44-
{
45-
File.Copy(
46-
sourceFileName: nativeDependency,
47-
destFileName: Path.Combine(foldeWithAutogeneratedExe, Path.GetFileName(nativeDependency)),
48-
overwrite: true);
49-
}
62+
return string.Join(Environment.NewLine, File.ReadAllLines(csproj.FullName).Where(line => line.Contains("<NativeAssemblyReference")));
5063
}
51-
52-
bool ContainsWithIgnoreCase(string text, string word) => text != null && text.IndexOf(word, StringComparison.InvariantCultureIgnoreCase) >= 0;
5364
}
5465
}

‎test/Microsoft.ML.Benchmarks/Helpers.cs renamed to ‎test/Microsoft.ML.Benchmarks/Helpers/EmptyWriter.cs

+1-5
Original file line numberDiff line numberDiff line change
@@ -7,16 +7,12 @@
77

88
namespace Microsoft.ML.Benchmarks
99
{
10-
internal class Helpers
11-
{
12-
public static string DatasetNotFound = "Could not find {0} Please ensure you have run 'build.cmd -- /t:DownloadExternalTestFiles /p:IncludeBenchmarkData=true' from the root";
13-
}
14-
1510
// Adding this class to not print anything to the console.
1611
// This is required for the current version of BenchmarkDotNet
1712
internal class EmptyWriter : TextWriter
1813
{
1914
internal static readonly EmptyWriter Instance = new EmptyWriter();
15+
2016
public override Encoding Encoding => null;
2117
}
2218
}

‎test/Microsoft.ML.Benchmarks/Helpers/EnvironmentFactory.cs

+45
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
// See the LICENSE file in the project root for more information.
4+
5+
using Microsoft.ML.Core.Data;
6+
using Microsoft.ML.Runtime;
7+
using Microsoft.ML.Runtime.Data;
8+
9+
namespace Microsoft.ML.Benchmarks
10+
{
11+
internal static class EnvironmentFactory
12+
{
13+
internal static ConsoleEnvironment CreateClassificationEnvironment<TLoader, TTransformer, TTrainer>()
14+
where TLoader : IDataReader<IMultiStreamSource>
15+
where TTransformer : ITransformer
16+
where TTrainer : ITrainer
17+
{
18+
var environment = new ConsoleEnvironment(verbose: false, sensitivity: MessageSensitivity.None, outWriter: EmptyWriter.Instance);
19+
20+
environment.ComponentCatalog.RegisterAssembly(typeof(TLoader).Assembly);
21+
environment.ComponentCatalog.RegisterAssembly(typeof(TTransformer).Assembly);
22+
environment.ComponentCatalog.RegisterAssembly(typeof(TTrainer).Assembly);
23+
24+
return environment;
25+
}
26+
27+
internal static ConsoleEnvironment CreateRankingEnvironment<TEvaluator, TLoader, TTransformer, TTrainer>()
28+
where TEvaluator : IEvaluator
29+
where TLoader : IDataReader<IMultiStreamSource>
30+
where TTransformer : ITransformer
31+
where TTrainer : ITrainer
32+
{
33+
var environment = new ConsoleEnvironment(verbose: false, sensitivity: MessageSensitivity.None, outWriter: EmptyWriter.Instance);
34+
35+
environment.ComponentCatalog.RegisterAssembly(typeof(TEvaluator).Assembly);
36+
environment.ComponentCatalog.RegisterAssembly(typeof(TLoader).Assembly);
37+
environment.ComponentCatalog.RegisterAssembly(typeof(TTransformer).Assembly);
38+
environment.ComponentCatalog.RegisterAssembly(typeof(TTrainer).Assembly);
39+
40+
environment.ComponentCatalog.RegisterAssembly(typeof(NAHandleTransform).Assembly);
41+
42+
return environment;
43+
}
44+
}
45+
}

‎test/Microsoft.ML.Benchmarks/Helpers/Errors.cs

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
// See the LICENSE file in the project root for more information.
4+
5+
namespace Microsoft.ML.Benchmarks
6+
{
7+
internal class Errors
8+
{
9+
public static string DatasetNotFound = "Could not find {0} Please ensure you have run 'build.cmd -- /t:DownloadExternalTestFiles /p:IncludeBenchmarkData=true' from the root";
10+
}
11+
}

0 commit comments

Comments
 (0)