diff --git a/Microsoft.ML.sln b/Microsoft.ML.sln index d6dd898ee3..1b12394544 100644 --- a/Microsoft.ML.sln +++ b/Microsoft.ML.sln @@ -121,6 +121,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.DnnAnalyzer", EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.ML.OnnxTransformTest", "test\Microsoft.ML.OnnxTransformTest\Microsoft.ML.OnnxTransformTest.csproj", "{49D03292-8AFE-4B82-823C-D047BF8420F7}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.Benchmarks.Tests", "test\Microsoft.ML.Benchmarks.Tests\Microsoft.ML.Benchmarks.Tests.csproj", "{B6C83F04-A04B-4F00-9E68-1EC411F9317C}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -449,6 +451,14 @@ Global {49D03292-8AFE-4B82-823C-D047BF8420F7}.Release|Any CPU.Build.0 = Release|Any CPU {49D03292-8AFE-4B82-823C-D047BF8420F7}.Release-Intrinsics|Any CPU.ActiveCfg = Release|Any CPU {49D03292-8AFE-4B82-823C-D047BF8420F7}.Release-Intrinsics|Any CPU.Build.0 = Release|Any CPU + {B6C83F04-A04B-4F00-9E68-1EC411F9317C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B6C83F04-A04B-4F00-9E68-1EC411F9317C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B6C83F04-A04B-4F00-9E68-1EC411F9317C}.Debug-Intrinsics|Any CPU.ActiveCfg = Debug|Any CPU + {B6C83F04-A04B-4F00-9E68-1EC411F9317C}.Debug-Intrinsics|Any CPU.Build.0 = Debug|Any CPU + {B6C83F04-A04B-4F00-9E68-1EC411F9317C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B6C83F04-A04B-4F00-9E68-1EC411F9317C}.Release|Any CPU.Build.0 = Release|Any CPU + {B6C83F04-A04B-4F00-9E68-1EC411F9317C}.Release-Intrinsics|Any CPU.ActiveCfg = Release|Any CPU + {B6C83F04-A04B-4F00-9E68-1EC411F9317C}.Release-Intrinsics|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -499,6 +509,7 @@ Global {8C05642D-C3AA-4972-B02C-93681161A6BC} = {09EADF06-BE25-4228-AB53-95AE3E15B530} {73DAAC82-D308-48CC-8FFE-3B037F8BBCCA} = {09EADF06-BE25-4228-AB53-95AE3E15B530} {49D03292-8AFE-4B82-823C-D047BF8420F7} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4} + {B6C83F04-A04B-4F00-9E68-1EC411F9317C} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {41165AF1-35BB-4832-A189-73060F82B01D} diff --git a/test/Microsoft.ML.Benchmarks.Tests/BenchmarksTest.cs b/test/Microsoft.ML.Benchmarks.Tests/BenchmarksTest.cs new file mode 100644 index 0000000000..d4e8499df3 --- /dev/null +++ b/test/Microsoft.ML.Benchmarks.Tests/BenchmarksTest.cs @@ -0,0 +1,97 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Running; +using Microsoft.ML.Runtime.Internal.CpuMath; +using System; +using System.Linq; +using Xunit; +using Xunit.Abstractions; + +namespace Microsoft.ML.Benchmarks.Tests +{ + public class TestConfig : RecommendedConfig + { + protected override Job GetJobDefinition() => Job.Dry; // Job.Dry runs the benchmark just once + } + + public class BenchmarkTouchingNativeDependency + { + [Benchmark] + public float Simple() => CpuMathUtils.Sum(Enumerable.Range(0, 1024).Select(Convert.ToSingle).ToArray(), 1024); + } + + public class BenchmarksTest + { + private const string SkipTheDebug = +#if DEBUG + "BenchmarkDotNet does not allow running the benchmarks in Debug, so this test is disabled for DEBUG"; +#else + ""; +#endif + + public BenchmarksTest(ITestOutputHelper output) => Output = output; + + private ITestOutputHelper Output { get; } + + [Fact(Skip = SkipTheDebug)] + public void BenchmarksProjectIsNotBroken() + { + var summary = BenchmarkRunner.Run(new TestConfig().With(new OutputLogger(Output))); + + Assert.False(summary.HasCriticalValidationErrors, "The \"Summary\" should have NOT \"HasCriticalValidationErrors\""); + + Assert.True(summary.Reports.Any(), "The \"Summary\" should contain at least one \"BenchmarkReport\" in the \"Reports\" collection"); + + Assert.True(summary.Reports.All(r => r.BuildResult.IsBuildSuccess), + "The following benchmarks are failed to build: " + + string.Join(", ", summary.Reports.Where(r => !r.BuildResult.IsBuildSuccess).Select(r => r.BenchmarkCase.DisplayInfo))); + + Assert.True(summary.Reports.All(r => r.ExecuteResults != null), + "The following benchmarks don't have any execution results: " + + string.Join(", ", summary.Reports.Where(r => r.ExecuteResults == null).Select(r => r.BenchmarkCase.DisplayInfo))); + + Assert.True(summary.Reports.All(r => r.ExecuteResults.Any(er => er.FoundExecutable && er.Data.Any())), + "All reports should have at least one \"ExecuteResult\" with \"FoundExecutable\" = true and at least one \"Data\" item"); + + Assert.True(summary.Reports.All(report => report.AllMeasurements.Any()), + "All reports should have at least one \"Measurement\" in the \"AllMeasurements\" collection"); + } + } + + public class OutputLogger : AccumulationLogger + { + private readonly ITestOutputHelper testOutputHelper; + private string currentLine = ""; + + public OutputLogger(ITestOutputHelper testOutputHelper) + { + this.testOutputHelper = testOutputHelper ?? throw new ArgumentNullException(nameof(testOutputHelper)); + } + + public override void Write(LogKind logKind, string text) + { + currentLine += text; + base.Write(logKind, text); + } + + public override void WriteLine() + { + testOutputHelper.WriteLine(currentLine); + currentLine = ""; + base.WriteLine(); + } + + public override void WriteLine(LogKind logKind, string text) + { + testOutputHelper.WriteLine(currentLine + text); + currentLine = ""; + base.WriteLine(logKind, text); + } + } +} diff --git a/test/Microsoft.ML.Benchmarks.Tests/Microsoft.ML.Benchmarks.Tests.csproj b/test/Microsoft.ML.Benchmarks.Tests/Microsoft.ML.Benchmarks.Tests.csproj new file mode 100644 index 0000000000..6d71d84c73 --- /dev/null +++ b/test/Microsoft.ML.Benchmarks.Tests/Microsoft.ML.Benchmarks.Tests.csproj @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/test/Microsoft.ML.Benchmarks/Harness/Configs.cs b/test/Microsoft.ML.Benchmarks/Harness/Configs.cs new file mode 100644 index 0000000000..c51a0ea3e2 --- /dev/null +++ b/test/Microsoft.ML.Benchmarks/Harness/Configs.cs @@ -0,0 +1,51 @@ +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Toolchains; +using BenchmarkDotNet.Toolchains.CsProj; +using BenchmarkDotNet.Toolchains.DotNetCli; +using Microsoft.ML.Benchmarks.Harness; + +namespace Microsoft.ML.Benchmarks +{ + public class RecommendedConfig : ManualConfig + { + public RecommendedConfig() + { + Add(DefaultConfig.Instance); // this config contains all of the basic settings (exporters, columns etc) + + Add(GetJobDefinition() // job defines how many times given benchmark should be executed + .With(CreateToolchain())); // toolchain is responsible for generating, building and running dedicated executable per benchmark + + Add(new ExtraMetricColumn()); // an extra colum that can display additional metric reported by the benchmarks + + UnionRule = ConfigUnionRule.AlwaysUseLocal; // global config can be overwritten with local (the one set via [ConfigAttribute]) + } + + protected virtual Job GetJobDefinition() + => Job.Default + .WithWarmupCount(1) // ML.NET benchmarks are typically CPU-heavy benchmarks, 1 warmup is usually enough + .WithMaxIterationCount(20); + + /// + /// we need our own toolchain because MSBuild by default does not copy recursive native dependencies to the output + /// + private IToolchain CreateToolchain() + { + var csProj = CsProjCoreToolchain.Current.Value; + var tfm = NetCoreAppSettings.Current.Value.TargetFrameworkMoniker; + + return new Toolchain( + tfm, + new ProjectGenerator(tfm), // custom generator that copies native dependencies + csProj.Builder, + csProj.Executor); + } + } + + public class TrainConfig : RecommendedConfig + { + protected override Job GetJobDefinition() + => Job.Dry // the "Dry" job runs the benchmark exactly once, without any warmup to mimic real-world scenario + .WithLaunchCount(3); // BDN will run 3 dedicated processes, sequentially + } +} diff --git a/test/Microsoft.ML.Benchmarks/Harness/ProjectGenerator.cs b/test/Microsoft.ML.Benchmarks/Harness/ProjectGenerator.cs index 7560efe562..2c77ce6d79 100644 --- a/test/Microsoft.ML.Benchmarks/Harness/ProjectGenerator.cs +++ b/test/Microsoft.ML.Benchmarks/Harness/ProjectGenerator.cs @@ -3,11 +3,14 @@ // See the LICENSE file in the project root for more information. using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Running; using BenchmarkDotNet.Toolchains; using BenchmarkDotNet.Toolchains.CsProj; using System; using System.IO; using System.Linq; +using System.Text; namespace Microsoft.ML.Benchmarks.Harness { @@ -18,7 +21,9 @@ namespace Microsoft.ML.Benchmarks.Harness /// the problem with ML.NET is that it has native dependencies, which are NOT copied by MSBuild to the output folder /// in case where A has native dependency and B references A /// - /// this is why this class exists: to copy the native dependencies to folder with .exe + /// this is why this class exists: + /// 1. to tell MSBuild to copy the native dependencies to folder with .exe (NativeAssemblyReference) + /// 2. to generate a .csproj file that does not exclude Directory.Build.props (default BDN behaviour) which contains custom NuGet feeds that are required for restore step /// public class ProjectGenerator : CsProjGenerator { @@ -26,29 +31,35 @@ public ProjectGenerator(string targetFrameworkMoniker) : base(targetFrameworkMon { } - protected override void CopyAllRequiredFiles(ArtifactsPaths artifactsPaths) - { - base.CopyAllRequiredFiles(artifactsPaths); - - CopyMissingNativeDependencies(artifactsPaths); - } + protected override void GenerateProject(BuildPartition buildPartition, ArtifactsPaths artifactsPaths, ILogger logger) + => File.WriteAllText(artifactsPaths.ProjectFilePath, $@" + + + Exe + bin\{buildPartition.BuildConfiguration} + {TargetFrameworkMoniker} + {artifactsPaths.ProgramName} + {artifactsPaths.ProgramName} + true + False + pdbonly + true + + {GetRuntimeSettings(buildPartition.RepresentativeBenchmarkCase.Job.Environment.Gc, buildPartition.Resolver)} + + + + + + {GenerateNativeReferences(buildPartition, logger)} + +"); - private void CopyMissingNativeDependencies(ArtifactsPaths artifactsPaths) + private string GenerateNativeReferences(BuildPartition buildPartition, ILogger logger) { - var foldeWithAutogeneratedExe = Path.GetDirectoryName(artifactsPaths.ExecutablePath); - var folderWithNativeDependencies = Path.GetDirectoryName(typeof(ProjectGenerator).Assembly.Location); + var csproj = GetProjectFilePath(buildPartition.RepresentativeBenchmarkCase.Descriptor.Type, logger); - foreach (var nativeDependency in Directory - .EnumerateFiles(folderWithNativeDependencies) - .Where(fileName => ContainsWithIgnoreCase(fileName, "native"))) - { - File.Copy( - sourceFileName: nativeDependency, - destFileName: Path.Combine(foldeWithAutogeneratedExe, Path.GetFileName(nativeDependency)), - overwrite: true); - } + return string.Join(Environment.NewLine, File.ReadAllLines(csproj.FullName).Where(line => line.Contains(" text != null && text.IndexOf(word, StringComparison.InvariantCultureIgnoreCase) >= 0; } } diff --git a/test/Microsoft.ML.Benchmarks/Helpers.cs b/test/Microsoft.ML.Benchmarks/Helpers/EmptyWriter.cs similarity index 72% rename from test/Microsoft.ML.Benchmarks/Helpers.cs rename to test/Microsoft.ML.Benchmarks/Helpers/EmptyWriter.cs index 55832fa13f..a54c8f54df 100644 --- a/test/Microsoft.ML.Benchmarks/Helpers.cs +++ b/test/Microsoft.ML.Benchmarks/Helpers/EmptyWriter.cs @@ -7,16 +7,12 @@ namespace Microsoft.ML.Benchmarks { - internal class Helpers - { - public static string DatasetNotFound = "Could not find {0} Please ensure you have run 'build.cmd -- /t:DownloadExternalTestFiles /p:IncludeBenchmarkData=true' from the root"; - } - // Adding this class to not print anything to the console. // This is required for the current version of BenchmarkDotNet internal class EmptyWriter : TextWriter { internal static readonly EmptyWriter Instance = new EmptyWriter(); + public override Encoding Encoding => null; } } diff --git a/test/Microsoft.ML.Benchmarks/Helpers/EnvironmentFactory.cs b/test/Microsoft.ML.Benchmarks/Helpers/EnvironmentFactory.cs new file mode 100644 index 0000000000..bf76e67231 --- /dev/null +++ b/test/Microsoft.ML.Benchmarks/Helpers/EnvironmentFactory.cs @@ -0,0 +1,45 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.ML.Core.Data; +using Microsoft.ML.Runtime; +using Microsoft.ML.Runtime.Data; + +namespace Microsoft.ML.Benchmarks +{ + internal static class EnvironmentFactory + { + internal static ConsoleEnvironment CreateClassificationEnvironment() + where TLoader : IDataReader + where TTransformer : ITransformer + where TTrainer : ITrainer + { + var environment = new ConsoleEnvironment(verbose: false, sensitivity: MessageSensitivity.None, outWriter: EmptyWriter.Instance); + + environment.ComponentCatalog.RegisterAssembly(typeof(TLoader).Assembly); + environment.ComponentCatalog.RegisterAssembly(typeof(TTransformer).Assembly); + environment.ComponentCatalog.RegisterAssembly(typeof(TTrainer).Assembly); + + return environment; + } + + internal static ConsoleEnvironment CreateRankingEnvironment() + where TEvaluator : IEvaluator + where TLoader : IDataReader + where TTransformer : ITransformer + where TTrainer : ITrainer + { + var environment = new ConsoleEnvironment(verbose: false, sensitivity: MessageSensitivity.None, outWriter: EmptyWriter.Instance); + + environment.ComponentCatalog.RegisterAssembly(typeof(TEvaluator).Assembly); + environment.ComponentCatalog.RegisterAssembly(typeof(TLoader).Assembly); + environment.ComponentCatalog.RegisterAssembly(typeof(TTransformer).Assembly); + environment.ComponentCatalog.RegisterAssembly(typeof(TTrainer).Assembly); + + environment.ComponentCatalog.RegisterAssembly(typeof(NAHandleTransform).Assembly); + + return environment; + } + } +} diff --git a/test/Microsoft.ML.Benchmarks/Helpers/Errors.cs b/test/Microsoft.ML.Benchmarks/Helpers/Errors.cs new file mode 100644 index 0000000000..344cbbccb7 --- /dev/null +++ b/test/Microsoft.ML.Benchmarks/Helpers/Errors.cs @@ -0,0 +1,11 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace Microsoft.ML.Benchmarks +{ + internal class Errors + { + public static string DatasetNotFound = "Could not find {0} Please ensure you have run 'build.cmd -- /t:DownloadExternalTestFiles /p:IncludeBenchmarkData=true' from the root"; + } +} diff --git a/test/Microsoft.ML.Benchmarks/Numeric/Ranking.cs b/test/Microsoft.ML.Benchmarks/Numeric/Ranking.cs index 81e8e196ff..68a78c5c5b 100644 --- a/test/Microsoft.ML.Benchmarks/Numeric/Ranking.cs +++ b/test/Microsoft.ML.Benchmarks/Numeric/Ranking.cs @@ -3,10 +3,12 @@ // See the LICENSE file in the project root for more information. using BenchmarkDotNet.Attributes; -using Microsoft.ML.Runtime; using Microsoft.ML.Runtime.Data; +using Microsoft.ML.Runtime.FastTree; +using Microsoft.ML.Runtime.LightGBM; using Microsoft.ML.Runtime.RunTests; using Microsoft.ML.Runtime.Tools; +using Microsoft.ML.Transforms; using System.IO; namespace Microsoft.ML.Benchmarks @@ -24,10 +26,10 @@ public void SetupTrainingSpeedTests() _mslrWeb10k_Train = Path.GetFullPath(TestDatasets.MSLRWeb.trainFilename); if (!File.Exists(_mslrWeb10k_Validate)) - throw new FileNotFoundException(string.Format(Helpers.DatasetNotFound, _mslrWeb10k_Validate)); + throw new FileNotFoundException(string.Format(Errors.DatasetNotFound, _mslrWeb10k_Validate)); if (!File.Exists(_mslrWeb10k_Train)) - throw new FileNotFoundException(string.Format(Helpers.DatasetNotFound, _mslrWeb10k_Train)); + throw new FileNotFoundException(string.Format(Errors.DatasetNotFound, _mslrWeb10k_Train)); } [Benchmark] @@ -40,7 +42,7 @@ public void TrainTest_Ranking_MSLRWeb10K_RawNumericFeatures_FastTreeRanking() " xf=HashTransform{col=GroupId} xf=NAHandleTransform{col=Features}" + " tr=FastTreeRanking{}"; - using (var environment = new ConsoleEnvironment(verbose: false, sensitivity: MessageSensitivity.None, outWriter: EmptyWriter.Instance)) + using (var environment = EnvironmentFactory.CreateRankingEnvironment()) { Maml.MainCore(environment, cmd, alwaysPrintStacktrace: false); } @@ -57,14 +59,13 @@ public void TrainTest_Ranking_MSLRWeb10K_RawNumericFeatures_LightGBMRanking() " xf=NAHandleTransform{col=Features}" + " tr=LightGBMRanking{}"; - using (var environment = new ConsoleEnvironment(verbose: false, sensitivity: MessageSensitivity.None, outWriter: EmptyWriter.Instance)) + using (var environment = EnvironmentFactory.CreateRankingEnvironment()) { Maml.MainCore(environment, cmd, alwaysPrintStacktrace: false); } } } - [Config(typeof(PredictConfig))] public class RankingTest { private string _mslrWeb10k_Validate; @@ -80,13 +81,13 @@ public void SetupScoringSpeedTests() _mslrWeb10k_Train = Path.GetFullPath(TestDatasets.MSLRWeb.trainFilename); if (!File.Exists(_mslrWeb10k_Test)) - throw new FileNotFoundException(string.Format(Helpers.DatasetNotFound, _mslrWeb10k_Test)); + throw new FileNotFoundException(string.Format(Errors.DatasetNotFound, _mslrWeb10k_Test)); if (!File.Exists(_mslrWeb10k_Validate)) - throw new FileNotFoundException(string.Format(Helpers.DatasetNotFound, _mslrWeb10k_Validate)); + throw new FileNotFoundException(string.Format(Errors.DatasetNotFound, _mslrWeb10k_Validate)); if (!File.Exists(_mslrWeb10k_Train)) - throw new FileNotFoundException(string.Format(Helpers.DatasetNotFound, _mslrWeb10k_Train)); + throw new FileNotFoundException(string.Format(Errors.DatasetNotFound, _mslrWeb10k_Train)); _modelPath_MSLR = Path.Combine(Directory.GetCurrentDirectory(), @"FastTreeRankingModel.zip"); @@ -99,7 +100,7 @@ public void SetupScoringSpeedTests() " tr=FastTreeRanking{}" + " out={" + _modelPath_MSLR + "}"; - using (var environment = new ConsoleEnvironment(verbose: false, sensitivity: MessageSensitivity.None, outWriter: EmptyWriter.Instance)) + using (var environment = EnvironmentFactory.CreateRankingEnvironment()) { Maml.MainCore(environment, cmd, alwaysPrintStacktrace: false); } @@ -110,7 +111,8 @@ public void Test_Ranking_MSLRWeb10K_RawNumericFeatures_FastTreeRanking() { // This benchmark is profiling bulk scoring speed and not training speed. string cmd = @"Test data=" + _mslrWeb10k_Test + " in=" + _modelPath_MSLR; - using (var environment = new ConsoleEnvironment(verbose: false, sensitivity: MessageSensitivity.None, outWriter: EmptyWriter.Instance)) + + using (var environment = EnvironmentFactory.CreateRankingEnvironment()) { Maml.MainCore(environment, cmd, alwaysPrintStacktrace: false); } diff --git a/test/Microsoft.ML.Benchmarks/PredictConfig.cs b/test/Microsoft.ML.Benchmarks/PredictConfig.cs deleted file mode 100644 index 21165b8ae1..0000000000 --- a/test/Microsoft.ML.Benchmarks/PredictConfig.cs +++ /dev/null @@ -1,24 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using BenchmarkDotNet.Configs; -using BenchmarkDotNet.Diagnosers; -using BenchmarkDotNet.Jobs; - -namespace Microsoft.ML.Benchmarks -{ - internal class PredictConfig : ManualConfig - { - public PredictConfig() - { - Add(DefaultConfig.Instance - .With(Job.Default - .WithWarmupCount(1) - .WithMaxIterationCount(20) - .With(Program.CreateToolchain())) - .With(new ExtraMetricColumn()) - .With(MemoryDiagnoser.Default)); - } - } -} diff --git a/test/Microsoft.ML.Benchmarks/PredictionEngineBench.cs b/test/Microsoft.ML.Benchmarks/PredictionEngineBench.cs index 1ea4802738..9302d9b3eb 100644 --- a/test/Microsoft.ML.Benchmarks/PredictionEngineBench.cs +++ b/test/Microsoft.ML.Benchmarks/PredictionEngineBench.cs @@ -10,7 +10,6 @@ namespace Microsoft.ML.Benchmarks { - [Config(typeof(PredictConfig))] public class PredictionEngineBench { private IrisData _irisExample; diff --git a/test/Microsoft.ML.Benchmarks/Program.cs b/test/Microsoft.ML.Benchmarks/Program.cs index a4aff095c3..937300ffa2 100644 --- a/test/Microsoft.ML.Benchmarks/Program.cs +++ b/test/Microsoft.ML.Benchmarks/Program.cs @@ -3,10 +3,6 @@ // See the LICENSE file in the project root for more information. using BenchmarkDotNet.Running; -using BenchmarkDotNet.Toolchains; -using BenchmarkDotNet.Toolchains.CsProj; -using BenchmarkDotNet.Toolchains.DotNetCli; -using Microsoft.ML.Benchmarks.Harness; using System.Globalization; using System.IO; using System.Threading; @@ -22,22 +18,7 @@ class Program static void Main(string[] args) => BenchmarkSwitcher .FromAssembly(typeof(Program).Assembly) - .Run(args); - - /// - /// we need our own toolchain because MSBuild by default does not copy recursive native dependencies to the output - /// - internal static IToolchain CreateToolchain() - { - var csProj = CsProjCoreToolchain.Current.Value; - var tfm = NetCoreAppSettings.Current.Value.TargetFrameworkMoniker; - - return new Toolchain( - tfm, - new ProjectGenerator(tfm), - csProj.Builder, - csProj.Executor); - } + .Run(args, new RecommendedConfig()); internal static string GetInvariantCultureDataPath(string name) { diff --git a/test/Microsoft.ML.Benchmarks/README.md b/test/Microsoft.ML.Benchmarks/README.md index c40de49c1f..aa7e59ad15 100644 --- a/test/Microsoft.ML.Benchmarks/README.md +++ b/test/Microsoft.ML.Benchmarks/README.md @@ -7,6 +7,10 @@ This project contains performance benchmarks. **Pre-requisite:** On a clean repo, `build.cmd` at the root installs the right version of dotnet.exe and builds the solution. You need to build the solution in `Release` with native dependencies. build.cmd -release -buildNative + +Moreover, to run some of the benchmarks you have to download external dependencies. + + build.cmd -- /t:DownloadExternalTestFiles /p:IncludeBenchmarkData=true 1. Navigate to the benchmarks directory (machinelearning\test\Microsoft.ML.Benchmarks) @@ -25,8 +29,39 @@ This project contains performance benchmarks. dotnet run -c Release -- --filter namespace.typeName.methodName ``` -4. To find out more about supported command line arguments run +4. GC Statistics + +To get the total number of allocated managed memory please pass additional console argument: `--memory` or just `-m`. This feature is disabled by default because it requires an additional iteration (which is expensive for time consuming benchmarks). + +| Gen 0 | Gen 1 | Gen 2 | Allocated | +|------------:|-----------:|----------:|----------:| +| 175000.0000 | 33000.0000 | 7000.0000 | 238.26 MB | + +5. To find out more about supported command line arguments run ```log dotnet run -c Release -- --help ``` + +## Authoring new benchmarks + +1. The type which contains benchmark(s) has to be a public, non-sealed, non-static class. +2. Put the initialization logic into a separate public method with `[GlobalSetup]` attribute. You can use `Target` property to make it specific for selected benchmark. Example: `[GlobalSetup(Target = nameof(MakeIrisPredictions))]`. +3. Put the benchmarked code into a separate public method with `[Benchmark]` attribute. If the benchmark method computes some result, please return it from the benchmark. Harness will consume it to avoid dead code elimination. +4. If given benchmark is a Training benchmark, please apply `[Config(typeof(TrainConfig))]` to the class. It will tell BenchmarkDotNet to run the benchmark only once in a dedicated process to mimic the real-world scenario for training. + +Examples: + +```cs +public class NonTrainingBenchmark +{ + [GlobalSetup(Target = nameof(TheBenchmark))] + public void Setup() { /* setup logic goes here */ } + + [Benchmark] + public SomeResult TheBenchmark() { /* benchmarked code goes here */ } +} + +[Config(typeof(TrainConfig))] +public class TrainingBenchmark +``` diff --git a/test/Microsoft.ML.Benchmarks/Text/MultiClassClassification.cs b/test/Microsoft.ML.Benchmarks/Text/MultiClassClassification.cs index 656c320b26..22d84c90ab 100644 --- a/test/Microsoft.ML.Benchmarks/Text/MultiClassClassification.cs +++ b/test/Microsoft.ML.Benchmarks/Text/MultiClassClassification.cs @@ -3,8 +3,9 @@ // See the LICENSE file in the project root for more information. using BenchmarkDotNet.Attributes; -using Microsoft.ML.Runtime; using Microsoft.ML.Runtime.Data; +using Microsoft.ML.Runtime.Learners; +using Microsoft.ML.Runtime.LightGBM; using Microsoft.ML.Runtime.RunTests; using Microsoft.ML.Runtime.Tools; using System.IO; @@ -22,7 +23,7 @@ public void SetupTrainingSpeedTests() _dataPath_Wiki = Path.GetFullPath(TestDatasets.WikiDetox.trainFilename); if (!File.Exists(_dataPath_Wiki)) - throw new FileNotFoundException(string.Format(Helpers.DatasetNotFound, _dataPath_Wiki)); + throw new FileNotFoundException(string.Format(Errors.DatasetNotFound, _dataPath_Wiki)); } [Benchmark] @@ -36,7 +37,7 @@ public void CV_Multiclass_WikiDetox_BigramsAndTrichar_OVAAveragedPerceptron() " xf=Concat{col=Features:FeaturesText,logged_in,ns}" + " tr=OVA{p=AveragedPerceptron{iter=10}}"; - using (var environment = new ConsoleEnvironment(verbose: false, sensitivity: MessageSensitivity.None, outWriter: EmptyWriter.Instance)) + using (var environment = EnvironmentFactory.CreateClassificationEnvironment()) { Maml.MainCore(environment, cmd, alwaysPrintStacktrace: false); } @@ -53,7 +54,7 @@ public void CV_Multiclass_WikiDetox_BigramsAndTrichar_LightGBMMulticlass() " xf=Concat{col=Features:FeaturesText,logged_in,ns}" + " tr=LightGBMMulticlass{iter=10}"; - using (var environment = new ConsoleEnvironment(verbose: false, sensitivity: MessageSensitivity.None, outWriter: EmptyWriter.Instance)) + using (var environment = EnvironmentFactory.CreateClassificationEnvironment()) { Maml.MainCore(environment, cmd, alwaysPrintStacktrace: false); } @@ -71,7 +72,7 @@ public void CV_Multiclass_WikiDetox_WordEmbeddings_OVAAveragedPerceptron() " xf=WordEmbeddingsTransform{col=FeaturesWordEmbedding:FeaturesText_TransformedText model=FastTextWikipedia300D}" + " xf=Concat{col=Features:FeaturesText,FeaturesWordEmbedding,logged_in,ns}"; - using (var environment = new ConsoleEnvironment(verbose: false, sensitivity: MessageSensitivity.None, outWriter: EmptyWriter.Instance)) + using (var environment = EnvironmentFactory.CreateClassificationEnvironment()) { Maml.MainCore(environment, cmd, alwaysPrintStacktrace: false); } @@ -89,14 +90,13 @@ public void CV_Multiclass_WikiDetox_WordEmbeddings_SDCAMC() " xf=WordEmbeddingsTransform{col=FeaturesWordEmbedding:FeaturesText_TransformedText model=FastTextWikipedia300D}" + " xf=Concat{col=Features:FeaturesWordEmbedding,logged_in,ns}"; - using (var environment = new ConsoleEnvironment(verbose: false, sensitivity: MessageSensitivity.None, outWriter: EmptyWriter.Instance)) + using (var environment = EnvironmentFactory.CreateClassificationEnvironment()) { Maml.MainCore(environment, cmd, alwaysPrintStacktrace: false); } } } - [Config(typeof(PredictConfig))] public class MultiClassClassificationTest { private string _dataPath_Wiki; @@ -108,7 +108,7 @@ public void SetupScoringSpeedTests() _dataPath_Wiki = Path.GetFullPath(TestDatasets.WikiDetox.trainFilename); if (!File.Exists(_dataPath_Wiki)) - throw new FileNotFoundException(string.Format(Helpers.DatasetNotFound, _dataPath_Wiki)); + throw new FileNotFoundException(string.Format(Errors.DatasetNotFound, _dataPath_Wiki)); _modelPath_Wiki = Path.Combine(Directory.GetCurrentDirectory(), @"WikiModel.zip"); @@ -120,7 +120,7 @@ public void SetupScoringSpeedTests() " tr=OVA{p=AveragedPerceptron{iter=10}}" + " out={" + _modelPath_Wiki + "}"; - using (var environment = new ConsoleEnvironment(verbose: false, sensitivity: MessageSensitivity.None, outWriter: EmptyWriter.Instance)) + using (var environment = EnvironmentFactory.CreateClassificationEnvironment()) { Maml.MainCore(environment, cmd, alwaysPrintStacktrace: false); } @@ -132,7 +132,8 @@ public void Test_Multiclass_WikiDetox_BigramsAndTrichar_OVAAveragedPerceptron() // This benchmark is profiling bulk scoring speed and not training speed. string modelpath = Path.Combine(Directory.GetCurrentDirectory(), @"WikiModel.fold000.zip"); string cmd = @"Test data=" + _dataPath_Wiki + " in=" + modelpath; - using (var environment = new ConsoleEnvironment(verbose: false, sensitivity: MessageSensitivity.None, outWriter: EmptyWriter.Instance)) + + using (var environment = EnvironmentFactory.CreateClassificationEnvironment()) { Maml.MainCore(environment, cmd, alwaysPrintStacktrace: false); } diff --git a/test/Microsoft.ML.Benchmarks/TrainConfig.cs b/test/Microsoft.ML.Benchmarks/TrainConfig.cs deleted file mode 100644 index 442c259afc..0000000000 --- a/test/Microsoft.ML.Benchmarks/TrainConfig.cs +++ /dev/null @@ -1,25 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using BenchmarkDotNet.Configs; -using BenchmarkDotNet.Diagnosers; -using BenchmarkDotNet.Jobs; - -namespace Microsoft.ML.Benchmarks -{ - public class TrainConfig : ManualConfig - { - public TrainConfig() - { - Add(DefaultConfig.Instance - .With(Job.Default - .WithWarmupCount(0) - .WithIterationCount(1) - .WithLaunchCount(3) // BDN will start 3 dedicated processes, each of them will just run given benchmark once, without any warm up to mimic the real world. - .With(Program.CreateToolchain())) - .With(new ExtraMetricColumn()) - .With(MemoryDiagnoser.Default)); - } - } -}