Skip to content

Commit 6b9f589

Browse files
AnipikAnipik
authored
Added Dry job for benchmarks CI (dotnet#2109)
* Added Dry job for benchmarks ci * Increasing Timeout and not copying large datasets to output directory * disabling failing benchmark tests * using GetDataPath function and simplyfying code * Name changed to datapathprovider, make variables private, make xunit variable public * making can execute static * Removed Datasets from output and dead import * Dynamically finding the root of the repo * using Assembly.load, making getrepo root private, removing touching dependencyTest * using typeof.assembly, adding summary comment to attribute and adding using statement for TestFramework
1 parent ef638f4 commit 6b9f589

15 files changed

+130
-82
lines changed

‎.vsts-dotnet-ci.yml

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,11 @@ phases:
1616
Build_Debug_Intrinsics:
1717
_configuration: Debug-Intrinsics
1818
_config_short: DI
19+
_includeBenchmarkData: false
1920
Build_Release_Intrinsics:
2021
_configuration: Release-Intrinsics
2122
_config_short: RI
23+
_includeBenchmarkData: true
2224
queue:
2325
name: Hosted VS2017
2426

@@ -45,9 +47,11 @@ phases:
4547
Build_Debug_netfx:
4648
_configuration: Debug-netfx
4749
_config_short: DFX
50+
_includeBenchmarkData: false
4851
Build_Release_netfx:
4952
_configuration: Release-netfx
50-
_config_short: RFX
53+
_config_short: RFX
54+
_includeBenchmarkData: false
5155
queue:
5256
name: Hosted VS2017
5357

‎build/ci/phase-template.yml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,15 +19,19 @@ phases:
1919
Build_Debug:
2020
_configuration: Debug
2121
_config_short: D
22+
_includeBenchmarkData: false
2223
Build_Release:
2324
_configuration: Release
2425
_config_short: R
26+
_includeBenchmarkData: true
2527
${{ if ne(parameters.customMatrixes, '') }}:
2628
${{ insert }}: ${{ parameters.customMatrixes }}
2729
${{ insert }}: ${{ parameters.queue }}
2830
steps:
2931
- script: $(_buildScript) -$(_configuration) -buildArch=$(_arch)
3032
displayName: Build
33+
- script: $(_buildScript) -- /t:DownloadExternalTestFiles /p:IncludeBenchmarkData=$(_includeBenchmarkData)
34+
displayName: Download Benchmark Data
3135
- ${{ if eq(parameters.name, 'MacOS') }}:
3236
- script: brew update && brew install libomp mono-libgdiplus gettext && brew link gettext --force
3337
displayName: Install runtime dependencies

‎test/Microsoft.ML.Benchmarks.Tests/BenchmarksTest.cs

Lines changed: 29 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,15 @@
33
// See the LICENSE file in the project root for more information.
44

55
using System;
6+
using System.Collections.Generic;
67
using System.Linq;
8+
using System.Reflection;
79
using BenchmarkDotNet.Attributes;
810
using BenchmarkDotNet.Configs;
911
using BenchmarkDotNet.Jobs;
1012
using BenchmarkDotNet.Loggers;
1113
using BenchmarkDotNet.Running;
14+
using Microsoft.ML.Benchmarks.Harness;
1215
using Microsoft.ML.Internal.CpuMath;
1316
using Xunit;
1417
using Xunit.Abstractions;
@@ -20,33 +23,42 @@ public class TestConfig : RecommendedConfig
2023
protected override Job GetJobDefinition() => Job.Dry; // Job.Dry runs the benchmark just once
2124
}
2225

23-
public class BenchmarkTouchingNativeDependency
24-
{
25-
[Benchmark]
26-
public float Simple() => CpuMathUtils.Sum(Enumerable.Range(0, 1024).Select(Convert.ToSingle).ToArray());
27-
}
28-
2926
public class BenchmarksTest
3027
{
31-
private const string SkipTheDebug =
32-
#if DEBUG
33-
"BenchmarkDotNet does not allow running the benchmarks in Debug, so this test is disabled for DEBUG";
34-
#else
35-
"";
36-
#endif
37-
3828
public BenchmarksTest(ITestOutputHelper output) => Output = output;
3929

4030
private ITestOutputHelper Output { get; }
4131

32+
public static bool CanExecute =>
4233
#if DEBUG
43-
[Fact(Skip = SkipTheDebug)]
34+
false; // BenchmarkDotNet does not allow running the benchmarks in Debug, so this test is disabled for DEBUG
35+
#elif NET462
36+
false; // We are currently not running Benchmarks for FullFramework
4437
#else
45-
[ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))]
38+
Environment.Is64BitProcess; // we don't support 32 bit yet
4639
#endif
47-
public void BenchmarksProjectIsNotBroken()
40+
41+
public static TheoryData<Type> GetBenchmarks()
42+
{
43+
TheoryData<Type> benchmarks = new TheoryData<Type>();
44+
Assembly asm = typeof(StochasticDualCoordinateAscentClassifierBench).Assembly;
45+
46+
var types = from type in asm.GetTypes()
47+
where Attribute.IsDefined(type, typeof(CIBenchmark))
48+
select type;
49+
50+
foreach (Type type in types)
51+
{
52+
benchmarks.Add(type);
53+
}
54+
return benchmarks;
55+
}
56+
57+
[ConditionalTheory(typeof(BenchmarksTest), nameof(CanExecute))]
58+
[MemberData(nameof(GetBenchmarks))]
59+
public void BenchmarksProjectIsNotBroken(Type type)
4860
{
49-
var summary = BenchmarkRunner.Run<BenchmarkTouchingNativeDependency>(new TestConfig().With(new OutputLogger(Output)));
61+
var summary = BenchmarkRunner.Run(type, new TestConfig().With(new OutputLogger(Output)));
5062

5163
Assert.False(summary.HasCriticalValidationErrors, "The \"Summary\" should have NOT \"HasCriticalValidationErrors\"");
5264

‎test/Microsoft.ML.Benchmarks/CacheDataViewBench.cs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,12 @@
44

55
using System;
66
using BenchmarkDotNet.Attributes;
7+
using Microsoft.ML.Benchmarks.Harness;
78
using Microsoft.ML.Data;
89

910
namespace Microsoft.ML.Benchmarks
1011
{
12+
[CIBenchmark]
1113
public class CacheDataViewBench
1214
{
1315
private const int Length = 100000;

‎test/Microsoft.ML.Benchmarks/HashBench.cs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,13 @@
55
using System;
66
using System.Linq;
77
using BenchmarkDotNet.Attributes;
8+
using Microsoft.ML.Benchmarks.Harness;
89
using Microsoft.ML.Data;
910
using Microsoft.ML.Transforms.Conversions;
1011

1112
namespace Microsoft.ML.Benchmarks
1213
{
14+
[CIBenchmark]
1315
public class HashBench
1416
{
1517
private sealed class RowImpl : Row

‎test/Microsoft.ML.Benchmarks/Helpers/CIBenchmark.cs

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
// See the LICENSE file in the project root for more information.
4+
5+
using System;
6+
7+
namespace Microsoft.ML.Benchmarks.Harness
8+
{
9+
/// <summary>
10+
/// This attribute is used to identify the benchmarks
11+
/// which we want to run on the CI.
12+
/// </summary>
13+
[AttributeUsage(AttributeTargets.Class, AllowMultiple = false)]
14+
public class CIBenchmark : Attribute
15+
{
16+
}
17+
}

‎test/Microsoft.ML.Benchmarks/KMeansAndLogisticRegressionBench.cs

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,14 +3,17 @@
33
// See the LICENSE file in the project root for more information.
44

55
using BenchmarkDotNet.Attributes;
6+
using Microsoft.ML.Benchmarks.Harness;
67
using Microsoft.ML.Data;
78
using Microsoft.ML.Internal.Calibration;
9+
using Microsoft.ML.TestFramework;
810

911
namespace Microsoft.ML.Benchmarks
1012
{
13+
[CIBenchmark]
1114
public class KMeansAndLogisticRegressionBench
1215
{
13-
private readonly string _dataPath = Program.GetInvariantCultureDataPath("adult.tiny.with-schema.txt");
16+
private readonly string _dataPath = BaseTestClass.GetDataPath("adult.tiny.with-schema.txt");
1417

1518
[Benchmark]
1619
public ParameterMixingCalibratedPredictor TrainKMeansAndLR()

‎test/Microsoft.ML.Benchmarks/Microsoft.ML.Benchmarks.csproj

Lines changed: 1 addition & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
<Project Sdk="Microsoft.NET.Sdk">
2-
<Import Project="$(RepoRoot)\build\ExternalBenchmarkDataFiles.props" />
32
<PropertyGroup>
43
<OutputType>Exe</OutputType>
54
<LangVersion>7.2</LangVersion>
@@ -24,28 +23,6 @@
2423
<ItemGroup>
2524
<NativeAssemblyReference Include="CpuMathNative" />
2625
<NativeAssemblyReference Include="FastTreeNative" />
27-
</ItemGroup>
28-
<ItemGroup>
29-
<Folder Include="Input\" />
30-
<Content Include="..\data\iris.txt" Link="Input\iris.txt">
31-
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
32-
</Content>
33-
<None Include="..\data\adult.tiny.with-schema.txt" Link="Input\adult.tiny.with-schema.txt">
34-
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
35-
</None>
36-
<None Include="..\data\wikipedia-detox-250-line-data.tsv" Link="Input\wikipedia-detox-250-line-data.tsv">
37-
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
38-
</None>
39-
<None Include="..\data\breast-cancer.txt" Link="Input\breast-cancer.txt">
40-
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
41-
</None>
42-
43-
<BenchmarkFile Update="@(BenchmarkFile)">
44-
<Link>external\%(Identity)</Link>
45-
</BenchmarkFile>
46-
47-
<None Condition="Exists('@(BenchmarkFile->'..\data\external\%(Identity)')')" Include="@(BenchmarkFile->'..\data\external\%(Identity)')">
48-
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
49-
</None>
26+
<NativeAssemblyReference Include="MklImports" />
5027
</ItemGroup>
5128
</Project>

‎test/Microsoft.ML.Benchmarks/Numeric/Ranking.cs

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
using Microsoft.ML.Data;
77
using Microsoft.ML.LightGBM;
88
using Microsoft.ML.RunTests;
9+
using Microsoft.ML.TestFramework;
910
using Microsoft.ML.Trainers.FastTree;
1011
using Microsoft.ML.Transforms.Conversions;
1112
using System.IO;
@@ -21,8 +22,8 @@ public class RankingTrain
2122
[GlobalSetup]
2223
public void SetupTrainingSpeedTests()
2324
{
24-
_mslrWeb10k_Validate = Path.GetFullPath(TestDatasets.MSLRWeb.validFilename);
25-
_mslrWeb10k_Train = Path.GetFullPath(TestDatasets.MSLRWeb.trainFilename);
25+
_mslrWeb10k_Validate = BaseTestClass.GetDataPath(TestDatasets.MSLRWeb.validFilename);
26+
_mslrWeb10k_Train = BaseTestClass.GetDataPath(TestDatasets.MSLRWeb.trainFilename);
2627

2728
if (!File.Exists(_mslrWeb10k_Validate))
2829
throw new FileNotFoundException(string.Format(Errors.DatasetNotFound, _mslrWeb10k_Validate));
@@ -71,9 +72,9 @@ public class RankingTest
7172
[GlobalSetup]
7273
public void SetupScoringSpeedTests()
7374
{
74-
_mslrWeb10k_Test = Path.GetFullPath(TestDatasets.MSLRWeb.testFilename);
75-
_mslrWeb10k_Validate = Path.GetFullPath(TestDatasets.MSLRWeb.validFilename);
76-
_mslrWeb10k_Train = Path.GetFullPath(TestDatasets.MSLRWeb.trainFilename);
75+
_mslrWeb10k_Test = BaseTestClass.GetDataPath(TestDatasets.MSLRWeb.testFilename);
76+
_mslrWeb10k_Validate = BaseTestClass.GetDataPath(TestDatasets.MSLRWeb.validFilename);
77+
_mslrWeb10k_Train = BaseTestClass.GetDataPath(TestDatasets.MSLRWeb.trainFilename);
7778

7879
if (!File.Exists(_mslrWeb10k_Test))
7980
throw new FileNotFoundException(string.Format(Errors.DatasetNotFound, _mslrWeb10k_Test));

‎test/Microsoft.ML.Benchmarks/PredictionEngineBench.cs

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,16 @@
33
// See the LICENSE file in the project root for more information.
44

55
using BenchmarkDotNet.Attributes;
6+
using Microsoft.ML.Benchmarks.Harness;
67
using Microsoft.ML.Data;
8+
using Microsoft.ML.TestFramework;
79
using Microsoft.ML.Trainers;
810
using Microsoft.ML.Transforms;
911
using Microsoft.ML.Transforms.Text;
1012

1113
namespace Microsoft.ML.Benchmarks
1214
{
15+
[CIBenchmark]
1316
public class PredictionEngineBench
1417
{
1518
private IrisData _irisExample;
@@ -32,7 +35,7 @@ public void SetupIrisPipeline()
3235
PetalWidth = 5.1f,
3336
};
3437

35-
string _irisDataPath = Program.GetInvariantCultureDataPath("iris.txt");
38+
string _irisDataPath = BaseTestClass.GetDataPath("iris.txt");
3639

3740
var env = new MLContext(seed: 1, conc: 1);
3841
var reader = new TextLoader(env,
@@ -65,7 +68,7 @@ public void SetupSentimentPipeline()
6568
SentimentText = "Not a big fan of this."
6669
};
6770

68-
string _sentimentDataPath = Program.GetInvariantCultureDataPath("wikipedia-detox-250-line-data.tsv");
71+
string _sentimentDataPath = BaseTestClass.GetDataPath("wikipedia-detox-250-line-data.tsv");
6972

7073
var env = new MLContext(seed: 1, conc: 1);
7174
var reader = new TextLoader(env, columns: new[]
@@ -94,7 +97,7 @@ public void SetupBreastCancerPipeline()
9497
Features = new[] { 5f, 1f, 1f, 1f, 2f, 1f, 3f, 1f, 1f }
9598
};
9699

97-
string _breastCancerDataPath = Program.GetInvariantCultureDataPath("breast-cancer.txt");
100+
string _breastCancerDataPath = BaseTestClass.GetDataPath("breast-cancer.txt");
98101

99102
var env = new MLContext(seed: 1, conc: 1);
100103
var reader = new TextLoader(env, columns: new[]

‎test/Microsoft.ML.Benchmarks/Program.cs

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
// See the LICENSE file in the project root for more information.
44

55
using System.Globalization;
6-
using System.IO;
76
using System.Threading;
87
using BenchmarkDotNet.Running;
98

@@ -15,17 +14,14 @@ class Program
1514
/// execute dotnet run -c Release and choose the benchmarks you want to run
1615
/// </summary>
1716
/// <param name="args"></param>
18-
static void Main(string[] args)
19-
=> BenchmarkSwitcher
20-
.FromAssembly(typeof(Program).Assembly)
21-
.Run(args, new RecommendedConfig());
22-
23-
internal static string GetInvariantCultureDataPath(string name)
17+
static void Main(string[] args)
2418
{
2519
// enforce Neutral Language as "en-us" because the input data files use dot as decimal separator (and it fails for cultures with ",")
2620
Thread.CurrentThread.CurrentCulture = CultureInfo.InvariantCulture;
2721

28-
return Path.Combine(Path.GetDirectoryName(typeof(Program).Assembly.Location), "Input", name);
22+
BenchmarkSwitcher
23+
.FromAssembly(typeof(Program).Assembly)
24+
.Run(args, new RecommendedConfig());
2925
}
3026
}
3127
}

‎test/Microsoft.ML.Benchmarks/RffTransform.cs

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,20 +4,23 @@
44

55
using System.IO;
66
using BenchmarkDotNet.Attributes;
7+
using Microsoft.ML.Benchmarks.Harness;
78
using Microsoft.ML.Data;
89
using Microsoft.ML.RunTests;
10+
using Microsoft.ML.TestFramework;
911
using Microsoft.ML.Transforms.Conversions;
1012

1113
namespace Microsoft.ML.Benchmarks
1214
{
15+
[CIBenchmark]
1316
public class RffTransformTrain
1417
{
1518
private string _dataPath_Digits;
1619

1720
[GlobalSetup]
1821
public void SetupTrainingSpeedTests()
1922
{
20-
_dataPath_Digits = Path.GetFullPath(TestDatasets.Digits.trainFilename);
23+
_dataPath_Digits = BaseTestClass.GetDataPath(TestDatasets.Digits.trainFilename);
2124

2225
if (!File.Exists(_dataPath_Digits))
2326
throw new FileNotFoundException(string.Format(Errors.DatasetNotFound, _dataPath_Digits));

‎test/Microsoft.ML.Benchmarks/StochasticDualCoordinateAscentClassifierBench.cs

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,18 +6,21 @@
66
using System.Globalization;
77
using BenchmarkDotNet.Attributes;
88
using BenchmarkDotNet.Engines;
9+
using Microsoft.ML.Benchmarks.Harness;
910
using Microsoft.ML.Data;
1011
using Microsoft.ML.Learners;
12+
using Microsoft.ML.TestFramework;
1113
using Microsoft.ML.Trainers;
1214
using Microsoft.ML.Transforms;
1315
using Microsoft.ML.Transforms.Text;
1416

1517
namespace Microsoft.ML.Benchmarks
1618
{
19+
[CIBenchmark]
1720
public class StochasticDualCoordinateAscentClassifierBench : WithExtraMetrics
1821
{
19-
private readonly string _dataPath = Program.GetInvariantCultureDataPath("iris.txt");
20-
private readonly string _sentimentDataPath = Program.GetInvariantCultureDataPath("wikipedia-detox-250-line-data.tsv");
22+
private readonly string _dataPath = BaseTestClass.GetDataPath("iris.txt");
23+
private readonly string _sentimentDataPath = BaseTestClass.GetDataPath("wikipedia-detox-250-line-data.tsv");
2124
private readonly Consumer _consumer = new Consumer(); // BenchmarkDotNet utility type used to prevent dead code elimination
2225

2326
private readonly MLContext _env = new MLContext(seed: 1);

0 commit comments

Comments
 (0)