Skip to content

Commit b88cc09

Browse files
Anipikjustinormont
Anipik
authored and
justinormont
committed
Different config files for train and predict benchmarks (#954)
Fixes #982 * different Config files for train and test * solves problem of long running time * train benchmarks contain only one iteration as it gives more idea on how the users will use. (with no warmup iteration) * predict config is the original version
1 parent 1f5f696 commit b88cc09

File tree

5 files changed

+139
-80
lines changed

5 files changed

+139
-80
lines changed

‎test/Microsoft.ML.Benchmarks/Numeric/Ranking.cs

Lines changed: 40 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -11,16 +11,13 @@
1111

1212
namespace Microsoft.ML.Benchmarks
1313
{
14-
public class Ranking
14+
[Config(typeof(TrainConfig))]
15+
public class RankingTrain
1516
{
1617
private string _mslrWeb10k_Validate;
1718
private string _mslrWeb10k_Train;
18-
private string _mslrWeb10k_Test;
19-
private string _modelPath_MSLR;
2019

21-
[GlobalSetup(Targets = new string[] {
22-
nameof(TrainTest_Ranking_MSLRWeb10K_RawNumericFeatures_FastTreeRanking),
23-
nameof(TrainTest_Ranking_MSLRWeb10K_RawNumericFeatures_LightGBMRanking) })]
20+
[GlobalSetup]
2421
public void SetupTrainingSpeedTests()
2522
{
2623
_mslrWeb10k_Validate = Path.GetFullPath(TestDatasets.MSLRWeb.validFilename);
@@ -33,24 +30,15 @@ public void SetupTrainingSpeedTests()
3330
throw new FileNotFoundException(string.Format(Helpers.DatasetNotFound, _mslrWeb10k_Train));
3431
}
3532

36-
[GlobalSetup(Target = nameof(Test_Ranking_MSLRWeb10K_RawNumericFeatures_FastTreeRanking))]
37-
public void SetupScoringSpeedTests()
33+
[Benchmark]
34+
public void TrainTest_Ranking_MSLRWeb10K_RawNumericFeatures_FastTreeRanking()
3835
{
39-
_mslrWeb10k_Test = Path.GetFullPath(TestDatasets.MSLRWeb.testFilename);
40-
if (!File.Exists(_mslrWeb10k_Test))
41-
throw new FileNotFoundException(string.Format(Helpers.DatasetNotFound, _mslrWeb10k_Test));
42-
43-
SetupTrainingSpeedTests();
44-
_modelPath_MSLR = Path.Combine(Directory.GetCurrentDirectory(), @"FastTreeRankingModel.zip");
45-
4636
string cmd = @"TrainTest test=" + _mslrWeb10k_Validate +
4737
" eval=RankingEvaluator{t=10}" +
4838
" data=" + _mslrWeb10k_Train +
4939
" loader=TextLoader{col=Label:R4:0 col=GroupId:TX:1 col=Features:R4:2-138}" +
50-
" xf=HashTransform{col=GroupId}" +
51-
" xf=NAHandleTransform{col=Features}" +
52-
" tr=FastTreeRanking{}" +
53-
" out={" + _modelPath_MSLR + "}";
40+
" xf=HashTransform{col=GroupId} xf=NAHandleTransform{col=Features}" +
41+
" tr=FastTreeRanking{}";
5442

5543
using (var environment = new ConsoleEnvironment(verbose: false, sensitivity: MessageSensitivity.None, outWriter: EmptyWriter.Instance))
5644
{
@@ -59,31 +47,57 @@ public void SetupScoringSpeedTests()
5947
}
6048

6149
[Benchmark]
62-
public void TrainTest_Ranking_MSLRWeb10K_RawNumericFeatures_FastTreeRanking()
50+
public void TrainTest_Ranking_MSLRWeb10K_RawNumericFeatures_LightGBMRanking()
6351
{
6452
string cmd = @"TrainTest test=" + _mslrWeb10k_Validate +
6553
" eval=RankingEvaluator{t=10}" +
6654
" data=" + _mslrWeb10k_Train +
6755
" loader=TextLoader{col=Label:R4:0 col=GroupId:TX:1 col=Features:R4:2-138}" +
68-
" xf=HashTransform{col=GroupId} xf=NAHandleTransform{col=Features}" +
69-
" tr=FastTreeRanking{}";
56+
" xf=HashTransform{col=GroupId}" +
57+
" xf=NAHandleTransform{col=Features}" +
58+
" tr=LightGBMRanking{}";
7059

7160
using (var environment = new ConsoleEnvironment(verbose: false, sensitivity: MessageSensitivity.None, outWriter: EmptyWriter.Instance))
7261
{
7362
Maml.MainCore(environment, cmd, alwaysPrintStacktrace: false);
7463
}
7564
}
65+
}
7666

77-
[Benchmark]
78-
public void TrainTest_Ranking_MSLRWeb10K_RawNumericFeatures_LightGBMRanking()
67+
[Config(typeof(PredictConfig))]
68+
public class RankingTest
69+
{
70+
private string _mslrWeb10k_Validate;
71+
private string _mslrWeb10k_Train;
72+
private string _mslrWeb10k_Test;
73+
private string _modelPath_MSLR;
74+
75+
[GlobalSetup]
76+
public void SetupScoringSpeedTests()
7977
{
78+
_mslrWeb10k_Test = Path.GetFullPath(TestDatasets.MSLRWeb.testFilename);
79+
_mslrWeb10k_Validate = Path.GetFullPath(TestDatasets.MSLRWeb.validFilename);
80+
_mslrWeb10k_Train = Path.GetFullPath(TestDatasets.MSLRWeb.trainFilename);
81+
82+
if (!File.Exists(_mslrWeb10k_Test))
83+
throw new FileNotFoundException(string.Format(Helpers.DatasetNotFound, _mslrWeb10k_Test));
84+
85+
if (!File.Exists(_mslrWeb10k_Validate))
86+
throw new FileNotFoundException(string.Format(Helpers.DatasetNotFound, _mslrWeb10k_Validate));
87+
88+
if (!File.Exists(_mslrWeb10k_Train))
89+
throw new FileNotFoundException(string.Format(Helpers.DatasetNotFound, _mslrWeb10k_Train));
90+
91+
_modelPath_MSLR = Path.Combine(Directory.GetCurrentDirectory(), @"FastTreeRankingModel.zip");
92+
8093
string cmd = @"TrainTest test=" + _mslrWeb10k_Validate +
8194
" eval=RankingEvaluator{t=10}" +
8295
" data=" + _mslrWeb10k_Train +
8396
" loader=TextLoader{col=Label:R4:0 col=GroupId:TX:1 col=Features:R4:2-138}" +
8497
" xf=HashTransform{col=GroupId}" +
8598
" xf=NAHandleTransform{col=Features}" +
86-
" tr=LightGBMRanking{}";
99+
" tr=FastTreeRanking{}" +
100+
" out={" + _modelPath_MSLR + "}";
87101

88102
using (var environment = new ConsoleEnvironment(verbose: false, sensitivity: MessageSensitivity.None, outWriter: EmptyWriter.Instance))
89103
{
@@ -95,7 +109,7 @@ public void TrainTest_Ranking_MSLRWeb10K_RawNumericFeatures_LightGBMRanking()
95109
public void Test_Ranking_MSLRWeb10K_RawNumericFeatures_FastTreeRanking()
96110
{
97111
// This benchmark is profiling bulk scoring speed and not training speed.
98-
string cmd = @"Test data=" + _mslrWeb10k_Test + " in="+ _modelPath_MSLR;
112+
string cmd = @"Test data=" + _mslrWeb10k_Test + " in=" + _modelPath_MSLR;
99113
using (var environment = new ConsoleEnvironment(verbose: false, sensitivity: MessageSensitivity.None, outWriter: EmptyWriter.Instance))
100114
{
101115
Maml.MainCore(environment, cmd, alwaysPrintStacktrace: false);

‎test/Microsoft.ML.Benchmarks/PredictConfig.cs

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
// See the LICENSE file in the project root for more information.
4+
5+
using BenchmarkDotNet.Configs;
6+
using BenchmarkDotNet.Diagnosers;
7+
using BenchmarkDotNet.Jobs;
8+
9+
namespace Microsoft.ML.Benchmarks
10+
{
11+
internal class PredictConfig : ManualConfig
12+
{
13+
public PredictConfig()
14+
{
15+
Add(DefaultConfig.Instance
16+
.With(Job.Default
17+
.WithWarmupCount(1)
18+
.WithMaxIterationCount(20)
19+
.With(Program.CreateToolchain()))
20+
.With(new ExtraMetricColumn())
21+
.With(MemoryDiagnoser.Default));
22+
}
23+
}
24+
}

‎test/Microsoft.ML.Benchmarks/Program.cs

Lines changed: 2 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,6 @@
22
// The .NET Foundation licenses this file to you under the MIT license.
33
// See the LICENSE file in the project root for more information.
44

5-
using BenchmarkDotNet.Configs;
6-
using BenchmarkDotNet.Diagnosers;
7-
using BenchmarkDotNet.Jobs;
85
using BenchmarkDotNet.Running;
96
using BenchmarkDotNet.Toolchains;
107
using BenchmarkDotNet.Toolchains.CsProj;
@@ -25,21 +22,12 @@ class Program
2522
static void Main(string[] args)
2623
=> BenchmarkSwitcher
2724
.FromAssembly(typeof(Program).Assembly)
28-
.Run(args, CreateCustomConfig());
29-
30-
private static IConfig CreateCustomConfig()
31-
=> DefaultConfig.Instance
32-
.With(Job.Default
33-
.WithWarmupCount(1) // for our time consuming benchmarks 1 warmup iteration is enough
34-
.WithMaxIterationCount(20)
35-
.With(CreateToolchain()))
36-
.With(new ExtraMetricColumn())
37-
.With(MemoryDiagnoser.Default);
25+
.Run(args);
3826

3927
/// <summary>
4028
/// we need our own toolchain because MSBuild by default does not copy recursive native dependencies to the output
4129
/// </summary>
42-
private static IToolchain CreateToolchain()
30+
internal static IToolchain CreateToolchain()
4331
{
4432
var csProj = CsProjCoreToolchain.Current.Value;
4533
var tfm = NetCoreAppSettings.Current.Value.TargetFrameworkMoniker;

‎test/Microsoft.ML.Benchmarks/Text/MultiClassClassification.cs

Lines changed: 48 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -11,16 +11,12 @@
1111

1212
namespace Microsoft.ML.Benchmarks
1313
{
14-
public class MultiClassClassification
14+
[Config(typeof(TrainConfig))]
15+
public class MultiClassClassificationTrain
1516
{
1617
private string _dataPath_Wiki;
17-
private string _modelPath_Wiki;
1818

19-
[GlobalSetup(Targets = new string[] {
20-
nameof(CV_Multiclass_WikiDetox_BigramsAndTrichar_OVAAveragedPerceptron),
21-
nameof(CV_Multiclass_WikiDetox_BigramsAndTrichar_LightGBMMulticlass),
22-
nameof(CV_Multiclass_WikiDetox_WordEmbeddings_OVAAveragedPerceptron),
23-
nameof(CV_Multiclass_WikiDetox_WordEmbeddings_SDCAMC)})]
19+
[GlobalSetup]
2420
public void SetupTrainingSpeedTests()
2521
{
2622
_dataPath_Wiki = Path.GetFullPath(TestDatasets.WikiDetox.trainFilename);
@@ -29,26 +25,6 @@ public void SetupTrainingSpeedTests()
2925
throw new FileNotFoundException(string.Format(Helpers.DatasetNotFound, _dataPath_Wiki));
3026
}
3127

32-
[GlobalSetup(Target = nameof(Test_Multiclass_WikiDetox_BigramsAndTrichar_OVAAveragedPerceptron))]
33-
public void SetupScoringSpeedTests()
34-
{
35-
SetupTrainingSpeedTests();
36-
_modelPath_Wiki = Path.Combine(Directory.GetCurrentDirectory(), @"WikiModel.zip");
37-
38-
string cmd = @"CV k=5 data=" + _dataPath_Wiki +
39-
" loader=TextLoader{quote=- sparse=- col=Label:R4:0 col=rev_id:TX:1 col=comment:TX:2 col=logged_in:BL:4 col=ns:TX:5 col=sample:TX:6 col=split:TX:7 col=year:R4:3 header=+} xf=Convert{col=logged_in type=R4}" +
40-
" xf=CategoricalTransform{col=ns}" +
41-
" xf=TextTransform{col=FeaturesText:comment wordExtractor=NGramExtractorTransform{ngram=2}}" +
42-
" xf=Concat{col=Features:FeaturesText,logged_in,ns}" +
43-
" tr=OVA{p=AveragedPerceptron{iter=10}}" +
44-
" out={" + _modelPath_Wiki + "}";
45-
46-
using (var environment = new ConsoleEnvironment(verbose: false, sensitivity: MessageSensitivity.None, outWriter: EmptyWriter.Instance))
47-
{
48-
Maml.MainCore(environment, cmd, alwaysPrintStacktrace: false);
49-
}
50-
}
51-
5228
[Benchmark]
5329
public void CV_Multiclass_WikiDetox_BigramsAndTrichar_OVAAveragedPerceptron()
5430
{
@@ -74,26 +50,15 @@ public void CV_Multiclass_WikiDetox_BigramsAndTrichar_LightGBMMulticlass()
7450
" xf=Convert{col=logged_in type=R4}" +
7551
" xf=CategoricalTransform{col=ns}" +
7652
" xf=TextTransform{col=FeaturesText:comment wordExtractor=NGramExtractorTransform{ngram=2}}" +
77-
" xf=Concat{col=Features:FeaturesText,logged_in,ns} tr=LightGBMMulticlass{}";
53+
" xf=Concat{col=Features:FeaturesText,logged_in,ns}" +
54+
" tr=LightGBMMulticlass{iter=10}";
7855

7956
using (var environment = new ConsoleEnvironment(verbose: false, sensitivity: MessageSensitivity.None, outWriter: EmptyWriter.Instance))
8057
{
8158
Maml.MainCore(environment, cmd, alwaysPrintStacktrace: false);
8259
}
8360
}
8461

85-
[Benchmark]
86-
public void Test_Multiclass_WikiDetox_BigramsAndTrichar_OVAAveragedPerceptron()
87-
{
88-
// This benchmark is profiling bulk scoring speed and not training speed.
89-
string modelpath = Path.Combine(Directory.GetCurrentDirectory(), @"WikiModel.fold000.zip");
90-
string cmd = @"Test data=" + _dataPath_Wiki + " in=" + modelpath;
91-
using (var environment = new ConsoleEnvironment(verbose: false, sensitivity: MessageSensitivity.None, outWriter: EmptyWriter.Instance))
92-
{
93-
Maml.MainCore(environment, cmd, alwaysPrintStacktrace: false);
94-
}
95-
}
96-
9762
[Benchmark]
9863
public void CV_Multiclass_WikiDetox_WordEmbeddings_OVAAveragedPerceptron()
9964
{
@@ -130,4 +95,47 @@ public void CV_Multiclass_WikiDetox_WordEmbeddings_SDCAMC()
13095
}
13196
}
13297
}
98+
99+
[Config(typeof(PredictConfig))]
100+
public class MultiClassClassificationTest
101+
{
102+
private string _dataPath_Wiki;
103+
private string _modelPath_Wiki;
104+
105+
[GlobalSetup]
106+
public void SetupScoringSpeedTests()
107+
{
108+
_dataPath_Wiki = Path.GetFullPath(TestDatasets.WikiDetox.trainFilename);
109+
110+
if (!File.Exists(_dataPath_Wiki))
111+
throw new FileNotFoundException(string.Format(Helpers.DatasetNotFound, _dataPath_Wiki));
112+
113+
_modelPath_Wiki = Path.Combine(Directory.GetCurrentDirectory(), @"WikiModel.zip");
114+
115+
string cmd = @"CV k=5 data=" + _dataPath_Wiki +
116+
" loader=TextLoader{quote=- sparse=- col=Label:R4:0 col=rev_id:TX:1 col=comment:TX:2 col=logged_in:BL:4 col=ns:TX:5 col=sample:TX:6 col=split:TX:7 col=year:R4:3 header=+} xf=Convert{col=logged_in type=R4}" +
117+
" xf=CategoricalTransform{col=ns}" +
118+
" xf=TextTransform{col=FeaturesText:comment wordExtractor=NGramExtractorTransform{ngram=2}}" +
119+
" xf=Concat{col=Features:FeaturesText,logged_in,ns}" +
120+
" tr=OVA{p=AveragedPerceptron{iter=10}}" +
121+
" out={" + _modelPath_Wiki + "}";
122+
123+
using (var environment = new ConsoleEnvironment(verbose: false, sensitivity: MessageSensitivity.None, outWriter: EmptyWriter.Instance))
124+
{
125+
Maml.MainCore(environment, cmd, alwaysPrintStacktrace: false);
126+
}
127+
}
128+
129+
[Benchmark]
130+
public void Test_Multiclass_WikiDetox_BigramsAndTrichar_OVAAveragedPerceptron()
131+
{
132+
// This benchmark is profiling bulk scoring speed and not training speed.
133+
string modelpath = Path.Combine(Directory.GetCurrentDirectory(), @"WikiModel.fold000.zip");
134+
string cmd = @"Test data=" + _dataPath_Wiki + " in=" + modelpath;
135+
using (var environment = new ConsoleEnvironment(verbose: false, sensitivity: MessageSensitivity.None, outWriter: EmptyWriter.Instance))
136+
{
137+
Maml.MainCore(environment, cmd, alwaysPrintStacktrace: false);
138+
}
139+
}
140+
}
133141
}

‎test/Microsoft.ML.Benchmarks/TrainConfig.cs

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
// See the LICENSE file in the project root for more information.
4+
5+
using BenchmarkDotNet.Configs;
6+
using BenchmarkDotNet.Diagnosers;
7+
using BenchmarkDotNet.Jobs;
8+
9+
namespace Microsoft.ML.Benchmarks
10+
{
11+
public class TrainConfig : ManualConfig
12+
{
13+
public TrainConfig()
14+
{
15+
Add(DefaultConfig.Instance
16+
.With(Job.Default
17+
.WithWarmupCount(0)
18+
.WithIterationCount(1)
19+
.WithLaunchCount(3) // BDN will start 3 dedicated processes, each of them will just run given benchmark once, without any warm up to mimic the real world.
20+
.With(Program.CreateToolchain()))
21+
.With(new ExtraMetricColumn())
22+
.With(MemoryDiagnoser.Default));
23+
}
24+
}
25+
}

0 commit comments

Comments
 (0)