|
| 1 | +// Licensed to the .NET Foundation under one or more agreements. |
| 2 | +// The .NET Foundation licenses this file to you under the MIT license. |
| 3 | +// See the LICENSE file in the project root for more information. |
| 4 | + |
| 5 | +using BenchmarkDotNet.Attributes; |
| 6 | +using Microsoft.ML.Runtime; |
| 7 | +using Microsoft.ML.Runtime.Data; |
| 8 | +using Microsoft.ML.Runtime.RunTests; |
| 9 | +using Microsoft.ML.Runtime.Tools; |
| 10 | +using System.IO; |
| 11 | +using System.Text; |
| 12 | + |
| 13 | +namespace Microsoft.ML.Benchmarks |
| 14 | +{ |
| 15 | + // Adding this class to not print anything to the console. |
| 16 | + // This is required for the current version of BenchmarkDotNet |
| 17 | + internal class EmptyWriter : TextWriter |
| 18 | + { |
| 19 | + internal static readonly EmptyWriter Instance = new EmptyWriter(); |
| 20 | + public override Encoding Encoding => null; |
| 21 | + } |
| 22 | + |
| 23 | + public class BigramAndTrigramBenchmark |
| 24 | + { |
| 25 | + private string _dataPath_Wiki; |
| 26 | + private string _modelPath_Wiki; |
| 27 | + |
| 28 | + [GlobalSetup(Targets = new string[] { |
| 29 | + nameof(CV_Multiclass_WikiDetox_BigramsAndTrichar_OVAAveragedPerceptron), |
| 30 | + nameof(CV_Multiclass_WikiDetox_BigramsAndTrichar_LightGBMMulticlass) })] |
| 31 | + public void SetupTrainingSpeedTests() |
| 32 | + { |
| 33 | + _dataPath_Wiki = Path.GetFullPath(TestDatasets.WikiDetox.trainFilename); |
| 34 | + |
| 35 | + if (!File.Exists(_dataPath_Wiki)) |
| 36 | + { |
| 37 | + throw new FileNotFoundException($"Could not find {_dataPath_Wiki} Please ensure you have run 'build.cmd -- /t:DownloadExternalTestFiles /p:IncludeBenchmarkData=true' from the root"); |
| 38 | + } |
| 39 | + } |
| 40 | + |
| 41 | + [GlobalSetup(Target = nameof(Test_Multiclass_WikiDetox_BigramsAndTrichar_OVAAveragedPerceptron))] |
| 42 | + public void SetupScoringSpeedTests() |
| 43 | + { |
| 44 | + SetupTrainingSpeedTests(); |
| 45 | + _modelPath_Wiki = Path.Combine(Directory.GetCurrentDirectory(), @"WikiModel.zip"); |
| 46 | + string cmd = @"CV k=5 data=" + _dataPath_Wiki + " loader=TextLoader{quote=- sparse=- col=Label:R4:0 col=rev_id:TX:1 col=comment:TX:2 col=logged_in:BL:4 col=ns:TX:5 col=sample:TX:6 col=split:TX:7 col=year:R4:3 header=+} xf=Convert{col=logged_in type=R4} xf=CategoricalTransform{col=ns} xf=TextTransform{col=FeaturesText:comment wordExtractor=NGramExtractorTransform{ngram=2}} xf=Concat{col=Features:FeaturesText,logged_in,ns} tr=OVA{p=AveragedPerceptron{iter=10}} out={" + _modelPath_Wiki + "}"; |
| 47 | + using (var tlc = new TlcEnvironment(verbose: false, sensitivity: MessageSensitivity.None, outWriter: EmptyWriter.Instance)) |
| 48 | + { |
| 49 | + Maml.MainCore(tlc, cmd, alwaysPrintStacktrace: false); |
| 50 | + } |
| 51 | + } |
| 52 | + |
| 53 | + [Benchmark] |
| 54 | + public void CV_Multiclass_WikiDetox_BigramsAndTrichar_OVAAveragedPerceptron() |
| 55 | + { |
| 56 | + string cmd = @"CV k=5 data=" + _dataPath_Wiki + " loader=TextLoader{quote=- sparse=- col=Label:R4:0 col=rev_id:TX:1 col=comment:TX:2 col=logged_in:BL:4 col=ns:TX:5 col=sample:TX:6 col=split:TX:7 col=year:R4:3 header=+} xf=Convert{col=logged_in type=R4} xf=CategoricalTransform{col=ns} xf=TextTransform{col=FeaturesText:comment wordExtractor=NGramExtractorTransform{ngram=2}} xf=Concat{col=Features:FeaturesText,logged_in,ns} tr=OVA{p=AveragedPerceptron{iter=10}}"; |
| 57 | + using (var tlc = new TlcEnvironment(verbose: false, sensitivity: MessageSensitivity.None, outWriter: EmptyWriter.Instance)) |
| 58 | + { |
| 59 | + Maml.MainCore(tlc, cmd, alwaysPrintStacktrace: false); |
| 60 | + } |
| 61 | + } |
| 62 | + |
| 63 | + [Benchmark] |
| 64 | + public void CV_Multiclass_WikiDetox_BigramsAndTrichar_LightGBMMulticlass() |
| 65 | + { |
| 66 | + string cmd = @"CV k=5 data=" + _dataPath_Wiki + " loader=TextLoader{quote=- sparse=- col=Label:R4:0 col=rev_id:TX:1 col=comment:TX:2 col=logged_in:BL:4 col=ns:TX:5 col=sample:TX:6 col=split:TX:7 col=year:R4:3 header=+} xf=Convert{col=logged_in type=R4} xf=CategoricalTransform{col=ns} xf=TextTransform{col=FeaturesText:comment wordExtractor=NGramExtractorTransform{ngram=2}} xf=Concat{col=Features:FeaturesText,logged_in,ns} tr=LightGBMMulticlass{}"; |
| 67 | + using (var tlc = new TlcEnvironment(verbose: false, sensitivity: MessageSensitivity.None, outWriter: EmptyWriter.Instance)) |
| 68 | + { |
| 69 | + Maml.MainCore(tlc, cmd, alwaysPrintStacktrace: false); |
| 70 | + } |
| 71 | + } |
| 72 | + |
| 73 | + [Benchmark] |
| 74 | + public void Test_Multiclass_WikiDetox_BigramsAndTrichar_OVAAveragedPerceptron() |
| 75 | + { |
| 76 | + // This benchmark is profiling bulk scoring speed and not training speed. |
| 77 | + string modelpath = Path.Combine(Directory.GetCurrentDirectory(), @"WikiModel.fold000.zip"); |
| 78 | + string cmd = @"Test data=" + _dataPath_Wiki + " in=" + modelpath; |
| 79 | + using (var tlc = new TlcEnvironment(verbose: false, sensitivity: MessageSensitivity.None, outWriter: EmptyWriter.Instance)) |
| 80 | + { |
| 81 | + Maml.MainCore(tlc, cmd, alwaysPrintStacktrace: false); |
| 82 | + } |
| 83 | + } |
| 84 | + } |
| 85 | +} |
0 commit comments