Added numeric ranking Performance Tests

build.proj

@@ -7,6 +7,7 @@
 
   <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), Directory.Build.props))\Directory.Build.props" />
 
+  <Import Project="build/ExternalBenchmarkDataFiles.props" />
   <Import Project="$(ToolsDir)VersionTools.targets" Condition="Exists('$(ToolsDir)VersionTools.targets')" />
   <UsingTask TaskName="DownloadFilesFromUrl" AssemblyFile="$(ToolsDir)Microsoft.DotNet.Build.Tasks.dll"/>
   <PropertyGroup>
@@ -75,10 +76,19 @@
              Targets="Pack" />
   </Target>
 
-  <ItemGroup>  
-    <TestFile Condition="'$(IncludeBenchmarkData)' == 'true'" Include="$(MSBuildThisFileDirectory)/test/data/external/WikiDetoxAnnotated160kRows.tsv"
-      Url="https://aka.ms/tlc-resources/benchmarks/WikiDetoxAnnotated160kRows.tsv"
-      DestinationFile="$(MSBuildThisFileDirectory)test/data/external/WikiDetoxAnnotated160kRows.tsv" />
+  <ItemGroup>
+    <TestFile Include="$(MSBuildThisFileDirectory)/test/data/external/winequality-white.csv"
+      Url="https://archive.ics.uci.edu/ml/machine-learning-databases/wine-quality/winequality-white.csv"
+      DestinationFile="$(MSBuildThisFileDirectory)test/data/external/winequality-white.csv" />    
+  </ItemGroup>
+
+  <ItemGroup Condition="'$(IncludeBenchmarkData)' == 'true'">
+    <BenchmarkFile Update="@(BenchmarkFile)">
+      <Url>https://aka.ms/tlc-resources/benchmarks/%(Identity)</Url>
+      <DestinationFile>$(MSBuildThisFileDirectory)/test/data/external/%(Identity)</DestinationFile>
+    </BenchmarkFile>
+
+    <TestFile Include="@(BenchmarkFile->'$(MSBuildThisFileDirectory)/test/data/external/%(Identity)')" />
   </ItemGroup>
 
   <Target Name="DownloadExternalTestFiles" Inputs="@(TestFile)" Outputs="%(TestFile.DestinationFile)">

build/ExternalBenchmarkDataFiles.props

@@ -0,0 +1,8 @@
+<Project>
+  <ItemGroup>
+    <BenchmarkFile Include="MSLRWeb10KTest240kRows.tsv" />
+    <BenchmarkFile Include="MSLRWeb10KTrain720kRows.tsv" />
+    <BenchmarkFile Include="MSLRWeb10KValidate240kRows.tsv" />
+    <BenchmarkFile Include="WikiDetoxAnnotated160kRows.tsv" />
+  </ItemGroup>
+</Project>

test/Microsoft.ML.Benchmarks/Helpers.cs

@@ -0,0 +1,22 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.IO;
+using System.Text;
+
+namespace Microsoft.ML.Benchmarks
+{
+    internal class Helpers
+    {
+        public static string DatasetNotFound = "Could not find {0} Please ensure you have run 'build.cmd -- /t:DownloadExternalTestFiles /p:IncludeBenchmarkData=true' from the root";
+    }
+
+    // Adding this class to not print anything to the console.
+    // This is required for the current version of BenchmarkDotNet
+    internal class EmptyWriter : TextWriter
+    {
+        internal static readonly EmptyWriter Instance = new EmptyWriter();
+        public override Encoding Encoding => null;
+    }
+}

test/Microsoft.ML.Benchmarks/Microsoft.ML.Benchmarks.csproj

@@ -1,4 +1,5 @@
 <Project Sdk="Microsoft.NET.Sdk">
+  <Import Project="$(RepoRoot)\build\ExternalBenchmarkDataFiles.props" /> 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
     <LangVersion>7.2</LangVersion>
@@ -22,6 +23,7 @@
   </ItemGroup>
   <ItemGroup>
     <NativeAssemblyReference Include="CpuMathNative" />
+    <NativeAssemblyReference Include="FastTreeNative" />
   </ItemGroup>
   <ItemGroup>
     <Folder Include="Input\" />
@@ -34,9 +36,13 @@
     <None Include="..\data\wikipedia-detox-250-line-data.tsv" Link="Input\wikipedia-detox-250-line-data.tsv">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>
-    <None Condition="Exists('..\data\external\WikiDetoxAnnotated160kRows.tsv')"
-                  Include="..\data\external\WikiDetoxAnnotated160kRows.tsv"
-                  Link="external\WikiDetoxAnnotated160kRows.tsv">
+
+    <BenchmarkFile Update="@(BenchmarkFile)">
+      <Link>external\%(Identity)</Link>
+    </BenchmarkFile>
+
+    <None Condition="Exists('@(BenchmarkFile->'..\data\external\%(Identity)')')"
+          Include="@(BenchmarkFile->'..\data\external\%(Identity)')">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>
   </ItemGroup>

test/Microsoft.ML.Benchmarks/Numeric/Ranking.cs

@@ -0,0 +1,105 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using BenchmarkDotNet.Attributes;
+using Microsoft.ML.Runtime;
+using Microsoft.ML.Runtime.Data;
+using Microsoft.ML.Runtime.RunTests;
+using Microsoft.ML.Runtime.Tools;
+using System.IO;
+
+namespace Microsoft.ML.Benchmarks
+{
+    public class Ranking
+    {
+        private string _mslrWeb10k_Validate;
+        private string _mslrWeb10k_Train;
+        private string _mslrWeb10k_Test;
+        private string _modelPath_MSLR;
+
+        [GlobalSetup(Targets = new string[] {
+            nameof(TrainTest_Ranking_MSLRWeb10K_RawNumericFeatures_FastTreeRanking),
+            nameof(TrainTest_Ranking_MSLRWeb10K_RawNumericFeatures_LightGBMRanking) })]
+        public void SetupTrainingSpeedTests()
+        {
+            _mslrWeb10k_Validate = Path.GetFullPath(TestDatasets.MSLRWeb.validFilename);
+            _mslrWeb10k_Train = Path.GetFullPath(TestDatasets.MSLRWeb.trainFilename);
+
+            if (!File.Exists(_mslrWeb10k_Validate))
+                throw new FileNotFoundException(string.Format(Helpers.DatasetNotFound, _mslrWeb10k_Validate));
+
+            if (!File.Exists(_mslrWeb10k_Train))
+                throw new FileNotFoundException(string.Format(Helpers.DatasetNotFound, _mslrWeb10k_Train));
+        }
+
+        [GlobalSetup(Target = nameof(Test_Ranking_MSLRWeb10K_RawNumericFeatures_FastTreeRanking))]
+        public void SetupScoringSpeedTests()
+        {
+            _mslrWeb10k_Test = Path.GetFullPath(TestDatasets.MSLRWeb.testFilename);
+            if (!File.Exists(_mslrWeb10k_Test))
+                throw new FileNotFoundException(string.Format(Helpers.DatasetNotFound, _mslrWeb10k_Test));
+
+            SetupTrainingSpeedTests();
+            _modelPath_MSLR = Path.Combine(Directory.GetCurrentDirectory(), @"FastTreeRankingModel.zip");
+
+            string cmd = @"TrainTest test=" + _mslrWeb10k_Validate +
+                " eval=RankingEvaluator{t=10}" +
+                " data=" + _mslrWeb10k_Train +
+                " loader=TextLoader{col=Label:R4:0 col=GroupId:TX:1 col=Features:R4:2-138}" +
+                " xf=HashTransform{col=GroupId}" +
+                " xf=NAHandleTransform{col=Features}" +
+                " tr=FastTreeRanking{}" +
+                " out={" + _modelPath_MSLR + "}";
+
+            using (var environment = new ConsoleEnvironment(verbose: false, sensitivity: MessageSensitivity.None, outWriter: EmptyWriter.Instance))
+            {
+                Maml.MainCore(environment, cmd, alwaysPrintStacktrace: false);
+            }
+        }
+
+        [Benchmark]
+        public void TrainTest_Ranking_MSLRWeb10K_RawNumericFeatures_FastTreeRanking()
+        {
+            string cmd = @"TrainTest test=" + _mslrWeb10k_Validate +
+                " eval=RankingEvaluator{t=10}" +
+                " data=" + _mslrWeb10k_Train +
+                " loader=TextLoader{col=Label:R4:0 col=GroupId:TX:1 col=Features:R4:2-138}" +
+                " xf=HashTransform{col=GroupId} xf=NAHandleTransform{col=Features}" +
+                " tr=FastTreeRanking{}";
+
+            using (var environment = new ConsoleEnvironment(verbose: false, sensitivity: MessageSensitivity.None, outWriter: EmptyWriter.Instance))
+            {
+                Maml.MainCore(environment, cmd, alwaysPrintStacktrace: false);
+            }
+        }
+
+        [Benchmark]
+        public void TrainTest_Ranking_MSLRWeb10K_RawNumericFeatures_LightGBMRanking()
+        {
+            string cmd = @"TrainTest test=" + _mslrWeb10k_Validate +
+                " eval=RankingEvaluator{t=10}" +
+                " data=" + _mslrWeb10k_Train +
+                " loader=TextLoader{col=Label:R4:0 col=GroupId:TX:1 col=Features:R4:2-138}" +
+                " xf=HashTransform{col=GroupId}" +
+                " xf=NAHandleTransform{col=Features}" +
+                " tr=LightGBMRanking{}";
+
+            using (var environment = new ConsoleEnvironment(verbose: false, sensitivity: MessageSensitivity.None, outWriter: EmptyWriter.Instance))
+            {
+                Maml.MainCore(environment, cmd, alwaysPrintStacktrace: false);
+            }
+        }
+
+        [Benchmark]
+        public void Test_Ranking_MSLRWeb10K_RawNumericFeatures_FastTreeRanking()
+        {
+            // This benchmark is profiling bulk scoring speed and not training speed. 
+            string cmd = @"Test data=" + _mslrWeb10k_Test + " in="+ _modelPath_MSLR;
+            using (var environment = new ConsoleEnvironment(verbose: false, sensitivity: MessageSensitivity.None, outWriter: EmptyWriter.Instance))
+            {
+                Maml.MainCore(environment, cmd, alwaysPrintStacktrace: false);
+            }
+        }
+    }
+}

test/Microsoft.ML.Benchmarks/Text/MultiClassClassification.cs

@@ -8,18 +8,9 @@
 using Microsoft.ML.Runtime.RunTests;
 using Microsoft.ML.Runtime.Tools;
 using System.IO;
-using System.Text;
 
 namespace Microsoft.ML.Benchmarks
 {
-    // Adding this class to not print anything to the console.
-    // This is required for the current version of BenchmarkDotNet
-    internal class EmptyWriter : TextWriter
-    {
-        internal static readonly EmptyWriter Instance = new EmptyWriter();
-        public override Encoding Encoding => null;
-    }
-
     public class MultiClassClassification
     {
         private string _dataPath_Wiki;
@@ -35,40 +26,59 @@ public void SetupTrainingSpeedTests()
             _dataPath_Wiki = Path.GetFullPath(TestDatasets.WikiDetox.trainFilename);
 
             if (!File.Exists(_dataPath_Wiki))
-            {
-                throw new FileNotFoundException($"Could not find {_dataPath_Wiki} Please ensure you have run 'build.cmd -- /t:DownloadExternalTestFiles /p:IncludeBenchmarkData=true' from the root");
-            }
+                throw new FileNotFoundException(string.Format(Helpers.DatasetNotFound, _dataPath_Wiki));           
         }
 
         [GlobalSetup(Target = nameof(Test_Multiclass_WikiDetox_BigramsAndTrichar_OVAAveragedPerceptron))]
         public void SetupScoringSpeedTests()
         {
             SetupTrainingSpeedTests();
             _modelPath_Wiki = Path.Combine(Directory.GetCurrentDirectory(), @"WikiModel.zip");
-            string cmd = @"CV k=5 data=" + _dataPath_Wiki + " loader=TextLoader{quote=- sparse=- col=Label:R4:0 col=rev_id:TX:1 col=comment:TX:2 col=logged_in:BL:4 col=ns:TX:5 col=sample:TX:6 col=split:TX:7 col=year:R4:3 header=+} xf=Convert{col=logged_in type=R4} xf=CategoricalTransform{col=ns} xf=TextTransform{col=FeaturesText:comment wordExtractor=NGramExtractorTransform{ngram=2}} xf=Concat{col=Features:FeaturesText,logged_in,ns} tr=OVA{p=AveragedPerceptron{iter=10}} out={" + _modelPath_Wiki + "}";
-            using (var tlc = new ConsoleEnvironment(verbose: false, sensitivity: MessageSensitivity.None, outWriter: EmptyWriter.Instance))
+
+            string cmd = @"CV k=5 data=" + _dataPath_Wiki +
+                " loader=TextLoader{quote=- sparse=- col=Label:R4:0 col=rev_id:TX:1 col=comment:TX:2 col=logged_in:BL:4 col=ns:TX:5 col=sample:TX:6 col=split:TX:7 col=year:R4:3 header=+} xf=Convert{col=logged_in type=R4}" +
+                " xf=CategoricalTransform{col=ns}" +
+                " xf=TextTransform{col=FeaturesText:comment wordExtractor=NGramExtractorTransform{ngram=2}}" +
+                " xf=Concat{col=Features:FeaturesText,logged_in,ns}" +
+                " tr=OVA{p=AveragedPerceptron{iter=10}}" +
+                " out={" + _modelPath_Wiki + "}";
+
+            using (var environment = new ConsoleEnvironment(verbose: false, sensitivity: MessageSensitivity.None, outWriter: EmptyWriter.Instance))
             {
-                Maml.MainCore(tlc, cmd, alwaysPrintStacktrace: false);
+                Maml.MainCore(environment, cmd, alwaysPrintStacktrace: false);
             }
         }
 
         [Benchmark]
         public void CV_Multiclass_WikiDetox_BigramsAndTrichar_OVAAveragedPerceptron()
         {
-            string cmd = @"CV k=5 data=" + _dataPath_Wiki + " loader=TextLoader{quote=- sparse=- col=Label:R4:0 col=rev_id:TX:1 col=comment:TX:2 col=logged_in:BL:4 col=ns:TX:5 col=sample:TX:6 col=split:TX:7 col=year:R4:3 header=+} xf=Convert{col=logged_in type=R4} xf=CategoricalTransform{col=ns} xf=TextTransform{col=FeaturesText:comment wordExtractor=NGramExtractorTransform{ngram=2}} xf=Concat{col=Features:FeaturesText,logged_in,ns} tr=OVA{p=AveragedPerceptron{iter=10}}";
-            using (var tlc = new ConsoleEnvironment(verbose: false, sensitivity: MessageSensitivity.None, outWriter: EmptyWriter.Instance))
+            string cmd = @"CV k=5 data=" + _dataPath_Wiki + 
+                        " loader=TextLoader{quote=- sparse=- col=Label:R4:0 col=rev_id:TX:1 col=comment:TX:2 col=logged_in:BL:4 col=ns:TX:5 col=sample:TX:6 col=split:TX:7 col=year:R4:3 header=+}" +
+                        " xf=Convert{col=logged_in type=R4}" +
+                        " xf=CategoricalTransform{col=ns}" +
+                        " xf=TextTransform{col=FeaturesText:comment wordExtractor=NGramExtractorTransform{ngram=2}}" +
+                        " xf=Concat{col=Features:FeaturesText,logged_in,ns}" +
+                        " tr=OVA{p=AveragedPerceptron{iter=10}}";
+
+            using (var environment = new ConsoleEnvironment(verbose: false, sensitivity: MessageSensitivity.None, outWriter: EmptyWriter.Instance))
             {
-                Maml.MainCore(tlc, cmd, alwaysPrintStacktrace: false);
+                Maml.MainCore(environment, cmd, alwaysPrintStacktrace: false);
             }
         }
 
         [Benchmark]
         public void CV_Multiclass_WikiDetox_BigramsAndTrichar_LightGBMMulticlass()
         {
-            string cmd = @"CV k=5 data=" + _dataPath_Wiki + " loader=TextLoader{quote=- sparse=- col=Label:R4:0 col=rev_id:TX:1 col=comment:TX:2 col=logged_in:BL:4 col=ns:TX:5 col=sample:TX:6 col=split:TX:7 col=year:R4:3 header=+} xf=Convert{col=logged_in type=R4} xf=CategoricalTransform{col=ns} xf=TextTransform{col=FeaturesText:comment wordExtractor=NGramExtractorTransform{ngram=2}} xf=Concat{col=Features:FeaturesText,logged_in,ns} tr=LightGBMMulticlass{}";
-            using (var tlc = new ConsoleEnvironment(verbose: false, sensitivity: MessageSensitivity.None, outWriter: EmptyWriter.Instance))
+            string cmd = @"CV k=5 data=" + _dataPath_Wiki +
+                    " loader=TextLoader{quote=- sparse=- col=Label:R4:0 col=rev_id:TX:1 col=comment:TX:2 col=logged_in:BL:4 col=ns:TX:5 col=sample:TX:6 col=split:TX:7 col=year:R4:3 header=+}" +
+                    " xf=Convert{col=logged_in type=R4}" +
+                    " xf=CategoricalTransform{col=ns}" +
+                    " xf=TextTransform{col=FeaturesText:comment wordExtractor=NGramExtractorTransform{ngram=2}}" +
+                    " xf=Concat{col=Features:FeaturesText,logged_in,ns} tr=LightGBMMulticlass{}";
+
+            using (var environment = new ConsoleEnvironment(verbose: false, sensitivity: MessageSensitivity.None, outWriter: EmptyWriter.Instance))
             {
-                Maml.MainCore(tlc, cmd, alwaysPrintStacktrace: false);
+                Maml.MainCore(environment, cmd, alwaysPrintStacktrace: false);
             }
         }
 
@@ -78,29 +88,45 @@ public void Test_Multiclass_WikiDetox_BigramsAndTrichar_OVAAveragedPerceptron()
             // This benchmark is profiling bulk scoring speed and not training speed. 
             string modelpath = Path.Combine(Directory.GetCurrentDirectory(), @"WikiModel.fold000.zip");
             string cmd = @"Test data=" + _dataPath_Wiki + " in=" + modelpath;
-            using (var tlc = new ConsoleEnvironment(verbose: false, sensitivity: MessageSensitivity.None, outWriter: EmptyWriter.Instance))
+            using (var environment = new ConsoleEnvironment(verbose: false, sensitivity: MessageSensitivity.None, outWriter: EmptyWriter.Instance))
             {
-                Maml.MainCore(tlc, cmd, alwaysPrintStacktrace: false);
+                Maml.MainCore(environment, cmd, alwaysPrintStacktrace: false);
             }
         }
 
         [Benchmark]
         public void CV_Multiclass_WikiDetox_WordEmbeddings_OVAAveragedPerceptron()
         {
-            string cmd = @"CV tr=OVA{p=AveragedPerceptron{iter=10}} k=5 loader=TextLoader{quote=- sparse=- col=Label:R4:0 col=rev_id:TX:1 col=comment:TX:2 col=logged_in:BL:4 col=ns:TX:5 col=sample:TX:6 col=split:TX:7 col=year:R4:3 header=+} data=" + _dataPath_Wiki + " xf=Convert{col=logged_in type=R4} xf=CategoricalTransform{col=ns} xf=TextTransform{col=FeaturesText:comment tokens=+ wordExtractor=NGramExtractorTransform{ngram=2}} xf=WordEmbeddingsTransform{col=FeaturesWordEmbedding:FeaturesText_TransformedText model=FastTextWikipedia300D} xf=Concat{col=Features:FeaturesText,FeaturesWordEmbedding,logged_in,ns}";
-            using (var tlc = new ConsoleEnvironment(verbose: false, sensitivity: MessageSensitivity.None, outWriter: EmptyWriter.Instance))
+            string cmd = @"CV k=5  data=" + _dataPath_Wiki +
+                " tr=OVA{p=AveragedPerceptron{iter=10}}" +
+                " loader=TextLoader{quote=- sparse=- col=Label:R4:0 col=rev_id:TX:1 col=comment:TX:2 col=logged_in:BL:4 col=ns:TX:5 col=sample:TX:6 col=split:TX:7 col=year:R4:3 header=+}" +
+                " xf=Convert{col=logged_in type=R4}" +
+                " xf=CategoricalTransform{col=ns}" +
+                " xf=TextTransform{col=FeaturesText:comment tokens=+ wordExtractor=NGramExtractorTransform{ngram=2}}" +
+                " xf=WordEmbeddingsTransform{col=FeaturesWordEmbedding:FeaturesText_TransformedText model=FastTextWikipedia300D}" +
+                " xf=Concat{col=Features:FeaturesText,FeaturesWordEmbedding,logged_in,ns}";
+
+            using (var environment = new ConsoleEnvironment(verbose: false, sensitivity: MessageSensitivity.None, outWriter: EmptyWriter.Instance))
             {
-                Maml.MainCore(tlc, cmd, alwaysPrintStacktrace: false);
+                Maml.MainCore(environment, cmd, alwaysPrintStacktrace: false);
             }
         }
 
         [Benchmark]
         public void CV_Multiclass_WikiDetox_WordEmbeddings_SDCAMC()
         {
-            string cmd = @"CV tr=SDCAMC k=5 loader=TextLoader{quote=- sparse=- col=Label:R4:0 col=rev_id:TX:1 col=comment:TX:2 col=logged_in:BL:4 col=ns:TX:5 col=sample:TX:6 col=split:TX:7 col=year:R4:3 header=+} data=" + _dataPath_Wiki + " xf=Convert{col=logged_in type=R4} xf=CategoricalTransform{col=ns} xf=TextTransform{col=FeaturesText:comment tokens=+ wordExtractor={} charExtractor={}} xf=WordEmbeddingsTransform{col=FeaturesWordEmbedding:FeaturesText_TransformedText model=FastTextWikipedia300D} xf=Concat{col=Features:FeaturesWordEmbedding,logged_in,ns}";
-            using (var tlc = new ConsoleEnvironment(verbose: false, sensitivity: MessageSensitivity.None, outWriter: EmptyWriter.Instance))
+            string cmd = @"CV k=5 data=" + _dataPath_Wiki +
+                " tr=SDCAMC" +
+                " loader=TextLoader{quote=- sparse=- col=Label:R4:0 col=rev_id:TX:1 col=comment:TX:2 col=logged_in:BL:4 col=ns:TX:5 col=sample:TX:6 col=split:TX:7 col=year:R4:3 header=+}" +
+                " xf=Convert{col=logged_in type=R4}" +
+                " xf=CategoricalTransform{col=ns}" +
+                " xf=TextTransform{col=FeaturesText:comment tokens=+ wordExtractor={} charExtractor={}}" +
+                " xf=WordEmbeddingsTransform{col=FeaturesWordEmbedding:FeaturesText_TransformedText model=FastTextWikipedia300D}" +
+                " xf=Concat{col=Features:FeaturesWordEmbedding,logged_in,ns}";
+
+            using (var environment = new ConsoleEnvironment(verbose: false, sensitivity: MessageSensitivity.None, outWriter: EmptyWriter.Instance))
             {
-                Maml.MainCore(tlc, cmd, alwaysPrintStacktrace: false);
+                Maml.MainCore(environment, cmd, alwaysPrintStacktrace: false);
             }
         }
     }