Add sampling key column (dotnet#268)

daholste · web-flow · commit 3ad0798a1665 · 2019-03-06T20:40:45.000-08:00
diff --git a/src/Microsoft.ML.Auto/API/BinaryClassificationExperiment.cs b/src/Microsoft.ML.Auto/API/BinaryClassificationExperiment.cs
@@ -54,9 +54,14 @@ internal BinaryClassificationExperiment(MLContext context, BinaryExperimentSetti
             _settings = settings;
         }
 
-        public IEnumerable<RunResult<BinaryClassificationMetrics>> Execute(IDataView trainData, string labelColumn = DefaultColumnNames.Label, IEstimator<ITransformer> preFeaturizers = null)
+        public IEnumerable<RunResult<BinaryClassificationMetrics>> Execute(IDataView trainData, string labelColumn = DefaultColumnNames.Label,
+            string samplingKeyColumn = null, IEstimator<ITransformer> preFeaturizers = null)
         {
-            var columnInformation = new ColumnInformation() { LabelColumn = labelColumn };
+            var columnInformation = new ColumnInformation()
+            {
+                LabelColumn = labelColumn,
+                SamplingKeyColumn = samplingKeyColumn
+            };
             return Execute(_context, trainData, columnInformation, null, preFeaturizers);
         }
 
diff --git a/src/Microsoft.ML.Auto/API/ColumnInference.cs b/src/Microsoft.ML.Auto/API/ColumnInference.cs
@@ -18,6 +18,7 @@ public sealed class ColumnInformation
     {
         public string LabelColumn { get; set; } = DefaultColumnNames.Label;
         public string WeightColumn { get; set; }
+        public string SamplingKeyColumn { get; set; }
         public ICollection<string> CategoricalColumns { get; } = new Collection<string>();
         public ICollection<string> NumericColumns { get; } = new Collection<string>();
         public ICollection<string> TextColumns { get; } = new Collection<string>();
diff --git a/src/Microsoft.ML.Auto/API/MulticlassClassificationExperiment.cs b/src/Microsoft.ML.Auto/API/MulticlassClassificationExperiment.cs
@@ -52,9 +52,14 @@ internal MulticlassClassificationExperiment(MLContext context, MulticlassExperim
             _settings = settings;
         }
 
-        public IEnumerable<RunResult<MultiClassClassifierMetrics>> Execute(IDataView trainData, string labelColumn = DefaultColumnNames.Label, IEstimator<ITransformer> preFeaturizers = null)
+        public IEnumerable<RunResult<MultiClassClassifierMetrics>> Execute(IDataView trainData, string labelColumn = DefaultColumnNames.Label,
+            string samplingKeyColumn = null, IEstimator<ITransformer> preFeaturizers = null)
         {
-            var columnInformation = new ColumnInformation() { LabelColumn = labelColumn };
+            var columnInformation = new ColumnInformation()
+            {
+                LabelColumn = labelColumn,
+                SamplingKeyColumn = samplingKeyColumn
+            };
             return Execute(_context, trainData, columnInformation, null, preFeaturizers);
         }
 
diff --git a/src/Microsoft.ML.Auto/API/RegressionExperiment.cs b/src/Microsoft.ML.Auto/API/RegressionExperiment.cs
@@ -49,9 +49,14 @@ internal RegressionExperiment(MLContext context, RegressionExperimentSettings se
             _settings = settings;
         }
 
-        public IEnumerable<RunResult<RegressionMetrics>> Execute(IDataView trainData, string labelColumn = DefaultColumnNames.Label, IEstimator<ITransformer> preFeaturizers = null)
+        public IEnumerable<RunResult<RegressionMetrics>> Execute(IDataView trainData, string labelColumn = DefaultColumnNames.Label,
+            string samplingKeyColumn = null, IEstimator<ITransformer> preFeaturizers = null)
         {
-            var columnInformation = new ColumnInformation() { LabelColumn = labelColumn };
+            var columnInformation = new ColumnInformation()
+            {
+                LabelColumn = labelColumn,
+                SamplingKeyColumn = samplingKeyColumn
+            };
             return Execute(_context, trainData, columnInformation, null, preFeaturizers);
         }
 
diff --git a/src/Microsoft.ML.Auto/AutoMlUtils.cs b/src/Microsoft.ML.Auto/AutoMlUtils.cs
@@ -3,10 +3,7 @@
 // See the LICENSE file in the project root for more information.
 
 using System;
-using System.Collections.Generic;
-using System.Linq;
 using Microsoft.Data.DataView;
-using Microsoft.ML.Transforms;
 
 namespace Microsoft.ML.Auto
 {
@@ -29,10 +26,10 @@ public static IDataView DropLastColumn(this IDataView data, MLContext context)
         }
 
         public static (IDataView testData, IDataView validationData) TestValidateSplit(this TrainCatalogBase catalog, 
-            MLContext context, IDataView trainData)
+            MLContext context, IDataView trainData, ColumnInformation columnInfo)
         {
             IDataView validationData;
-            var splitData = catalog.TrainTestSplit(trainData);
+            var splitData = catalog.TrainTestSplit(trainData, samplingKeyColumn: columnInfo.SamplingKeyColumn);
             trainData = splitData.TrainSet;
             validationData = splitData.TestSet;
             trainData = trainData.DropLastColumn(context);
diff --git a/src/Microsoft.ML.Auto/ColumnInference/ColumnInformationUtil.cs b/src/Microsoft.ML.Auto/ColumnInference/ColumnInformationUtil.cs
@@ -23,6 +23,11 @@ internal static class ColumnInformationUtil
                 return ColumnPurpose.Weight;
             }
 
+            if (columnName == columnInfo.SamplingKeyColumn)
+            {
+                return ColumnPurpose.SamplingKey;
+            }
+
             if (columnInfo.CategoricalColumns.Contains(columnName))
             {
                 return ColumnPurpose.CategoricalFeature;
@@ -60,6 +65,9 @@ internal static ColumnInformation BuildColumnInfo(IEnumerable<(string name, Colu
                     case ColumnPurpose.Weight:
                         columnInfo.WeightColumn = column.name;
                         break;
+                    case ColumnPurpose.SamplingKey:
+                        columnInfo.SamplingKeyColumn = column.name;
+                        break;
                     case ColumnPurpose.CategoricalFeature:
                         columnInfo.CategoricalColumns.Add(column.name);
                         break;
diff --git a/src/Microsoft.ML.Auto/ColumnInference/ColumnPurpose.cs b/src/Microsoft.ML.Auto/ColumnInference/ColumnPurpose.cs
@@ -12,6 +12,7 @@ internal enum ColumnPurpose
         CategoricalFeature = 3,
         TextFeature = 4,
         Weight = 5,
-        ImagePath = 6
+        ImagePath = 6,
+        SamplingKey = 7
     }
 }
diff --git a/src/Microsoft.ML.Auto/Experiment/Experiment.cs b/src/Microsoft.ML.Auto/Experiment/Experiment.cs
@@ -42,7 +42,7 @@ public Experiment(MLContext context,
         {
             if (validationData == null)
             {
-                (trainData, validationData) = context.Regression.TestValidateSplit(context, trainData);
+                (trainData, validationData) = context.Regression.TestValidateSplit(context, trainData, columnInfo);
             }
             _trainData = trainData;
             _validationData = validationData;
diff --git a/src/Microsoft.ML.Auto/Utils/UserInputValidationUtil.cs b/src/Microsoft.ML.Auto/Utils/UserInputValidationUtil.cs
@@ -20,6 +20,7 @@ internal static class UserInputValidationUtil
         private const string CategoricalColumnPurposeName = "categorical";
         private const string TextColumnPurposeName = "text";
         private const string IgnoredColumnPurposeName = "ignored";
+        private const string SamplingKeyColumnPurposeName = "sampling key";
 
         public static void ValidateExperimentExecuteArgs(IDataView trainData, ColumnInformation columnInformation,
             IDataView validationData)
@@ -65,6 +66,7 @@ private static void ValidateColumnInformation(IDataView trainData, ColumnInforma
             ValidateColumnInformation(columnInformation);
             ValidateTrainDataColumn(trainData, columnInformation.LabelColumn, LabelColumnPurposeName);
             ValidateTrainDataColumn(trainData, columnInformation.WeightColumn, WeightColumnPurposeName);
+            ValidateTrainDataColumn(trainData, columnInformation.SamplingKeyColumn, SamplingKeyColumnPurposeName);
             ValidateTrainDataColumns(trainData, columnInformation.CategoricalColumns, CategoricalColumnPurposeName,
                 new DataViewType[] { NumberDataViewType.Single, TextDataViewType.Instance });
             ValidateTrainDataColumns(trainData, columnInformation.NumericColumns, NumericColumnPurposeName,
@@ -190,7 +192,7 @@ private static void ValidateTrainDataColumn(IDataView trainData, string columnNa
             var nullableColumn = trainData.Schema.GetColumnOrNull(columnName);
             if (nullableColumn == null)
             {
-                throw new ArgumentException($"Provided {columnPurpose} column {columnName} '{columnName}' not found in training data.");
+                throw new ArgumentException($"Provided {columnPurpose} column '{columnName}' not found in training data.");
             }
 
             if(allowedTypes == null)
diff --git a/src/Samples/Helpers/ConsoleHelper.cs b/src/Samples/Helpers/ConsoleHelper.cs
@@ -105,6 +105,7 @@ public void Print()
             var info = _results.ColumnInformation;
             AppendTableRow(tableRows, info.LabelColumn, "Label");
             AppendTableRow(tableRows, info.WeightColumn, "Weight");
+            AppendTableRow(tableRows, info.SamplingKeyColumn, "Sampling Key");
             AppendTableRows(tableRows, info.CategoricalColumns, "Categorical");
             AppendTableRows(tableRows, info.NumericColumns, "Numeric");
             AppendTableRows(tableRows, info.TextColumns, "Text");
diff --git a/src/Test/ColumnInformationUtilTests.cs b/src/Test/ColumnInformationUtilTests.cs
@@ -1,6 +1,7 @@
-﻿using System;
-using System.Collections.Generic;
-using System.Text;
+﻿// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
 using Microsoft.VisualStudio.TestTools.UnitTesting;
 
 namespace Microsoft.ML.Auto.Test
@@ -15,6 +16,7 @@ public void GetColumnPurpose()
             {
                 LabelColumn = "Label",
                 WeightColumn = "Weight",
+                SamplingKeyColumn = "SamplingKey",
             };
             columnInfo.CategoricalColumns.Add("Cat");
             columnInfo.NumericColumns.Add("Num");
@@ -23,6 +25,7 @@ public void GetColumnPurpose()
 
             Assert.AreEqual(ColumnPurpose.Label, ColumnInformationUtil.GetColumnPurpose(columnInfo, "Label"));
             Assert.AreEqual(ColumnPurpose.Weight, ColumnInformationUtil.GetColumnPurpose(columnInfo, "Weight"));
+            Assert.AreEqual(ColumnPurpose.SamplingKey, ColumnInformationUtil.GetColumnPurpose(columnInfo, "SamplingKey"));
             Assert.AreEqual(ColumnPurpose.CategoricalFeature, ColumnInformationUtil.GetColumnPurpose(columnInfo, "Cat"));
             Assert.AreEqual(ColumnPurpose.NumericFeature, ColumnInformationUtil.GetColumnPurpose(columnInfo, "Num"));
             Assert.AreEqual(ColumnPurpose.TextFeature, ColumnInformationUtil.GetColumnPurpose(columnInfo, "Text"));

Original file line number	Diff line number	Diff line change
`@@ -54,9 +54,14 @@ internal BinaryClassificationExperiment(MLContext context, BinaryExperimentSetti`
`54`	`54`	`_settings = settings;`
`55`	`55`	`}`
`56`	`56`
`57`		`- public IEnumerable<RunResult<BinaryClassificationMetrics>> Execute(IDataView trainData, string labelColumn = DefaultColumnNames.Label, IEstimator<ITransformer> preFeaturizers = null)`
	`57`	`+ public IEnumerable<RunResult<BinaryClassificationMetrics>> Execute(IDataView trainData, string labelColumn = DefaultColumnNames.Label,`
	`58`	`+ string samplingKeyColumn = null, IEstimator<ITransformer> preFeaturizers = null)`
`58`	`59`	`{`
`59`		`- var columnInformation = new ColumnInformation() { LabelColumn = labelColumn };`
	`60`	`+ var columnInformation = new ColumnInformation()`
	`61`	`+ {`
	`62`	`+ LabelColumn = labelColumn,`
	`63`	`+ SamplingKeyColumn = samplingKeyColumn`
	`64`	`+ };`
`60`	`65`	`return Execute(_context, trainData, columnInformation, null, preFeaturizers);`
`61`	`66`	`}`
`62`	`67`
Original file line number	Diff line number	Diff line change
`@@ -18,6 +18,7 @@ public sealed class ColumnInformation`
`18`	`18`	`{`
`19`	`19`	`public string LabelColumn { get; set; } = DefaultColumnNames.Label;`
`20`	`20`	`public string WeightColumn { get; set; }`
	`21`	`+ public string SamplingKeyColumn { get; set; }`
`21`	`22`	`public ICollection<string> CategoricalColumns { get; } = new Collection<string>();`
`22`	`23`	`public ICollection<string> NumericColumns { get; } = new Collection<string>();`
`23`	`24`	`public ICollection<string> TextColumns { get; } = new Collection<string>();`
Original file line number	Diff line number	Diff line change
`@@ -52,9 +52,14 @@ internal MulticlassClassificationExperiment(MLContext context, MulticlassExperim`
`52`	`52`	`_settings = settings;`
`53`	`53`	`}`
`54`	`54`
`55`		`- public IEnumerable<RunResult<MultiClassClassifierMetrics>> Execute(IDataView trainData, string labelColumn = DefaultColumnNames.Label, IEstimator<ITransformer> preFeaturizers = null)`
	`55`	`+ public IEnumerable<RunResult<MultiClassClassifierMetrics>> Execute(IDataView trainData, string labelColumn = DefaultColumnNames.Label,`
	`56`	`+ string samplingKeyColumn = null, IEstimator<ITransformer> preFeaturizers = null)`
`56`	`57`	`{`
`57`		`- var columnInformation = new ColumnInformation() { LabelColumn = labelColumn };`
	`58`	`+ var columnInformation = new ColumnInformation()`
	`59`	`+ {`
	`60`	`+ LabelColumn = labelColumn,`
	`61`	`+ SamplingKeyColumn = samplingKeyColumn`
	`62`	`+ };`
`58`	`63`	`return Execute(_context, trainData, columnInformation, null, preFeaturizers);`
`59`	`64`	`}`
`60`	`65`
Original file line number	Diff line number	Diff line change
`@@ -49,9 +49,14 @@ internal RegressionExperiment(MLContext context, RegressionExperimentSettings se`
`49`	`49`	`_settings = settings;`
`50`	`50`	`}`
`51`	`51`
`52`		`- public IEnumerable<RunResult<RegressionMetrics>> Execute(IDataView trainData, string labelColumn = DefaultColumnNames.Label, IEstimator<ITransformer> preFeaturizers = null)`
	`52`	`+ public IEnumerable<RunResult<RegressionMetrics>> Execute(IDataView trainData, string labelColumn = DefaultColumnNames.Label,`
	`53`	`+ string samplingKeyColumn = null, IEstimator<ITransformer> preFeaturizers = null)`
`53`	`54`	`{`
`54`		`- var columnInformation = new ColumnInformation() { LabelColumn = labelColumn };`
	`55`	`+ var columnInformation = new ColumnInformation()`
	`56`	`+ {`
	`57`	`+ LabelColumn = labelColumn,`
	`58`	`+ SamplingKeyColumn = samplingKeyColumn`
	`59`	`+ };`
`55`	`60`	`return Execute(_context, trainData, columnInformation, null, preFeaturizers);`
`56`	`61`	`}`
`57`	`62`
Original file line number	Diff line number	Diff line change
`@@ -12,6 +12,7 @@ internal enum ColumnPurpose`
`12`	`12`	`CategoricalFeature = 3,`
`13`	`13`	`TextFeature = 4,`
`14`	`14`	`Weight = 5,`
`15`		`- ImagePath = 6`
	`15`	`+ ImagePath = 6,`
	`16`	`+ SamplingKey = 7`
`16`	`17`	`}`
`17`	`18`	`}`
Original file line number	Diff line number	Diff line change
`@@ -42,7 +42,7 @@ public Experiment(MLContext context,`
`42`	`42`	`{`
`43`	`43`	`if (validationData == null)`
`44`	`44`	`{`
`45`		`- (trainData, validationData) = context.Regression.TestValidateSplit(context, trainData);`
	`45`	`+ (trainData, validationData) = context.Regression.TestValidateSplit(context, trainData, columnInfo);`
`46`	`46`	`}`
`47`	`47`	`_trainData = trainData;`
`48`	`48`	`_validationData = validationData;`