Remove duplicate value-to-key mapping transform for multiclass string labels (dotnet#283)

daholste · web-flow · commit f25e4f90d2f6 · 2019-03-13T17:52:48.000-07:00
diff --git a/src/Microsoft.ML.Auto/PipelineSuggesters/PipelineSuggester.cs b/src/Microsoft.ML.Auto/PipelineSuggesters/PipelineSuggester.cs
@@ -35,8 +35,7 @@ public static SuggestedPipeline GetNextInferredPipeline(MLContext context,
         {
             var availableTrainers = RecipeInference.AllowedTrainers(context, task, 
                 ColumnInformationUtil.BuildColumnInfo(columns), trainerWhitelist);
-            var transforms = CalculateTransforms(context, columns, task);
-            //var transforms = TransformInferenceApi.InferTransforms(context, columns, task);
+            var transforms = TransformInferenceApi.InferTransforms(context, task, columns);
 
             // if we haven't run all pipelines once
             if (history.Count() < availableTrainers.Count())
@@ -213,21 +212,5 @@ private static bool SampleHyperparameters(MLContext context, SuggestedTrainer tr
 
             return true;
         }
-
-        private static IEnumerable<SuggestedTransform> CalculateTransforms(
-            MLContext context,
-            (string, DataViewType, ColumnPurpose, ColumnDimensions)[] columns,
-            TaskKind task)
-        {
-            var transforms = TransformInferenceApi.InferTransforms(context, columns).ToList();
-            // this is a work-around for ML.NET bug tracked by https://github.com/dotnet/machinelearning/issues/1969
-            if (task == TaskKind.MulticlassClassification)
-            {
-                var labelColumn = columns.First(c => c.Item3 == ColumnPurpose.Label).Item1;
-                var transform = ValueToKeyMappingExtension.CreateSuggestedTransform(context, labelColumn, labelColumn);
-                transforms.Add(transform);
-            }
-            return transforms;
-        }
     }
 }
diff --git a/src/Microsoft.ML.Auto/TransformInference/TransformInference.cs b/src/Microsoft.ML.Auto/TransformInference/TransformInference.cs
@@ -117,12 +117,12 @@ public bool Equals(ColumnRoutingStructure obj)
 
         internal interface ITransformInferenceExpert
         {
-            IEnumerable<SuggestedTransform> Apply(IntermediateColumn[] columns);
+            IEnumerable<SuggestedTransform> Apply(IntermediateColumn[] columns, TaskKind task);
         }
 
         public abstract class TransformInferenceExpertBase : ITransformInferenceExpert
         {
-            public abstract IEnumerable<SuggestedTransform> Apply(IntermediateColumn[] columns);
+            public abstract IEnumerable<SuggestedTransform> Apply(IntermediateColumn[] columns, TaskKind task);
 
             protected readonly MLContext Context;
 
@@ -137,8 +137,8 @@ private static IEnumerable<ITransformInferenceExpert> GetExperts(MLContext conte
             // The expert work independently of each other, the sequence is irrelevant
             // (it only determines the sequence of resulting transforms).
 
-            // For text labels, convert to categories.
-            yield return new Experts.AutoLabel(context);
+            // For multiclass tasks, convert label column to key
+            yield return new Experts.Label(context);
 
             // For boolean columns use convert transform
             yield return new Experts.Boolean(context);
@@ -155,21 +155,26 @@ private static IEnumerable<ITransformInferenceExpert> GetExperts(MLContext conte
 
         internal static class Experts
         {
-            internal sealed class AutoLabel : TransformInferenceExpertBase
+            internal sealed class Label : TransformInferenceExpertBase
             {
-                public AutoLabel(MLContext context) : base(context)
+                public Label(MLContext context) : base(context)
                 {
                 }
 
-                public override IEnumerable<SuggestedTransform> Apply(IntermediateColumn[] columns)
+                public override IEnumerable<SuggestedTransform> Apply(IntermediateColumn[] columns, TaskKind task)
                 {
+                    if (task != TaskKind.MulticlassClassification)
+                    {
+                        yield break;
+                    }
+
                     var lastLabelColId = Array.FindLastIndex(columns, x => x.Purpose == ColumnPurpose.Label);
                     if (lastLabelColId < 0)
                         yield break;
 
                     var col = columns[lastLabelColId];
 
-                    if (col.Type.IsText())
+                    if (!col.Type.IsKey())
                     {
                         yield return ValueToKeyMappingExtension.CreateSuggestedTransform(Context, col.ColumnName, col.ColumnName);
                     }
@@ -182,7 +187,7 @@ public Categorical(MLContext context) : base(context)
                 {
                 }
 
-                public override IEnumerable<SuggestedTransform> Apply(IntermediateColumn[] columns)
+                public override IEnumerable<SuggestedTransform> Apply(IntermediateColumn[] columns, TaskKind task)
                 {
                     bool foundCat = false;
                     bool foundCatHash = false;
@@ -232,7 +237,7 @@ public Boolean(MLContext context) : base(context)
                 {
                 }
 
-                public override IEnumerable<SuggestedTransform> Apply(IntermediateColumn[] columns)
+                public override IEnumerable<SuggestedTransform> Apply(IntermediateColumn[] columns, TaskKind task)
                 {
                     var newColumns = new List<string>();
 
@@ -260,7 +265,7 @@ public Text(MLContext context) : base(context)
                 {
                 }
 
-                public override IEnumerable<SuggestedTransform> Apply(IntermediateColumn[] columns)
+                public override IEnumerable<SuggestedTransform> Apply(IntermediateColumn[] columns, TaskKind task)
                 {
                     var featureCols = new List<string>();
 
@@ -286,7 +291,7 @@ public NumericMissing(MLContext context) : base(context)
                 {
                 }
 
-                public override IEnumerable<SuggestedTransform> Apply(IntermediateColumn[] columns)
+                public override IEnumerable<SuggestedTransform> Apply(IntermediateColumn[] columns, TaskKind task)
                 {
                     var columnsWithMissing = new List<string>();
                     foreach (var column in columns)
@@ -313,7 +318,7 @@ public override IEnumerable<SuggestedTransform> Apply(IntermediateColumn[] colum
         /// <summary>
         /// Automatically infer transforms for the data view
         /// </summary>
-        public static SuggestedTransform[] InferTransforms(MLContext context, (string, DataViewType, ColumnPurpose, ColumnDimensions)[] columns)
+        public static SuggestedTransform[] InferTransforms(MLContext context, TaskKind task, (string, DataViewType, ColumnPurpose, ColumnDimensions)[] columns)
         {
             var intermediateCols = columns.Where(c => c.Item3 != ColumnPurpose.Ignore)
                 .Select(c => new IntermediateColumn(c.Item1, c.Item2, c.Item3, c.Item4))
@@ -322,7 +327,7 @@ public static SuggestedTransform[] InferTransforms(MLContext context, (string, D
             var suggestedTransforms = new List<SuggestedTransform>();
             foreach (var expert in GetExperts(context))
             {
-                SuggestedTransform[] suggestions = expert.Apply(intermediateCols).ToArray();
+                SuggestedTransform[] suggestions = expert.Apply(intermediateCols, task).ToArray();
                 suggestedTransforms.AddRange(suggestions);
             }
 
diff --git a/src/Microsoft.ML.Auto/TransformInference/TransformInferenceApi.cs b/src/Microsoft.ML.Auto/TransformInference/TransformInferenceApi.cs
@@ -9,9 +9,9 @@ namespace Microsoft.ML.Auto
 {
     internal static class TransformInferenceApi
     {
-        public static IEnumerable<SuggestedTransform> InferTransforms(MLContext context, (string, DataViewType, ColumnPurpose, ColumnDimensions)[] columns)
+        public static IEnumerable<SuggestedTransform> InferTransforms(MLContext context, TaskKind task, (string, DataViewType, ColumnPurpose, ColumnDimensions)[] columns)
         {
-            return TransformInference.InferTransforms(context, columns);
+            return TransformInference.InferTransforms(context, task, columns);
         }
     }
 }
diff --git a/src/Microsoft.ML.Auto/Utils/MLNetUtils/ColumnTypeExtensions.cs b/src/Microsoft.ML.Auto/Utils/MLNetUtils/ColumnTypeExtensions.cs
@@ -29,6 +29,11 @@ public static bool IsVector(this DataViewType columnType)
             return columnType is VectorType;
         }
 
+        public static bool IsKey(this DataViewType columnType)
+        {
+            return columnType is KeyType;
+        }
+
         public static bool IsKnownSizeVector(this DataViewType columnType)
         {
             var vector = columnType as VectorType;
diff --git a/src/Test/TransformInferenceTests.cs b/src/Test/TransformInferenceTests.cs
@@ -645,7 +645,7 @@ public void TransformInferenceCustomLabelCol()
         }
 
         [TestMethod]
-        public void TransformInferenceCustomTextLabelCol()
+        public void TransformInferenceCustomTextLabelColMulticlass()
         {
             TransformInferenceTestCore(new (string, DataViewType, ColumnPurpose, ColumnDimensions)[]
                 {
@@ -663,7 +663,7 @@ public void TransformInferenceCustomTextLabelCol()
     ],
     ""Properties"": {}
   }
-]");
+]", TaskKind.MulticlassClassification);
         }
 
         [TestMethod]
@@ -727,9 +727,10 @@ public void TransformInferenceMissingNameCollision()
 
         private static void TransformInferenceTestCore(
             (string name, DataViewType type, ColumnPurpose purpose, ColumnDimensions dimensions)[] columns,
-            string expectedJson)
+            string expectedJson,
+            TaskKind task = TaskKind.BinaryClassification)
         {
-            var transforms = TransformInferenceApi.InferTransforms(new MLContext(), columns);
+            var transforms = TransformInferenceApi.InferTransforms(new MLContext(), task, columns);
             TestApplyTransformsToRealDataView(transforms, columns);
             var pipelineNodes = transforms.Select(t => t.PipelineNode);
             Util.AssertObjectMatchesJson(expectedJson, pipelineNodes);

Original file line number	Diff line number	Diff line change
`@@ -117,12 +117,12 @@ public bool Equals(ColumnRoutingStructure obj)`
`117`	`117`
`118`	`118`	`internal interface ITransformInferenceExpert`
`119`	`119`	`{`
`120`		`- IEnumerable<SuggestedTransform> Apply(IntermediateColumn[] columns);`
	`120`	`+ IEnumerable<SuggestedTransform> Apply(IntermediateColumn[] columns, TaskKind task);`
`121`	`121`	`}`
`122`	`122`
`123`	`123`	`public abstract class TransformInferenceExpertBase : ITransformInferenceExpert`
`124`	`124`	`{`
`125`		`- public abstract IEnumerable<SuggestedTransform> Apply(IntermediateColumn[] columns);`
	`125`	`+ public abstract IEnumerable<SuggestedTransform> Apply(IntermediateColumn[] columns, TaskKind task);`
`126`	`126`
`127`	`127`	`protected readonly MLContext Context;`
`128`	`128`
`@@ -137,8 +137,8 @@ private static IEnumerable<ITransformInferenceExpert> GetExperts(MLContext conte`
`137`	`137`	`// The expert work independently of each other, the sequence is irrelevant`
`138`	`138`	`// (it only determines the sequence of resulting transforms).`
`139`	`139`
`140`		`- // For text labels, convert to categories.`
`141`		`- yield return new Experts.AutoLabel(context);`
	`140`	`+ // For multiclass tasks, convert label column to key`
	`141`	`+ yield return new Experts.Label(context);`
`142`	`142`
`143`	`143`	`// For boolean columns use convert transform`
`144`	`144`	`yield return new Experts.Boolean(context);`
`@@ -155,21 +155,26 @@ private static IEnumerable<ITransformInferenceExpert> GetExperts(MLContext conte`
`155`	`155`
`156`	`156`	`internal static class Experts`
`157`	`157`	`{`
`158`		`- internal sealed class AutoLabel : TransformInferenceExpertBase`
	`158`	`+ internal sealed class Label : TransformInferenceExpertBase`
`159`	`159`	`{`
`160`		`- public AutoLabel(MLContext context) : base(context)`
	`160`	`+ public Label(MLContext context) : base(context)`
`161`	`161`	`{`
`162`	`162`	`}`
`163`	`163`
`164`		`- public override IEnumerable<SuggestedTransform> Apply(IntermediateColumn[] columns)`
	`164`	`+ public override IEnumerable<SuggestedTransform> Apply(IntermediateColumn[] columns, TaskKind task)`
`165`	`165`	`{`
	`166`	`+ if (task != TaskKind.MulticlassClassification)`
	`167`	`+ {`
	`168`	`+ yield break;`
	`169`	`+ }`
	`170`	`+`
`166`	`171`	`var lastLabelColId = Array.FindLastIndex(columns, x => x.Purpose == ColumnPurpose.Label);`
`167`	`172`	`if (lastLabelColId < 0)`
`168`	`173`	`yield break;`
`169`	`174`
`170`	`175`	`var col = columns[lastLabelColId];`
`171`	`176`
`172`		`- if (col.Type.IsText())`
	`177`	`+ if (!col.Type.IsKey())`
`173`	`178`	`{`
`174`	`179`	`yield return ValueToKeyMappingExtension.CreateSuggestedTransform(Context, col.ColumnName, col.ColumnName);`
`175`	`180`	`}`
`@@ -182,7 +187,7 @@ public Categorical(MLContext context) : base(context)`
`182`	`187`	`{`
`183`	`188`	`}`
`184`	`189`
`185`		`- public override IEnumerable<SuggestedTransform> Apply(IntermediateColumn[] columns)`
	`190`	`+ public override IEnumerable<SuggestedTransform> Apply(IntermediateColumn[] columns, TaskKind task)`
`186`	`191`	`{`
`187`	`192`	`bool foundCat = false;`
`188`	`193`	`bool foundCatHash = false;`
`@@ -232,7 +237,7 @@ public Boolean(MLContext context) : base(context)`
`232`	`237`	`{`
`233`	`238`	`}`
`234`	`239`
`235`		`- public override IEnumerable<SuggestedTransform> Apply(IntermediateColumn[] columns)`
	`240`	`+ public override IEnumerable<SuggestedTransform> Apply(IntermediateColumn[] columns, TaskKind task)`
`236`	`241`	`{`
`237`	`242`	`var newColumns = new List<string>();`
`238`	`243`
`@@ -260,7 +265,7 @@ public Text(MLContext context) : base(context)`
`260`	`265`	`{`
`261`	`266`	`}`
`262`	`267`
`263`		`- public override IEnumerable<SuggestedTransform> Apply(IntermediateColumn[] columns)`
	`268`	`+ public override IEnumerable<SuggestedTransform> Apply(IntermediateColumn[] columns, TaskKind task)`
`264`	`269`	`{`
`265`	`270`	`var featureCols = new List<string>();`
`266`	`271`
`@@ -286,7 +291,7 @@ public NumericMissing(MLContext context) : base(context)`
`286`	`291`	`{`
`287`	`292`	`}`
`288`	`293`
`289`		`- public override IEnumerable<SuggestedTransform> Apply(IntermediateColumn[] columns)`
	`294`	`+ public override IEnumerable<SuggestedTransform> Apply(IntermediateColumn[] columns, TaskKind task)`
`290`	`295`	`{`
`291`	`296`	`var columnsWithMissing = new List<string>();`
`292`	`297`	`foreach (var column in columns)`
`@@ -313,7 +318,7 @@ public override IEnumerable<SuggestedTransform> Apply(IntermediateColumn[] colum`
`313`	`318`	`/// <summary>`
`314`	`319`	`/// Automatically infer transforms for the data view`
`315`	`320`	`/// </summary>`
`316`		`- public static SuggestedTransform[] InferTransforms(MLContext context, (string, DataViewType, ColumnPurpose, ColumnDimensions)[] columns)`
	`321`	`+ public static SuggestedTransform[] InferTransforms(MLContext context, TaskKind task, (string, DataViewType, ColumnPurpose, ColumnDimensions)[] columns)`
`317`	`322`	`{`
`318`	`323`	`var intermediateCols = columns.Where(c => c.Item3 != ColumnPurpose.Ignore)`
`319`	`324`	`.Select(c => new IntermediateColumn(c.Item1, c.Item2, c.Item3, c.Item4))`
`@@ -322,7 +327,7 @@ public static SuggestedTransform[] InferTransforms(MLContext context, (string, D`
`322`	`327`	`var suggestedTransforms = new List<SuggestedTransform>();`
`323`	`328`	`foreach (var expert in GetExperts(context))`
`324`	`329`	`{`
`325`		`- SuggestedTransform[] suggestions = expert.Apply(intermediateCols).ToArray();`
	`330`	`+ SuggestedTransform[] suggestions = expert.Apply(intermediateCols, task).ToArray();`
`326`	`331`	`suggestedTransforms.AddRange(suggestions);`
`327`	`332`	`}`
`328`	`333`
Original file line number	Diff line number	Diff line change
`@@ -9,9 +9,9 @@ namespace Microsoft.ML.Auto`
`9`	`9`	`{`
`10`	`10`	`internal static class TransformInferenceApi`
`11`	`11`	`{`
`12`		`- public static IEnumerable<SuggestedTransform> InferTransforms(MLContext context, (string, DataViewType, ColumnPurpose, ColumnDimensions)[] columns)`
	`12`	`+ public static IEnumerable<SuggestedTransform> InferTransforms(MLContext context, TaskKind task, (string, DataViewType, ColumnPurpose, ColumnDimensions)[] columns)`
`13`	`13`	`{`
`14`		`- return TransformInference.InferTransforms(context, columns);`
	`14`	`+ return TransformInference.InferTransforms(context, task, columns);`
`15`	`15`	`}`
`16`	`16`	`}`
`17`	`17`	`}`
Original file line number	Diff line number	Diff line change
`@@ -29,6 +29,11 @@ public static bool IsVector(this DataViewType columnType)`
`29`	`29`	`return columnType is VectorType;`
`30`	`30`	`}`
`31`	`31`
	`32`	`+ public static bool IsKey(this DataViewType columnType)`
	`33`	`+ {`
	`34`	`+ return columnType is KeyType;`
	`35`	`+ }`
	`36`	`+`
`32`	`37`	`public static bool IsKnownSizeVector(this DataViewType columnType)`
`33`	`38`	`{`
`34`	`39`	`var vector = columnType as VectorType;`
Original file line number	Diff line number	Diff line change
`@@ -645,7 +645,7 @@ public void TransformInferenceCustomLabelCol()`
`645`	`645`	`}`
`646`	`646`
`647`	`647`	`[TestMethod]`
`648`		`- public void TransformInferenceCustomTextLabelCol()`
	`648`	`+ public void TransformInferenceCustomTextLabelColMulticlass()`
`649`	`649`	`{`
`650`	`650`	`TransformInferenceTestCore(new (string, DataViewType, ColumnPurpose, ColumnDimensions)[]`
`651`	`651`	`{`
`@@ -663,7 +663,7 @@ public void TransformInferenceCustomTextLabelCol()`
`663`	`663`	`],`
`664`	`664`	`""Properties"": {}`
`665`	`665`	`}`
`666`		`-]");`
	`666`	`+]", TaskKind.MulticlassClassification);`
`667`	`667`	`}`
`668`	`668`
`669`	`669`	`[TestMethod]`
`@@ -727,9 +727,10 @@ public void TransformInferenceMissingNameCollision()`
`727`	`727`
`728`	`728`	`private static void TransformInferenceTestCore(`
`729`	`729`	`(string name, DataViewType type, ColumnPurpose purpose, ColumnDimensions dimensions)[] columns,`
`730`		`- string expectedJson)`
	`730`	`+ string expectedJson,`
	`731`	`+ TaskKind task = TaskKind.BinaryClassification)`
`731`	`732`	`{`
`732`		`- var transforms = TransformInferenceApi.InferTransforms(new MLContext(), columns);`
	`733`	`+ var transforms = TransformInferenceApi.InferTransforms(new MLContext(), task, columns);`
`733`	`734`	`TestApplyTransformsToRealDataView(transforms, columns);`
`734`	`735`	`var pipelineNodes = transforms.Select(t => t.PipelineNode);`
`735`	`736`	`Util.AssertObjectMatchesJson(expectedJson, pipelineNodes);`