make GetNextPipeline API w/ public Pipeline method on PipelineSuggester; write GetNextPipeline API test; fix public Pipeline object serialization; fix header inferencing bug; write test utils for fetching datasets (dotnet#18)

daholste · Dmitry-A · commit 816e8e804344 · 2019-08-22T14:34:33.000-07:00
diff --git a/src/AutoML/API/Pipeline.cs b/src/AutoML/API/Pipeline.cs
@@ -1,24 +1,36 @@
 ﻿using System.Collections.Generic;
+using Microsoft.ML.Core.Data;
 
 namespace Microsoft.ML.Auto
 {
     public class Pipeline
     {
-        public readonly PipelineNode[] Elements;
+        public PipelineNode[] Elements { get; set; }
 
         public Pipeline(PipelineNode[] elements)
         {
             Elements = elements;
         }
+
+        // (used by Newtonsoft)
+        internal Pipeline()
+        {
+        }
+        
+        public IEstimator<ITransformer> ToEstimator()
+        {
+            var inferredPipeline = InferredPipeline.FromPipeline(this);
+            return inferredPipeline.ToEstimator();
+        }
     }
 
     public class PipelineNode
     {
-        public readonly string Name;
-        public readonly PipelineNodeType ElementType;
-        public readonly string[] InColumns;
-        public readonly string[] OutColumns;
-        public readonly IDictionary<string, object> Properties;
+        public string Name { get; set; }
+        public PipelineNodeType ElementType { get; set; }
+        public string[] InColumns { get; set; }
+        public string[] OutColumns { get; set; }
+        public IDictionary<string, object> Properties { get; set; }
 
         public PipelineNode(string name, PipelineNodeType elementType,
             string[] inColumns, string[] outColumns,
@@ -42,6 +54,11 @@ public PipelineNode(string name, PipelineNodeType elementType,
             this(name, elementType, inColumns, new string[] { outColumn }, properties)
         {
         }
+
+        // (used by Newtonsoft)
+        internal PipelineNode()
+        {
+        }
     }
 
     public enum PipelineNodeType
@@ -55,4 +72,23 @@ public class CustomProperty
         public readonly string Name;
         public readonly IDictionary<string, object> Properties;
     }
+
+    public class PipelineRunResult
+    {
+        public readonly Pipeline Pipeline;
+        public readonly double Score;
+
+        /// <summary>
+        /// This setting is true if the pipeline run succeeded & ran to completion.
+        /// Else, it is false if some exception was thrown before the run could complete.
+        /// </summary>
+        public readonly bool RunSucceded;
+
+        public PipelineRunResult(Pipeline pipeline, double score, bool runSucceeded)
+        {
+            Pipeline = pipeline;
+            Score = score;
+            RunSucceded = runSucceeded;
+        }
+    }
 }
diff --git a/src/AutoML/AutoFitter/AutoFitApi.cs b/src/AutoML/AutoFitter/AutoFitApi.cs
@@ -6,7 +6,7 @@ namespace Microsoft.ML.Auto
 {
     internal static class AutoFitApi
     {
-        public static (PipelineRunResult[] allPipelines, PipelineRunResult bestPipeline) Fit(IDataView trainData, 
+        public static (InferredPipelineRunResult[] allPipelines, InferredPipelineRunResult bestPipeline) Fit(IDataView trainData, 
             IDataView validationData, string label, AutoFitSettings settings, TaskKind task, OptimizingMetric metric, 
             IEnumerable<(string, ColumnPurpose)> purposeOverrides, IDebugLogger debugLogger)
         {
diff --git a/src/AutoML/AutoFitter/AutoFitter.cs b/src/AutoML/AutoFitter/AutoFitter.cs
@@ -14,7 +14,7 @@ namespace Microsoft.ML.Auto
     internal class AutoFitter
     {
         private readonly IDebugLogger _debugLogger;
-        private readonly IList<PipelineRunResult> _history;
+        private readonly IList<InferredPipelineRunResult> _history;
         private readonly string _label;
         private readonly MLContext _mlContext;
         private readonly OptimizingMetricInfo _optimizingMetricInfo;
@@ -29,7 +29,7 @@ public AutoFitter(MLContext mlContext, OptimizingMetricInfo metricInfo, AutoFitS
             IDictionary<string, ColumnPurpose> purposeOverrides, IDebugLogger debugLogger)
         {
             _debugLogger = debugLogger;
-            _history = new List<PipelineRunResult>();
+            _history = new List<InferredPipelineRunResult>();
             _label = label;
             _mlContext = mlContext;
             _optimizingMetricInfo = metricInfo;
@@ -40,7 +40,7 @@ public AutoFitter(MLContext mlContext, OptimizingMetricInfo metricInfo, AutoFitS
             _validationData = validationData;
         }
 
-        public PipelineRunResult[] Fit()
+        public InferredPipelineRunResult[] Fit()
         {
             IteratePipelinesAndFit();
             return _history.ToArray();
@@ -55,7 +55,7 @@ private void IteratePipelinesAndFit()
             do
             {
                 // get next pipeline
-                var pipeline = PipelineSuggester.GetNextPipeline(_history, transforms, availableTrainers, _optimizingMetricInfo.IsMaximizing);
+                var pipeline = PipelineSuggester.GetNextInferredPipeline(_history, transforms, availableTrainers, _optimizingMetricInfo.IsMaximizing);
 
                 // break if no candidates returned, means no valid pipeline available
                 if (pipeline == null)
@@ -75,19 +75,19 @@ private void ProcessPipeline(InferredPipeline pipeline)
             // run pipeline
             var stopwatch = Stopwatch.StartNew();
 
-            PipelineRunResult runResult;
+            InferredPipelineRunResult runResult;
             try
             {
                 var pipelineModel = pipeline.TrainTransformer(_trainData);
                 var scoredValidationData = pipelineModel.Transform(_validationData);
                 var evaluatedMetrics = GetEvaluatedMetrics(scoredValidationData);
                 var score = GetPipelineScore(evaluatedMetrics);
-                runResult = new PipelineRunResult(evaluatedMetrics, pipelineModel, pipeline, score, scoredValidationData);
+                runResult = new InferredPipelineRunResult(evaluatedMetrics, pipelineModel, pipeline, score, scoredValidationData);
             }
             catch(Exception ex)
             {
                 WriteDebugLog(DebugStream.Exception, $"{pipeline.Trainer} Crashed {ex}");
-                runResult = new PipelineRunResult(pipeline, false);
+                runResult = new InferredPipelineRunResult(pipeline, false);
             }
 
             // save pipeline run
diff --git a/src/AutoML/AutoFitter/InferredPipeline.cs b/src/AutoML/AutoFitter/InferredPipeline.cs
@@ -93,7 +93,7 @@ public static InferredPipeline FromPipeline(Pipeline pipeline)
             return new InferredPipeline(transforms, trainer, null, false);
         }
 
-        public ITransformer TrainTransformer(IDataView trainData)
+        public IEstimator<ITransformer> ToEstimator()
         {
             IEstimator<ITransformer> pipeline = new EstimatorChain<ITransformer>();
 
@@ -112,7 +112,13 @@ public ITransformer TrainTransformer(IDataView trainData)
             // append learner to pipeline
             pipeline = pipeline.Append(learner);
 
-            return pipeline.Fit(trainData);
+            return pipeline;
+        }
+
+        public ITransformer TrainTransformer(IDataView trainData)
+        {
+            var estimator = ToEstimator();
+            return estimator.Fit(trainData);
         }
 
         private void AddNormalizationTransforms()
diff --git a/src/AutoML/AutoFitter/InferredPipelineRunResult.cs b/src/AutoML/AutoFitter/InferredPipelineRunResult.cs
@@ -7,7 +7,7 @@
 
 namespace Microsoft.ML.Auto
 {
-    internal class PipelineRunResult
+    internal class InferredPipelineRunResult
     {
         public readonly object EvaluatedMetrics;
         public readonly InferredPipeline Pipeline;
@@ -22,7 +22,7 @@ internal class PipelineRunResult
 
         public ITransformer Model { get; set; }
 
-        public PipelineRunResult(object evaluatedMetrics, ITransformer model, InferredPipeline pipeline, double score, IDataView scoredValidationData,
+        public InferredPipelineRunResult(object evaluatedMetrics, ITransformer model, InferredPipeline pipeline, double score, IDataView scoredValidationData,
             bool runSucceeded = true)
         {
             EvaluatedMetrics = evaluatedMetrics;
@@ -33,12 +33,19 @@ public PipelineRunResult(object evaluatedMetrics, ITransformer model, InferredPi
             RunSucceded = runSucceeded;
         }
 
-        public PipelineRunResult(InferredPipeline pipeline, bool runSucceeded)
+        public InferredPipelineRunResult(InferredPipeline pipeline, bool runSucceeded)
         {
             Pipeline = pipeline;
             RunSucceded = runSucceeded;
         }
 
+        public static InferredPipelineRunResult FromPipelineRunResult(PipelineRunResult pipelineRunResult)
+        {
+            return new InferredPipelineRunResult(null, null, 
+                InferredPipeline.FromPipeline(pipelineRunResult.Pipeline),
+                pipelineRunResult.Score, null, pipelineRunResult.RunSucceded);
+        }
+
         public IRunResult ToRunResult(bool isMetricMaximizing)
         {
            return new RunResult(Pipeline.Trainer.HyperParamSet, Score, isMetricMaximizing);
diff --git a/src/AutoML/ColumnInference/ColumnInferenceApi.cs b/src/AutoML/ColumnInference/ColumnInferenceApi.cs
@@ -11,7 +11,7 @@ public static ColumnInferenceResult InferColumns(MLContext context, string path,
         {
             var sample = TextFileSample.CreateFromFullFile(path);
             var splitInference = InferSplit(sample, separatorChar, allowQuotedStrings, supportSparse);
-            var typeInference = InferColumnTypes(context, sample, splitInference);
+            var typeInference = InferColumnTypes(context, sample, splitInference, hasHeader);
             var typedLoaderArgs = new TextLoader.Arguments
             {
                 Column = ColumnTypeInference.GenerateLoaderColumns(typeInference.Columns),
@@ -59,7 +59,7 @@ private static TextFileContents.ColumnSplitResult InferSplit(TextFileSample samp
         }
 
         private static ColumnTypeInference.InferenceResult InferColumnTypes(MLContext context, TextFileSample sample,
-            TextFileContents.ColumnSplitResult splitInference)
+            TextFileContents.ColumnSplitResult splitInference, bool hasHeader)
         {
             // infer column types
             var typeInferenceResult = ColumnTypeInference.InferTextFileColumnTypes(context, sample,
@@ -69,6 +69,7 @@ private static ColumnTypeInference.InferenceResult InferColumnTypes(MLContext co
                     Separator = splitInference.Separator,
                     AllowSparse = splitInference.AllowSparse,
                     AllowQuote = splitInference.AllowQuote,
+                    HasHeader = hasHeader
                 });
 
             if (!typeInferenceResult.IsSuccess)
diff --git a/src/AutoML/ColumnInference/ColumnTypeInference.cs b/src/AutoML/ColumnInference/ColumnTypeInference.cs
@@ -27,6 +27,7 @@ internal sealed class Arguments
             public bool AllowSparse;
             public bool AllowQuote;
             public int ColumnCount;
+            public bool HasHeader;
             public int MaxRowsToRead;
 
             public Arguments()
@@ -325,23 +326,14 @@ private static InferenceResult InferTextFileColumnTypesCore(MLContext env, IMult
                     suspect--;
             }
 
-            // REVIEW: Why not use this for column names as well?
-            TextLoader.Arguments fileArgs;
-            bool hasHeader;
-            if (TextLoader.FileContainsValidSchema(env, fileSource, out fileArgs))
-                hasHeader = fileArgs.HasHeader;
-            else
-                hasHeader = suspect > 0;
-            hasHeader = true;
-
             // suggest names
             var names = new List<string>();
             usedNames.Clear();
             foreach (var col in cols)
             {
                 string name0;
                 string name;
-                name0 = name = SuggestName(col, hasHeader);
+                name0 = name = SuggestName(col, args.HasHeader);
                 int i = 0;
                 while (!usedNames.Add(name))
                     name = string.Format("{0}_{1:00}", name0, i++);
@@ -352,7 +344,7 @@ private static InferenceResult InferTextFileColumnTypesCore(MLContext env, IMult
 
             var numerics = outCols.Count(x => x.ItemType.IsNumber());
             
-            return InferenceResult.Success(outCols, hasHeader, cols.Select(col => col.RawData).ToArray());
+            return InferenceResult.Success(outCols, args.HasHeader, cols.Select(col => col.RawData).ToArray());
         }
 
         private static string SuggestName(IntermediateColumn column, bool hasHeader)
diff --git a/src/AutoML/PipelineSuggesters/PipelineSuggester.cs b/src/AutoML/PipelineSuggesters/PipelineSuggester.cs
@@ -12,7 +12,18 @@ internal static class PipelineSuggester
     {
         private const int TopKTrainers = 3;
 
-        public static InferredPipeline GetNextPipeline(IEnumerable<PipelineRunResult> history,
+        public static Pipeline GetNextPipeline(IEnumerable<PipelineRunResult> history,
+            IEnumerable<SuggestedTransform> transforms,
+            IEnumerable<SuggestedTrainer> availableTrainers,
+            bool isMaximizingMetric = true)
+        {
+            var inferredHistory = history.Select(r => InferredPipelineRunResult.FromPipelineRunResult(r));
+            var nextInferredPipeline = GetNextInferredPipeline(inferredHistory,
+                transforms, availableTrainers, isMaximizingMetric);
+            return nextInferredPipeline.ToPipeline();
+        }
+
+        public static InferredPipeline GetNextInferredPipeline(IEnumerable<InferredPipelineRunResult> history,
             IEnumerable<SuggestedTransform> transforms,
             IEnumerable<SuggestedTrainer> availableTrainers,
             bool isMaximizingMetric = true)
@@ -49,15 +60,15 @@ public static InferredPipeline GetNextPipeline(IEnumerable<PipelineRunResult> hi
         /// <summary>
         /// Get top trainers from first stage
         /// </summary>
-        private static IEnumerable<SuggestedTrainer> GetTopTrainers(IEnumerable<PipelineRunResult> history, 
+        private static IEnumerable<SuggestedTrainer> GetTopTrainers(IEnumerable<InferredPipelineRunResult> history, 
             IEnumerable<SuggestedTrainer> availableTrainers,
             bool isMaximizingMetric)
         {
             // narrow history to first stage runs
             history = history.Take(availableTrainers.Count());
 
             history = history.GroupBy(r => r.Pipeline.Trainer.TrainerName).Select(g => g.First());
-            IEnumerable<PipelineRunResult> sortedHistory = history.OrderBy(r => r.Score);
+            IEnumerable<InferredPipelineRunResult> sortedHistory = history.OrderBy(r => r.Score);
             if(isMaximizingMetric)
             {
                 sortedHistory = sortedHistory.Reverse();
@@ -66,7 +77,7 @@ private static IEnumerable<SuggestedTrainer> GetTopTrainers(IEnumerable<Pipeline
             return topTrainers;
         }
 
-        private static InferredPipeline GetNextFirstStagePipeline(IEnumerable<PipelineRunResult> history,
+        private static InferredPipeline GetNextFirstStagePipeline(IEnumerable<InferredPipelineRunResult> history,
             IEnumerable<SuggestedTrainer> availableTrainers,
             IEnumerable<SuggestedTransform> transforms)
         {
@@ -133,7 +144,7 @@ private static IValueGenerator[] ConvertToValueGenerators(IEnumerable<SweepableP
             return results;
         }
 
-        private static void SampleHyperparameters(SuggestedTrainer trainer, IEnumerable<PipelineRunResult> history, bool isMaximizingMetric)
+        private static void SampleHyperparameters(SuggestedTrainer trainer, IEnumerable<InferredPipelineRunResult> history, bool isMaximizingMetric)
         {
             var sps = ConvertToValueGenerators(trainer.SweepParams);
             var sweeper = new SmacSweeper(
@@ -142,7 +153,7 @@ private static void SampleHyperparameters(SuggestedTrainer trainer, IEnumerable<
                     SweptParameters = sps
                 });
 
-            IEnumerable<PipelineRunResult> historyToUse = history
+            IEnumerable<InferredPipelineRunResult> historyToUse = history
                 .Where(r => r.RunSucceded && r.Pipeline.Trainer.TrainerName == trainer.TrainerName && r.Pipeline.Trainer.HyperParamSet != null);
 
             // get new set of hyperparameter values
diff --git a/src/Test/DatasetUtil.cs b/src/Test/DatasetUtil.cs
@@ -0,0 +1,42 @@
+﻿using System;
+using System.IO;
+using System.Net;
+using Microsoft.ML.Data;
+
+namespace Microsoft.ML.Auto.Test
+{
+    internal static class DatasetUtil
+    {
+        public const string UciAdultLabel = DefaultColumnNames.Label;
+
+        private static IDataView _uciAdultDataView;
+
+        public static IDataView GetUciAdultDataView()
+        {
+            if(_uciAdultDataView == null)
+            {
+                var uciAdultDataFile = DownloadUciAdultDataset();
+                _uciAdultDataView = (new MLContext()).Data.AutoRead(uciAdultDataFile, UciAdultLabel, true);
+            }
+            return _uciAdultDataView;
+        }
+
+        // downloads the UCI Adult dataset from the ML.Net repo
+        private static string DownloadUciAdultDataset() =>
+            DownloadIfNotExists("https://raw.githubusercontent.com/dotnet/machinelearning/f0e639af5ffdc839aae8e65d19b5a9a1f0db634a/test/data/adult.tiny.with-schema.txt", "uciadult.dataset");
+
+        private static string DownloadIfNotExists(string baseGitPath, string dataFile)
+        {
+            // if file doesn't already exist, download it
+            if(!File.Exists(dataFile))
+            {
+                using (var client = new WebClient())
+                {
+                    client.DownloadFile(new Uri($"{baseGitPath}"), dataFile);
+                }
+            }
+
+            return dataFile;
+        }
+    }
+}
diff --git a/src/Test/GetNextPipelineTests.cs b/src/Test/GetNextPipelineTests.cs

Original file line number	Diff line number	Diff line change
`@@ -6,7 +6,7 @@ namespace Microsoft.ML.Auto`
`6`	`6`	`{`
`7`	`7`	`internal static class AutoFitApi`
`8`	`8`	`{`
`9`		`- public static (PipelineRunResult[] allPipelines, PipelineRunResult bestPipeline) Fit(IDataView trainData,`
	`9`	`+ public static (InferredPipelineRunResult[] allPipelines, InferredPipelineRunResult bestPipeline) Fit(IDataView trainData,`
`10`	`10`	`IDataView validationData, string label, AutoFitSettings settings, TaskKind task, OptimizingMetric metric,`
`11`	`11`	`IEnumerable<(string, ColumnPurpose)> purposeOverrides, IDebugLogger debugLogger)`
`12`	`12`	`{`
Original file line number	Diff line number	Diff line change
`@@ -93,7 +93,7 @@ public static InferredPipeline FromPipeline(Pipeline pipeline)`
`93`	`93`	`return new InferredPipeline(transforms, trainer, null, false);`
`94`	`94`	`}`
`95`	`95`
`96`		`- public ITransformer TrainTransformer(IDataView trainData)`
	`96`	`+ public IEstimator<ITransformer> ToEstimator()`
`97`	`97`	`{`
`98`	`98`	`IEstimator<ITransformer> pipeline = new EstimatorChain<ITransformer>();`
`99`	`99`
`@@ -112,7 +112,13 @@ public ITransformer TrainTransformer(IDataView trainData)`
`112`	`112`	`// append learner to pipeline`
`113`	`113`	`pipeline = pipeline.Append(learner);`
`114`	`114`
`115`		`- return pipeline.Fit(trainData);`
	`115`	`+ return pipeline;`
	`116`	`+ }`
	`117`	`+`
	`118`	`+ public ITransformer TrainTransformer(IDataView trainData)`
	`119`	`+ {`
	`120`	`+ var estimator = ToEstimator();`
	`121`	`+ return estimator.Fit(trainData);`
`116`	`122`	`}`
`117`	`123`
`118`	`124`	`private void AddNormalizationTransforms()`
Original file line number	Diff line number	Diff line change
`@@ -7,7 +7,7 @@`
`7`	`7`
`8`	`8`	`namespace Microsoft.ML.Auto`
`9`	`9`	`{`
`10`		`- internal class PipelineRunResult`
	`10`	`+ internal class InferredPipelineRunResult`
`11`	`11`	`{`
`12`	`12`	`public readonly object EvaluatedMetrics;`
`13`	`13`	`public readonly InferredPipeline Pipeline;`
`@@ -22,7 +22,7 @@ internal class PipelineRunResult`
`22`	`22`
`23`	`23`	`public ITransformer Model { get; set; }`
`24`	`24`
`25`		`- public PipelineRunResult(object evaluatedMetrics, ITransformer model, InferredPipeline pipeline, double score, IDataView scoredValidationData,`
	`25`	`+ public InferredPipelineRunResult(object evaluatedMetrics, ITransformer model, InferredPipeline pipeline, double score, IDataView scoredValidationData,`
`26`	`26`	`bool runSucceeded = true)`
`27`	`27`	`{`
`28`	`28`	`EvaluatedMetrics = evaluatedMetrics;`
`@@ -33,12 +33,19 @@ public PipelineRunResult(object evaluatedMetrics, ITransformer model, InferredPi`
`33`	`33`	`RunSucceded = runSucceeded;`
`34`	`34`	`}`
`35`	`35`
`36`		`- public PipelineRunResult(InferredPipeline pipeline, bool runSucceeded)`
	`36`	`+ public InferredPipelineRunResult(InferredPipeline pipeline, bool runSucceeded)`
`37`	`37`	`{`
`38`	`38`	`Pipeline = pipeline;`
`39`	`39`	`RunSucceded = runSucceeded;`
`40`	`40`	`}`
`41`	`41`
	`42`	`+ public static InferredPipelineRunResult FromPipelineRunResult(PipelineRunResult pipelineRunResult)`
	`43`	`+ {`
	`44`	`+ return new InferredPipelineRunResult(null, null,`
	`45`	`+ InferredPipeline.FromPipeline(pipelineRunResult.Pipeline),`
	`46`	`+ pipelineRunResult.Score, null, pipelineRunResult.RunSucceded);`
	`47`	`+ }`
	`48`	`+`
`42`	`49`	`public IRunResult ToRunResult(bool isMetricMaximizing)`
`43`	`50`	`{`
`44`	`51`	`return new RunResult(Pipeline.Trainer.HyperParamSet, Score, isMetricMaximizing);`
Original file line number	Diff line number	Diff line change
`@@ -11,7 +11,7 @@ public static ColumnInferenceResult InferColumns(MLContext context, string path,`
`11`	`11`	`{`
`12`	`12`	`var sample = TextFileSample.CreateFromFullFile(path);`
`13`	`13`	`var splitInference = InferSplit(sample, separatorChar, allowQuotedStrings, supportSparse);`
`14`		`- var typeInference = InferColumnTypes(context, sample, splitInference);`
	`14`	`+ var typeInference = InferColumnTypes(context, sample, splitInference, hasHeader);`
`15`	`15`	`var typedLoaderArgs = new TextLoader.Arguments`
`16`	`16`	`{`
`17`	`17`	`Column = ColumnTypeInference.GenerateLoaderColumns(typeInference.Columns),`
`@@ -59,7 +59,7 @@ private static TextFileContents.ColumnSplitResult InferSplit(TextFileSample samp`
`59`	`59`	`}`
`60`	`60`
`61`	`61`	`private static ColumnTypeInference.InferenceResult InferColumnTypes(MLContext context, TextFileSample sample,`
`62`		`- TextFileContents.ColumnSplitResult splitInference)`
	`62`	`+ TextFileContents.ColumnSplitResult splitInference, bool hasHeader)`
`63`	`63`	`{`
`64`	`64`	`// infer column types`
`65`	`65`	`var typeInferenceResult = ColumnTypeInference.InferTextFileColumnTypes(context, sample,`
`@@ -69,6 +69,7 @@ private static ColumnTypeInference.InferenceResult InferColumnTypes(MLContext co`
`69`	`69`	`Separator = splitInference.Separator,`
`70`	`70`	`AllowSparse = splitInference.AllowSparse,`
`71`	`71`	`AllowQuote = splitInference.AllowQuote,`
	`72`	`+ HasHeader = hasHeader`
`72`	`73`	`});`
`73`	`74`
`74`	`75`	`if (!typeInferenceResult.IsSuccess)`