benchmarking related changes (dotnet#63)

* Create test.txt

* Create test.txt

* changes needed for benchmarking

* forgot one file

* merge conflict fix

* fix build break

* back out my version of the fix for Label column issue and fix the original fix

* bogus file removal

* undo SuggestedPipeline change

* remove labelCol from pipeline suggester

* fix build break

Author: Dmitry-A

src/AutoML/API/MLContextAutoFitExtensions.cs

@@ -141,14 +141,26 @@ public class IterationResult<T>
         public readonly ITransformer Model;
         public readonly Exception Exception;
         public readonly string TrainerName;
-        internal readonly Pipeline Pipeline;
+        public readonly int RuntimeInSeconds;
 
-        internal IterationResult(ITransformer model, T metrics, Pipeline pipeline, Exception exception)
+        internal readonly Pipeline Pipeline;
+        internal readonly int PipelineInferenceTimeInSeconds;
+
+        internal IterationResult(
+            ITransformer model,
+            T metrics,
+            Pipeline pipeline,
+            Exception exception,
+            int runtimeInSeconds,
+            int pipelineInferenceTimeInSeconds)
         {
             Model = model;
             Metrics = metrics;
             Pipeline = pipeline;
             Exception = exception;
+            RuntimeInSeconds = runtimeInSeconds;
+            PipelineInferenceTimeInSeconds = pipelineInferenceTimeInSeconds;
+
             TrainerName = pipeline?.Nodes.Where(n => n.NodeType == PipelineNodeType.Trainer).Last().Name;
         }
     }

src/AutoML/Assembly.cs

@@ -6,4 +6,5 @@
 
 [assembly: InternalsVisibleTo("Test, PublicKey=00240000048000009400000006020000002400005253413100040000010001004b86c4cb78549b34bab61a3b1800e23bfeb5b3ec390074041536a7e3cbd97f5f04cf0f857155a8928eaa29ebfd11cfbbad3ba70efea7bda3226c6a8d370a4cd303f714486b6ebc225985a638471e6ef571cc92a4613c00b8fa65d61ccee0cbe5f36330c9a01f4183559f1bef24cc2917c6d913e3a541333a1d05d9bed22b38cb")]
 [assembly: InternalsVisibleTo("mlnet, PublicKey=00240000048000009400000006020000002400005253413100040000010001004b86c4cb78549b34bab61a3b1800e23bfeb5b3ec390074041536a7e3cbd97f5f04cf0f857155a8928eaa29ebfd11cfbbad3ba70efea7bda3226c6a8d370a4cd303f714486b6ebc225985a638471e6ef571cc92a4613c00b8fa65d61ccee0cbe5f36330c9a01f4183559f1bef24cc2917c6d913e3a541333a1d05d9bed22b38cb")]
-[assembly: InternalsVisibleTo("mlnet.Test, PublicKey=00240000048000009400000006020000002400005253413100040000010001004b86c4cb78549b34bab61a3b1800e23bfeb5b3ec390074041536a7e3cbd97f5f04cf0f857155a8928eaa29ebfd11cfbbad3ba70efea7bda3226c6a8d370a4cd303f714486b6ebc225985a638471e6ef571cc92a4613c00b8fa65d61ccee0cbe5f36330c9a01f4183559f1bef24cc2917c6d913e3a541333a1d05d9bed22b38cb")]
+[assembly: InternalsVisibleTo("mlnet.Test, PublicKey=00240000048000009400000006020000002400005253413100040000010001004b86c4cb78549b34bab61a3b1800e23bfeb5b3ec390074041536a7e3cbd97f5f04cf0f857155a8928eaa29ebfd11cfbbad3ba70efea7bda3226c6a8d370a4cd303f714486b6ebc225985a638471e6ef571cc92a4613c00b8fa65d61ccee0cbe5f36330c9a01f4183559f1bef24cc2917c6d913e3a541333a1d05d9bed22b38cb")]
+[assembly: InternalsVisibleTo("Benchmark, PublicKey=00240000048000009400000006020000002400005253413100040000010001004b86c4cb78549b34bab61a3b1800e23bfeb5b3ec390074041536a7e3cbd97f5f04cf0f857155a8928eaa29ebfd11cfbbad3ba70efea7bda3226c6a8d370a4cd303f714486b6ebc225985a638471e6ef571cc92a4613c00b8fa65d61ccee0cbe5f36330c9a01f4183559f1bef24cc2917c6d913e3a541333a1d05d9bed22b38cb")]

src/AutoML/AutoFitter/AutoFitter.cs

@@ -67,26 +67,52 @@ public IEnumerable<IterationResult<T>> Fit()
 
             do
             {
-                // get next pipeline
-                var iterationsRemaining = (int)_settings.StoppingCriteria.MaxIterations - _history.Count;
-                var pipeline = PipelineSuggester.GetNextInferredPipeline(_history, columns, _task, iterationsRemaining, _optimizingMetricInfo.IsMaximizing);
+                SuggestedPipeline pipeline = null;
+                SuggestedPipelineResult<T> runResult = null;
 
-                // break if no candidates returned, means no valid pipeline available
-                if (pipeline == null)
+                try
                 {
-                    break;
-                }
+                    var iterationStopwatch = Stopwatch.StartNew();
+                    var getPiplelineStopwatch = Stopwatch.StartNew();
+
+                    // get next pipeline
+                    var iterationsRemaining = (int)_settings.StoppingCriteria.MaxIterations - _history.Count;
+                    pipeline = PipelineSuggester.GetNextInferredPipeline(_history, columns, _task, iterationsRemaining, _optimizingMetricInfo.IsMaximizing);
+
+                    getPiplelineStopwatch.Stop();
+
+                    // break if no candidates returned, means no valid pipeline available
+                    if (pipeline == null)
+                    {
+                        break;
+                    }
+
+                    // evaluate pipeline
+                    runResult = ProcessPipeline(pipeline);
 
-                // evaluate pipeline
-                SuggestedPipelineResult<T> runResult = ProcessPipeline(pipeline);
+                    if (preprocessorTransform != null)
+                    {
+                        runResult.Model = preprocessorTransform.Append(runResult.Model);
+                    }
 
-                if (preprocessorTransform != null)
+                    runResult.RuntimeInSeconds = (int)iterationStopwatch.Elapsed.TotalSeconds;
+                    runResult.GetPipelineTimeInSeconds = (int)getPiplelineStopwatch.Elapsed.TotalSeconds;
+                }
+                catch (Exception ex)
                 {
-                    runResult.Model = preprocessorTransform.Append(runResult.Model);
+                    WriteDebugLog(DebugStream.Exception, $"{pipeline?.Trainer} Crashed {ex}");
+
+                    if (runResult == null)
+                    {
+                        runResult = new SuggestedPipelineResult<T>(null, null, pipeline, -1, ex);
+                    }
+                    else
+                    {
+                        runResult = new SuggestedPipelineResult<T>(runResult.EvaluatedMetrics, runResult.Model, runResult.Pipeline, runResult.Score, ex);
+                    }
                 }
 
                 yield return runResult.ToIterationResult();
-
             } while (_history.Count < _settings.StoppingCriteria.MaxIterations &&
                     stopwatch.Elapsed.TotalMinutes < _settings.StoppingCriteria.TimeOutInMinutes);
         }
@@ -96,6 +122,10 @@ private SuggestedPipelineResult<T> ProcessPipeline(SuggestedPipeline pipeline)
             // run pipeline
             var stopwatch = Stopwatch.StartNew();
 
+            var commandLineStr = $"{string.Join(" xf=", pipeline.Transforms)} tr={pipeline.Trainer}";
+
+            WriteDebugLog(DebugStream.RunResult, $"Processing pipeline {commandLineStr}.");
+
             SuggestedPipelineResult<T> runResult;
             try
             {

src/AutoML/AutoFitter/SuggestedPipeline.cs

@@ -36,7 +36,7 @@ public SuggestedPipeline(IEnumerable<SuggestedTransform> transforms,
             }
         }
 
-        public override string ToString() => $"{Trainer}+{string.Join("+", Transforms.Select(t => t.ToString()))}";
+        public override string ToString() => $"{string.Join(" xf=", this.Transforms)} tr={this.Trainer}";
 
         public override bool Equals(object obj)
         {

src/AutoML/AutoFitter/SuggestedPipelineResult.cs

@@ -37,6 +37,9 @@ internal class SuggestedPipelineResult<T> : SuggestedPipelineResult
         public ITransformer Model { get; set; }
         public Exception Exception { get; set; }
 
+        public int RuntimeInSeconds { get; set; }
+        public int GetPipelineTimeInSeconds { get; set; }
+
         public SuggestedPipelineResult(T evaluatedMetrics, ITransformer model, SuggestedPipeline pipeline, double score, Exception exception)
             : base(pipeline, score, exception == null)
         {
@@ -47,7 +50,9 @@ public SuggestedPipelineResult(T evaluatedMetrics, ITransformer model, Suggested
 
         public IterationResult<T> ToIterationResult()
         {
-            return new IterationResult<T>(Model, EvaluatedMetrics, Pipeline.ToPipeline(), Exception);
+            var ir = new IterationResult<T>(Model, EvaluatedMetrics, Pipeline.ToPipeline(), Exception, RuntimeInSeconds, GetPipelineTimeInSeconds);
+
+            return ir;
         }
     }
 }

src/AutoML/PipelineSuggesters/PipelineSuggester.cs

@@ -6,6 +6,7 @@
 using System.Collections.Generic;
 using System.Linq;
 using Microsoft.Data.DataView;
+using Microsoft.ML.Data;
 
 namespace Microsoft.ML.Auto
 {
@@ -34,6 +35,7 @@ public static SuggestedPipeline GetNextInferredPipeline(IEnumerable<SuggestedPip
 
             var availableTrainers = RecipeInference.AllowedTrainers(context, task, history.Count() + iterationsRemaining);
             var transforms = CalculateTransforms(context, columns, task);
+            //var transforms = TransformInferenceApi.InferTransforms(context, columns, task);
 
             // if we haven't run all pipelines once
             if (history.Count() < availableTrainers.Count())
@@ -209,16 +211,16 @@ private static bool SampleHyperparameters(SuggestedTrainer trainer, IEnumerable<
             return true;
         }
 
-        private static IEnumerable<SuggestedTransform> CalculateTransforms(MLContext context,
+        private static IEnumerable<SuggestedTransform> CalculateTransforms(
+            MLContext context,
             (string, ColumnType, ColumnPurpose, ColumnDimensions)[] columns,
             TaskKind task)
         {
             var transforms = TransformInferenceApi.InferTransforms(context, columns).ToList();
             // this is a work-around for ML.NET bug tracked by https://github.com/dotnet/machinelearning/issues/1969
             if (task == TaskKind.MulticlassClassification)
             {
-                var labelCol = columns.First(c => c.Item3 == ColumnPurpose.Label).Item1;
-                var transform = ValueToKeyMappingExtension.CreateSuggestedTransform(context, labelCol, labelCol);
+                var transform = ValueToKeyMappingExtension.CreateSuggestedTransform(context, DefaultColumnNames.Label, DefaultColumnNames.Label);
                 transforms.Add(transform);
             }
             return transforms;

src/AutoML/TrainerExtensions/TrainerExtensionUtil.cs

@@ -173,7 +173,9 @@ public static void UpdateFields(object obj, IEnumerable<SweepableParam> sweepPar
                 }
                 catch (Exception)
                 {
-                    throw new InvalidOperationException("cannot set learner parameter");
+                    // TODO: uncomment when this is resolved: https://github.com/dotnet/machinelearning/issues/1983
+                    // making this masked since otherwise we can't use this learner at all
+                    // throw new InvalidOperationException("cannot set learner parameter");
                 }
             }
         }

src/AutoML/TransformInference/TransformInference.cs

@@ -173,7 +173,7 @@ public override IEnumerable<SuggestedTransform> Apply(IntermediateColumn[] colum
                         yield break;
 
                     var col = columns[lastLabelColId];
-                    
+
                     if (col.Type.IsText())
                     {
                         yield return ValueToKeyMappingExtension.CreateSuggestedTransform(Context, col.ColumnName, DefaultColumnNames.Label);