Autofit overloads + cancellation + progress callbacks

1) Introduce AutoFit overloads (basic and advanced)
2) AutoFit Cancellation
3) AutoFit progress callbacks

Author: vinodshanbhag

src/Microsoft.ML.Auto/API/AutoFitRunResult.cs

@@ -0,0 +1,40 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Linq;
+using Microsoft.ML.Core.Data;
+
+namespace Microsoft.ML.Auto
+{
+    public class AutoFitRunResult<T>
+    {
+        public readonly T Metrics;
+        public readonly ITransformer Model;
+        public readonly Exception Exception;
+        public readonly string TrainerName;
+        public readonly int RuntimeInSeconds;
+
+        internal readonly Pipeline Pipeline;
+        internal readonly int PipelineInferenceTimeInSeconds;
+
+        internal AutoFitRunResult(
+            ITransformer model,
+            T metrics,
+            Pipeline pipeline,
+            Exception exception,
+            int runtimeInSeconds,
+            int pipelineInferenceTimeInSeconds)
+        {
+            Model = model;
+            Metrics = metrics;
+            Pipeline = pipeline;
+            Exception = exception;
+            RuntimeInSeconds = runtimeInSeconds;
+            PipelineInferenceTimeInSeconds = pipelineInferenceTimeInSeconds;
+
+            TrainerName = pipeline?.Nodes.Where(n => n.NodeType == PipelineNodeType.Trainer).Last().Name;
+        }
+    }
+}

src/Microsoft.ML.Auto/API/AutoFitSettings.cs

@@ -9,7 +9,7 @@ namespace Microsoft.ML.Auto
 {
     internal static class AutoFitDefaults
     {
-        public const uint TimeOutInMinutes = 24 * 60;
+        public const uint TimeoutInSeconds = 60 * 60;
         public const uint MaxIterations = 1000;
     }
 
@@ -32,12 +32,12 @@ internal class AutoFitSettings
         internal bool DisableSubSampling;
         internal bool DisableCaching;
         internal bool ExternalizeTraining;
-        internal TraceLevel TraceLevel; 
+        internal TraceLevel TraceLevel;
     }
 
     internal class ExperimentStoppingCriteria
     {
-        public uint TimeOutInMinutes = AutoFitDefaults.TimeOutInMinutes;
+        public uint TimeoutInSeconds = AutoFitDefaults.TimeoutInSeconds;
         public uint MaxIterations = AutoFitDefaults.MaxIterations;
         internal bool StopAfterConverging;
         internal double ExperimentExitScore;

src/Microsoft.ML.Auto/API/BinaryClassificationExtension.cs

@@ -0,0 +1,87 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using Microsoft.Data.DataView;
+using Microsoft.ML.Core.Data;
+using Microsoft.ML.Data;
+
+namespace Microsoft.ML.Auto
+{
+    public class AutoFitBinaryClassificationOptions
+    {
+        public IDataView TrainData;
+        public string LabelColumnName = DefaultColumnNames.Label;
+        public IDataView ValidationData;
+        public uint TimeoutInSeconds = AutoFitDefaults.TimeoutInSeconds;
+        public CancellationToken CancellationToken = default;
+        public IProgress<AutoFitRunResult<BinaryClassificationMetrics>> ProgressCallback;
+        public IEstimator<ITransformer> PreFeaturizers;
+        public IEnumerable<(string, ColumnPurpose)> ColumnPurposes;
+    }
+
+    public static class BinaryClassificationExtensions
+    {
+        public static List<AutoFitRunResult<BinaryClassificationMetrics>> AutoFit(this BinaryClassificationCatalog catalog,
+            IDataView trainData,
+            string labelColumnName = DefaultColumnNames.Label,
+            IDataView validationData = null,
+            uint timeoutInSeconds = AutoFitDefaults.TimeoutInSeconds,
+            CancellationToken cancellationToken = default,
+            IProgress<AutoFitRunResult<BinaryClassificationMetrics>> progressCallback = null)
+        {
+            var settings = new AutoFitSettings();
+            settings.StoppingCriteria.TimeoutInSeconds = timeoutInSeconds;
+
+            return AutoFit(catalog, trainData, labelColumnName, validationData, settings,
+                null, null, cancellationToken, progressCallback, null);
+        }
+
+        public static List<AutoFitRunResult<BinaryClassificationMetrics>> AutoFit(this BinaryClassificationCatalog catalog,
+                AutoFitBinaryClassificationOptions options)
+        {
+            var settings = new AutoFitSettings();
+            settings.StoppingCriteria.TimeoutInSeconds = options.TimeoutInSeconds;
+
+            return AutoFit(catalog, options.TrainData, options.LabelColumnName, options.ValidationData, settings,
+                options.PreFeaturizers, options.ColumnPurposes, options.CancellationToken, options.ProgressCallback, null);
+        }
+
+        internal static List<AutoFitRunResult<BinaryClassificationMetrics>> AutoFit(this BinaryClassificationCatalog catalog,
+            IDataView trainData,
+            string labelColumnName = DefaultColumnNames.Label,
+            IDataView validationData = null,
+            AutoFitSettings settings = null,
+            IEstimator<ITransformer> preFeaturizers = null,
+            IEnumerable<(string, ColumnPurpose)> columnPurposes = null,
+            CancellationToken cancellationToken = default,
+            IProgress<AutoFitRunResult<BinaryClassificationMetrics>> progressCallback = null,
+            IDebugLogger debugLogger = null)
+        {
+            UserInputValidationUtil.ValidateAutoFitArgs(trainData, labelColumnName, validationData, settings, columnPurposes);
+
+            if (validationData == null)
+            {
+                (trainData, validationData) = catalog.TestValidateSplit(trainData);
+            }
+
+            // run autofit & get all pipelines run in that process
+            var autoFitter = new AutoFitter<BinaryClassificationMetrics>(TaskKind.BinaryClassification, trainData, labelColumnName, validationData,
+                settings, preFeaturizers, columnPurposes,
+                OptimizingMetric.RSquared, cancellationToken, progressCallback, debugLogger);
+
+            return autoFitter.Fit();
+        }
+
+        public static AutoFitRunResult<BinaryClassificationMetrics> Best(this IEnumerable<AutoFitRunResult<BinaryClassificationMetrics>> results)
+        {
+            double maxScore = results.Select(r => r.Metrics.Accuracy).Max();
+            return results.First(r => r.Metrics.Accuracy == maxScore);
+        }
+    }
+
+}