Skip to content

[AutoML] Early stopping in CLI based on the exploration time #3641

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 17 commits into from
May 2, 2019
Merged

[AutoML] Early stopping in CLI based on the exploration time #3641

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
3dd2e57
early stopping in CLI
srsaggam May 1, 2019
e53d60e
remove unused variables
srsaggam May 1, 2019
82eebf5
change back to thread
srsaggam May 2, 2019
b5ae9cc
remove sleep
srsaggam May 2, 2019
877be15
fix review comments
srsaggam May 2, 2019
d62e9f6
remove ununsed usings
srsaggam May 2, 2019
c3cd5a5
format message
srsaggam May 2, 2019
db81f5c
Merge remote-tracking branch 'prod/features/automl' into dev/srsaggam…
srsaggam May 2, 2019
f61335c
collapse declaration
srsaggam May 2, 2019
80989b2
remove unused param
srsaggam May 2, 2019
6000418
Merge remote-tracking branch 'prod/features/automl' into dev/srsaggam…
srsaggam May 2, 2019
12af51f
added environment.exit and removal of error message
srsaggam May 2, 2019
6a9694e
correction in message
srsaggam May 2, 2019
4a1e019
secs-> seconds
srsaggam May 2, 2019
ef73cd8
exit code
srsaggam May 2, 2019
0f409e3
change value to 1
srsaggam May 2, 2019
ab2688f
reverse the declaration
srsaggam May 2, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 27 additions & 0 deletions src/Microsoft.ML.AutoML/Utils/BestResultUtil.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,36 @@

using System.Collections.Generic;
using System.Linq;
using Microsoft.ML.Data;

namespace Microsoft.ML.AutoML
{
internal class BestResultUtil
{
public static RunDetail<BinaryClassificationMetrics> GetBestRun(IEnumerable<RunDetail<BinaryClassificationMetrics>> results,
BinaryClassificationMetric metric)
{
var metricsAgent = new BinaryMetricsAgent(null, metric);
var metricInfo = new OptimizingMetricInfo(metric);
return GetBestRun(results, metricsAgent, metricInfo.IsMaximizing);
}

public static RunDetail<RegressionMetrics> GetBestRun(IEnumerable<RunDetail<RegressionMetrics>> results,
RegressionMetric metric)
{
var metricsAgent = new RegressionMetricsAgent(null, metric);
var metricInfo = new OptimizingMetricInfo(metric);
return GetBestRun(results, metricsAgent, metricInfo.IsMaximizing);
}

public static RunDetail<MulticlassClassificationMetrics> GetBestRun(IEnumerable<RunDetail<MulticlassClassificationMetrics>> results,
MulticlassClassificationMetric metric)
{
var metricsAgent = new MultiMetricsAgent(null, metric);
var metricInfo = new OptimizingMetricInfo(metric);
return GetBestRun(results, metricsAgent, metricInfo.IsMaximizing);
}

public static RunDetail<TMetrics> GetBestRun<TMetrics>(IEnumerable<RunDetail<TMetrics>> results,
IMetricsAgent<TMetrics> metricsAgent, bool isMetricMaximizing)
{
Expand All @@ -29,6 +54,8 @@ public static CrossValidationRunDetail<TMetrics> GetBestRun<TMetrics>(IEnumerabl
return results.ElementAt(indexOfBestScore);
}



public static IEnumerable<(RunDetail<T>, int)> GetTopNRunResults<T>(IEnumerable<RunDetail<T>> results,
IMetricsAgent<T> metricsAgent, int n, bool isMetricMaximizing)
{
Expand Down
34 changes: 22 additions & 12 deletions src/mlnet/AutoML/AutoMLEngine.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,10 @@
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.ML.AutoML;
using Microsoft.ML.CLI.Data;
using Microsoft.ML.CLI.ShellProgressBar;
Expand Down Expand Up @@ -44,45 +47,52 @@ public ColumnInferenceResults InferColumns(MLContext context, ColumnInformation
return columnInference;
}

ExperimentResult<BinaryClassificationMetrics> IAutoMLEngine.ExploreBinaryClassificationModels(MLContext context, IDataView trainData, IDataView validationData, ColumnInformation columnInformation, BinaryClassificationMetric optimizationMetric, ProgressBar progressBar)
ExperimentResult<BinaryClassificationMetrics> IAutoMLEngine.ExploreBinaryClassificationModels(MLContext context, IDataView trainData, IDataView validationData, ColumnInformation columnInformation, BinaryClassificationMetric optimizationMetric, TimeSpan experimentTimeout, ProgressHandlers.BinaryClassificationHandler handler, ProgressBar progressBar)
{
var progressReporter = new ProgressHandlers.BinaryClassificationHandler(optimizationMetric, progressBar);
var result = context.Auto()
ExperimentResult<BinaryClassificationMetrics> result = null;
result = context.Auto()
.CreateBinaryClassificationExperiment(new BinaryExperimentSettings()
{
MaxExperimentTimeInSeconds = settings.MaxExplorationTime,
CacheBeforeTrainer = this.cacheBeforeTrainer,
OptimizingMetric = optimizationMetric
})
.Execute(trainData, validationData, columnInformation, progressHandler: progressReporter);
.Execute(trainData, validationData, columnInformation, progressHandler: handler);

//progressReporter.Stop();
logger.Log(LogLevel.Trace, Strings.RetrieveBestPipeline);
return result;
}

ExperimentResult<RegressionMetrics> IAutoMLEngine.ExploreRegressionModels(MLContext context, IDataView trainData, IDataView validationData, ColumnInformation columnInformation, RegressionMetric optimizationMetric, ProgressBar progressBar)
ExperimentResult<RegressionMetrics> IAutoMLEngine.ExploreRegressionModels(MLContext context, IDataView trainData, IDataView validationData, ColumnInformation columnInformation, RegressionMetric optimizationMetric, TimeSpan experimentTimeout, ProgressHandlers.RegressionHandler handler, ProgressBar progressBar)
{
var progressReporter = new ProgressHandlers.RegressionHandler(optimizationMetric, progressBar);
var result = context.Auto()
ExperimentResult<RegressionMetrics> result = null;
result = context.Auto()
.CreateRegressionExperiment(new RegressionExperimentSettings()
{
MaxExperimentTimeInSeconds = settings.MaxExplorationTime,
OptimizingMetric = optimizationMetric,
CacheBeforeTrainer = this.cacheBeforeTrainer
}).Execute(trainData, validationData, columnInformation, progressHandler: progressReporter);
}).Execute(trainData, validationData, columnInformation, progressHandler: handler);

//progressReporter.Stop();
logger.Log(LogLevel.Trace, Strings.RetrieveBestPipeline);
return result;
}

ExperimentResult<MulticlassClassificationMetrics> IAutoMLEngine.ExploreMultiClassificationModels(MLContext context, IDataView trainData, IDataView validationData, ColumnInformation columnInformation, MulticlassClassificationMetric optimizationMetric, ProgressBar progressBar)
ExperimentResult<MulticlassClassificationMetrics> IAutoMLEngine.ExploreMultiClassificationModels(MLContext context, IDataView trainData, IDataView validationData, ColumnInformation columnInformation, MulticlassClassificationMetric optimizationMetric, TimeSpan experimentTimeout, ProgressHandlers.MulticlassClassificationHandler handler, ProgressBar progressBar)
{
var progressReporter = new ProgressHandlers.MulticlassClassificationHandler(optimizationMetric, progressBar);
var result = context.Auto()
ExperimentResult<MulticlassClassificationMetrics> result = null;
result = context.Auto()
.CreateMulticlassClassificationExperiment(new MulticlassExperimentSettings()
{
MaxExperimentTimeInSeconds = settings.MaxExplorationTime,
CacheBeforeTrainer = this.cacheBeforeTrainer,
OptimizingMetric = optimizationMetric
}).Execute(trainData, validationData, columnInformation, progressHandler: progressReporter);
}).Execute(trainData, validationData, columnInformation, progressHandler: handler);

// wait for experiment time
//progressReporter.Stop();
logger.Log(LogLevel.Trace, Strings.RetrieveBestPipeline);
return result;
}
Expand Down
8 changes: 5 additions & 3 deletions src/mlnet/AutoML/IAutoMLEngine.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,11 @@
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

using System;
using System.Collections.Generic;
using Microsoft.ML.AutoML;
using Microsoft.ML.CLI.ShellProgressBar;
using Microsoft.ML.CLI.Utilities;
using Microsoft.ML.Data;

namespace Microsoft.ML.CLI.CodeGenerator
Expand All @@ -13,11 +15,11 @@ internal interface IAutoMLEngine
{
ColumnInferenceResults InferColumns(MLContext context, ColumnInformation columnInformation);

ExperimentResult<BinaryClassificationMetrics> ExploreBinaryClassificationModels(MLContext context, IDataView trainData, IDataView validationData, ColumnInformation columnInformation, BinaryClassificationMetric optimizationMetric, ProgressBar progressBar = null);
ExperimentResult<BinaryClassificationMetrics> ExploreBinaryClassificationModels(MLContext context, IDataView trainData, IDataView validationData, ColumnInformation columnInformation, BinaryClassificationMetric optimizationMetric, TimeSpan experimentTimeout, ProgressHandlers.BinaryClassificationHandler handler, ProgressBar progressBar = null);

ExperimentResult<MulticlassClassificationMetrics> ExploreMultiClassificationModels(MLContext context, IDataView trainData, IDataView validationData, ColumnInformation columnInformation, MulticlassClassificationMetric optimizationMetric, ProgressBar progressBar = null);
ExperimentResult<MulticlassClassificationMetrics> ExploreMultiClassificationModels(MLContext context, IDataView trainData, IDataView validationData, ColumnInformation columnInformation, MulticlassClassificationMetric optimizationMetric, TimeSpan experimentTimeout, ProgressHandlers.MulticlassClassificationHandler handler, ProgressBar progressBar = null);

ExperimentResult<RegressionMetrics> ExploreRegressionModels(MLContext context, IDataView trainData, IDataView validationData, ColumnInformation columnInformation, RegressionMetric optimizationMetric, ProgressBar progressBar = null);
ExperimentResult<RegressionMetrics> ExploreRegressionModels(MLContext context, IDataView trainData, IDataView validationData, ColumnInformation columnInformation, RegressionMetric optimizationMetric, TimeSpan experimentTimeout, ProgressHandlers.RegressionHandler handler, ProgressBar progressBar = null);

}
}
Loading