Misc fixes (dotnet#39)

daholste · web-flow · commit 2a638ccc2355 · 2019-01-28T23:50:57.000-08:00
* misc fixes -- fix bug where SMAC returning already-seen values; fix param encoding return bug in pipeline object model; nit clean-up AutoFit; return in pipeline suggester when sweeper has no next proposal; null ref fix in public object model pipeline suggester

* fix in BuildPipelineNodePropsLightGbm test, fix / use correct 'newTrainer' variable in PipelneSuggester

* SMAC perf improvement
diff --git a/src/AutoML/AutoFitter/AutoFitter.cs b/src/AutoML/AutoFitter/AutoFitter.cs
@@ -41,12 +41,6 @@ public AutoFitter(MLContext context, OptimizingMetricInfo metricInfo, AutoFitSet
         }
 
         public InferredPipelineRunResult[] Fit()
-        {
-            IteratePipelinesAndFit();
-            return _history.ToArray();
-        }
-
-        private void IteratePipelinesAndFit()
         {
             var stopwatch = Stopwatch.StartNew();
             var columns = AutoMlUtils.GetColumnInfoTuples(_context, _trainData, _label, _purposeOverrides);
@@ -68,6 +62,8 @@ private void IteratePipelinesAndFit()
 
             } while (_history.Count < _settings.StoppingCriteria.MaxIterations &&
                     stopwatch.Elapsed.TotalMinutes < _settings.StoppingCriteria.TimeOutInMinutes);
+
+            return _history.ToArray();
         }
 
         private void ProcessPipeline(InferredPipeline pipeline)
diff --git a/src/AutoML/PipelineSuggesters/PipelineSuggester.cs b/src/AutoML/PipelineSuggesters/PipelineSuggester.cs
@@ -21,7 +21,7 @@ public static Pipeline GetNextPipeline(IEnumerable<PipelineRunResult> history,
         {
             var inferredHistory = history.Select(r => InferredPipelineRunResult.FromPipelineRunResult(r));
             var nextInferredPipeline = GetNextInferredPipeline(inferredHistory, columns, task, iterationsRemaining, isMaximizingMetric);
-            return nextInferredPipeline.ToPipeline();
+            return nextInferredPipeline?.ToPipeline();
         }
 
         public static InferredPipeline GetNextInferredPipeline(IEnumerable<InferredPipelineRunResult> history,
@@ -47,21 +47,31 @@ public static InferredPipeline GetNextInferredPipeline(IEnumerable<InferredPipel
             // sort top trainers by # of times they've been run, from lowest to highest
             var orderedTopTrainers = OrderTrainersByNumTrials(history, topTrainers);
 
+            // keep as hashset of previously visited pipelines
+            var visitedPipelines = new HashSet<InferredPipeline>(history.Select(h => h.Pipeline));
+
             // iterate over top trainers (from least run to most run),
             // to find next pipeline
-            foreach(var trainer in orderedTopTrainers)
+            foreach (var trainer in orderedTopTrainers)
             {
                 var newTrainer = trainer.Clone();
 
-                // make sure we have not seen pipeline before.
                 // repeat until passes or runs out of chances
-                var visitedPipelines = new HashSet<InferredPipeline>(history.Select(h => h.Pipeline));
                 const int maxNumberAttempts = 10;
                 var count = 0;
                 do
                 {
-                    SampleHyperparameters(newTrainer, history, isMaximizingMetric);
+                    // sample new hyperparameters for the learner
+                    if (!SampleHyperparameters(newTrainer, history, isMaximizingMetric))
+                    {
+                        // if unable to sample new hyperparameters for the learner
+                        // (ie SMAC returned 0 suggestions), break
+                        break;
+                    }
+
                     var pipeline = new InferredPipeline(transforms, newTrainer);
+
+                    // make sure we have not seen pipeline before
                     if (!visitedPipelines.Contains(pipeline))
                     {
                         return pipeline;
@@ -169,7 +179,11 @@ private static IValueGenerator[] ConvertToValueGenerators(IEnumerable<SweepableP
             return results;
         }
 
-        private static void SampleHyperparameters(SuggestedTrainer trainer, IEnumerable<InferredPipelineRunResult> history, bool isMaximizingMetric)
+        /// <summary>
+        /// Samples new hyperparameters for the trainer, and sets them.
+        /// Returns true if success (new hyperparams were suggested and set). Else, returns false.
+        /// </summary>
+        private static bool SampleHyperparameters(SuggestedTrainer trainer, IEnumerable<InferredPipelineRunResult> history, bool isMaximizingMetric)
         {
             var sps = ConvertToValueGenerators(trainer.SweepParams);
             var sweeper = new SmacSweeper(
@@ -179,14 +193,20 @@ private static void SampleHyperparameters(SuggestedTrainer trainer, IEnumerable<
                 });
 
             IEnumerable<InferredPipelineRunResult> historyToUse = history
-                .Where(r => r.RunSucceded && r.Pipeline.Trainer.TrainerName == trainer.TrainerName && r.Pipeline.Trainer.HyperParamSet != null);
+                .Where(r => r.RunSucceded && r.Pipeline.Trainer.TrainerName == trainer.TrainerName && r.Pipeline.Trainer.HyperParamSet != null && r.Pipeline.Trainer.HyperParamSet.Any());
 
             // get new set of hyperparameter values
             var proposedParamSet = sweeper.ProposeSweeps(1, historyToUse.Select(h => h.ToRunResult(isMaximizingMetric))).First();
+            if(!proposedParamSet.Any())
+            {
+                return false;
+            }
 
             // associate proposed param set with trainer, so that smart hyperparam
             // sweepers (like KDO) can map them back.
             trainer.SetHyperparamValues(proposedParamSet);
+
+            return true;
         }
 
         private static IEnumerable<SuggestedTransform> CalculateTransforms(MLContext context,
diff --git a/src/AutoML/Sweepers/SmacSweeper.cs b/src/AutoML/Sweepers/SmacSweeper.cs
@@ -190,13 +190,17 @@ private ParameterSet[] GreedyPlusRandomSearch(ParameterSet[] parents, FastForest
             for (int i = 0; i < randomConfigs.Length; i++)
                 configurations.Add(new Tuple<double, ParameterSet>(randomEIs[i], randomConfigs[i]));
 
-            HashSet<ParameterSet> retainedConfigs = new HashSet<ParameterSet>();
             IOrderedEnumerable<Tuple<double, ParameterSet>> bestConfigurations = configurations.OrderByDescending(x => x.Item1);
 
-            foreach (Tuple<double, ParameterSet> t in bestConfigurations.Take(numOfCandidates))
-                retainedConfigs.Add(t.Item2);
+            var retainedConfigs = new HashSet<ParameterSet>(bestConfigurations.Select(x => x.Item2));
 
-            return retainedConfigs.ToArray();
+            // remove configurations matching previous run
+            foreach(var previousRun in previousRuns)
+            {
+                retainedConfigs.Remove(previousRun.ParameterSet);
+            }
+
+            return retainedConfigs.Take(numOfCandidates).ToArray();
         }
 
         /// <summary>
diff --git a/src/AutoML/Sweepers/SweeperBase.cs b/src/AutoML/Sweepers/SweeperBase.cs
@@ -44,7 +44,7 @@ protected SweeperBase(ArgumentsBase args, IValueGenerator[] sweepParameters, str
 
         public virtual ParameterSet[] ProposeSweeps(int maxSweeps, IEnumerable<IRunResult> previousRuns = null)
         {
-            var prevParamSets = previousRuns?.Select(r => r.ParameterSet).ToList() ?? new List<ParameterSet>();
+            var prevParamSets = new HashSet<ParameterSet>(previousRuns?.Select(r => r.ParameterSet).ToList() ?? new List<ParameterSet>());
             var result = new HashSet<ParameterSet>();
             for (int i = 0; i < maxSweeps; i++)
             {
@@ -66,9 +66,9 @@ public virtual ParameterSet[] ProposeSweeps(int maxSweeps, IEnumerable<IRunResul
 
         protected abstract ParameterSet CreateParamSet();
 
-        protected static bool AlreadyGenerated(ParameterSet paramSet, IEnumerable<ParameterSet> previousRuns)
+        protected static bool AlreadyGenerated(ParameterSet paramSet, ISet<ParameterSet> previousRuns)
         {
-            return previousRuns.Any(previousRun => previousRun.Equals(paramSet));
+            return previousRuns.Contains(paramSet);
         }
     }
 }
diff --git a/src/AutoML/TrainerExtensions/TrainerExtensionUtil.cs b/src/AutoML/TrainerExtensions/TrainerExtensionUtil.cs
@@ -83,18 +83,18 @@ public static IDictionary<string, object> BuildPipelineNodeProps(TrainerName tra
                 return BuildLightGbmPipelineNodeProps(sweepParams);
             }
 
-            return sweepParams.ToDictionary(p => p.Name, p => (object)p.RawValue);
+            return sweepParams.ToDictionary(p => p.Name, p => (object)p.ProcessedValue());
         }
 
         private static IDictionary<string, object> BuildLightGbmPipelineNodeProps(IEnumerable<SweepableParam> sweepParams)
         {
             var treeBoosterParams = sweepParams.Where(p => _lightGbmTreeBoosterParamNames.Contains(p.Name));
             var parentArgParams = sweepParams.Except(treeBoosterParams);
 
-            var treeBoosterProps = treeBoosterParams.ToDictionary(p => p.Name, p => (object)p.RawValue);
+            var treeBoosterProps = treeBoosterParams.ToDictionary(p => p.Name, p => (object)p.ProcessedValue());
             var treeBoosterCustomProp = new CustomProperty("Microsoft.ML.LightGBM.TreeBooster", treeBoosterProps);
 
-            var props = parentArgParams.ToDictionary(p => p.Name, p => (object)p.RawValue);
+            var props = parentArgParams.ToDictionary(p => p.Name, p => (object)p.ProcessedValue());
             props[LightGbmTreeBoosterPropName] = treeBoosterCustomProp;
             
             return props;
diff --git a/src/Test/GetNextPipelineTests.cs b/src/Test/GetNextPipelineTests.cs
@@ -50,6 +50,10 @@ public void GetNextPipelineMock()
             {
                 // get next pipeline
                 var pipeline = PipelineSuggester.GetNextPipeline(history, columns, TaskKind.BinaryClassification, maxIterations - i);
+                if(pipeline == null)
+                {
+                    break;
+                }
 
                 var result = new PipelineRunResult(pipeline, AutoMlUtils.Random.NextDouble(), true);
                 history.Add(result);
diff --git a/src/Test/InferredPipelineTests.cs b/src/Test/InferredPipelineTests.cs
@@ -0,0 +1,64 @@
+﻿// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Generic;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+namespace Microsoft.ML.Auto.Test
+{
+    [TestClass]
+    public class InferredPipelineTests
+    {
+        [TestMethod]
+        public void InferredPipelinesHashTest()
+        {
+            var context = new MLContext();
+
+            // test same learners with no hyperparams have the same hash code
+            var trainer1 = new SuggestedTrainer(context, new LightGbmBinaryExtension());
+            var trainer2 = new SuggestedTrainer(context, new LightGbmBinaryExtension());
+            var transforms1 = new List<SuggestedTransform>();
+            var transforms2 = new List<SuggestedTransform>();
+            var inferredPipeline1 = new InferredPipeline(transforms1, trainer1);
+            var inferredPipeline2 = new InferredPipeline(transforms2, trainer2);
+            Assert.AreEqual(inferredPipeline1.GetHashCode(), inferredPipeline2.GetHashCode());
+
+            // test same learners with hyperparams set vs empty hyperparams have different hash codes
+            var hyperparams1 = new ParameterSet(new List<IParameterValue>() { new LongParameterValue("NumLeaves", 2) });
+            trainer1 = new SuggestedTrainer(context, new LightGbmBinaryExtension(), hyperparams1);
+            trainer2 = new SuggestedTrainer(context, new LightGbmBinaryExtension());
+            inferredPipeline1 = new InferredPipeline(transforms1, trainer1);
+            inferredPipeline2 = new InferredPipeline(transforms2, trainer2);
+            Assert.AreNotEqual(inferredPipeline1.GetHashCode(), inferredPipeline2.GetHashCode());
+
+            // same learners with different hyperparams
+            hyperparams1 = new ParameterSet(new List<IParameterValue>() { new LongParameterValue("NumLeaves", 2) });
+            var hyperparams2 = new ParameterSet(new List<IParameterValue>() { new LongParameterValue("NumLeaves", 6) });
+            trainer1 = new SuggestedTrainer(context, new LightGbmBinaryExtension(), hyperparams1);
+            trainer2 = new SuggestedTrainer(context, new LightGbmBinaryExtension(), hyperparams2);
+            inferredPipeline1 = new InferredPipeline(transforms1, trainer1);
+            inferredPipeline2 = new InferredPipeline(transforms2, trainer2);
+            Assert.AreNotEqual(inferredPipeline1.GetHashCode(), inferredPipeline2.GetHashCode());
+
+            // same learners with same transforms
+            trainer1 = new SuggestedTrainer(context, new LightGbmBinaryExtension());
+            trainer2 = new SuggestedTrainer(context, new LightGbmBinaryExtension());
+            transforms1 = new List<SuggestedTransform>() { ColumnConcatenatingExtension.CreateSuggestedTransform(context, new[] { "In" }, "Out") };
+            transforms2 = new List<SuggestedTransform>() { ColumnConcatenatingExtension.CreateSuggestedTransform(context, new[] { "In" }, "Out") };
+            inferredPipeline1 = new InferredPipeline(transforms1, trainer1);
+            inferredPipeline2 = new InferredPipeline(transforms2, trainer2);
+            Assert.AreEqual(inferredPipeline1.GetHashCode(), inferredPipeline2.GetHashCode());
+
+            // same transforms with different learners
+            trainer1 = new SuggestedTrainer(context, new SdcaBinaryExtension());
+            trainer2 = new SuggestedTrainer(context, new LightGbmBinaryExtension());
+            transforms1 = new List<SuggestedTransform>() { ColumnConcatenatingExtension.CreateSuggestedTransform(context, new[] { "In" }, "Out") };
+            transforms2 = new List<SuggestedTransform>() { ColumnConcatenatingExtension.CreateSuggestedTransform(context, new[] { "In" }, "Out") };
+            inferredPipeline1 = new InferredPipeline(transforms1, trainer1);
+            inferredPipeline2 = new InferredPipeline(transforms2, trainer2);
+            Assert.AreNotEqual(inferredPipeline1.GetHashCode(), inferredPipeline2.GetHashCode());
+        }
+    }
+}
diff --git a/src/Test/TrainerExtensionsTests.cs b/src/Test/TrainerExtensionsTests.cs
@@ -63,22 +63,22 @@ public void BuildPipelineNodePropsLightGbm()
 
             var expectedJson = @"
 {
-  ""NumBoostRound"": 1,
+  ""NumBoostRound"": 20,
   ""LearningRate"": 1,
   ""NumLeaves"": 1,
-  ""MinDataPerLeaf"": 1,
-  ""UseSoftmax"": 1,
-  ""UseCat"": 1,
-  ""UseMissing"": 1,
-  ""MinDataPerGroup"": 1,
-  ""MaxCatThreshold"": 1,
-  ""CatSmooth"": 1,
-  ""CatL2"": 1,
+  ""MinDataPerLeaf"": 10,
+  ""UseSoftmax"": false,
+  ""UseCat"": false,
+  ""UseMissing"": false,
+  ""MinDataPerGroup"": 50,
+  ""MaxCatThreshold"": 16,
+  ""CatSmooth"": 10,
+  ""CatL2"": 0.5,
   ""TreeBooster"": {
     ""Name"": ""Microsoft.ML.LightGBM.TreeBooster"",
     ""Properties"": {
-      ""RegLambda"": 1,
-      ""RegAlpha"": 1
+      ""RegLambda"": 0.5,
+      ""RegAlpha"": 0.5
     }
   }
 }";
@@ -99,12 +99,12 @@ public void BuildPipelineNodePropsSdca()
             var sdcaBinaryProps = TrainerExtensionUtil.BuildPipelineNodeProps(TrainerName.SdcaBinary, sweepParams);
             var expectedJson = @"
 {
-  ""L2Const"": 1,
-  ""L1Threshold"": 1,
-  ""ConvergenceTolerance"": 1,
-  ""MaxIterations"": 1,
-  ""Shuffle"": 1,
-  ""BiasLearningRate"": 1
+  ""L2Const"": 1E-07,
+  ""L1Threshold"": 0.0,
+  ""ConvergenceTolerance"": 0.01,
+  ""MaxIterations"": 10,
+  ""Shuffle"": true,
+  ""BiasLearningRate"": 0.01
 }";
             Util.AssertObjectMatchesJson(expectedJson, sdcaBinaryProps);
         }

Original file line number	Diff line number	Diff line change
`@@ -41,12 +41,6 @@ public AutoFitter(MLContext context, OptimizingMetricInfo metricInfo, AutoFitSet`
`41`	`41`	`}`
`42`	`42`
`43`	`43`	`public InferredPipelineRunResult[] Fit()`
`44`		`- {`
`45`		`- IteratePipelinesAndFit();`
`46`		`- return _history.ToArray();`
`47`		`- }`
`48`		`-`
`49`		`- private void IteratePipelinesAndFit()`
`50`	`44`	`{`
`51`	`45`	`var stopwatch = Stopwatch.StartNew();`
`52`	`46`	`var columns = AutoMlUtils.GetColumnInfoTuples(_context, _trainData, _label, _purposeOverrides);`
`@@ -68,6 +62,8 @@ private void IteratePipelinesAndFit()`
`68`	`62`
`69`	`63`	`} while (_history.Count < _settings.StoppingCriteria.MaxIterations &&`
`70`	`64`	`stopwatch.Elapsed.TotalMinutes < _settings.StoppingCriteria.TimeOutInMinutes);`
	`65`	`+`
	`66`	`+ return _history.ToArray();`
`71`	`67`	`}`
`72`	`68`
`73`	`69`	`private void ProcessPipeline(InferredPipeline pipeline)`
Original file line number	Diff line number	Diff line change
`@@ -44,7 +44,7 @@ protected SweeperBase(ArgumentsBase args, IValueGenerator[] sweepParameters, str`
`44`	`44`
`45`	`45`	`public virtual ParameterSet[] ProposeSweeps(int maxSweeps, IEnumerable<IRunResult> previousRuns = null)`
`46`	`46`	`{`
`47`		`- var prevParamSets = previousRuns?.Select(r => r.ParameterSet).ToList() ?? new List<ParameterSet>();`
	`47`	`+ var prevParamSets = new HashSet<ParameterSet>(previousRuns?.Select(r => r.ParameterSet).ToList() ?? new List<ParameterSet>());`
`48`	`48`	`var result = new HashSet<ParameterSet>();`
`49`	`49`	`for (int i = 0; i < maxSweeps; i++)`
`50`	`50`	`{`
`@@ -66,9 +66,9 @@ public virtual ParameterSet[] ProposeSweeps(int maxSweeps, IEnumerable<IRunResul`
`66`	`66`
`67`	`67`	`protected abstract ParameterSet CreateParamSet();`
`68`	`68`
`69`		`- protected static bool AlreadyGenerated(ParameterSet paramSet, IEnumerable<ParameterSet> previousRuns)`
	`69`	`+ protected static bool AlreadyGenerated(ParameterSet paramSet, ISet<ParameterSet> previousRuns)`
`70`	`70`	`{`
`71`		`- return previousRuns.Any(previousRun => previousRun.Equals(paramSet));`
	`71`	`+ return previousRuns.Contains(paramSet);`
`72`	`72`	`}`
`73`	`73`	`}`
`74`	`74`	`}`
Original file line number	Diff line number	Diff line change
`@@ -83,18 +83,18 @@ public static IDictionary<string, object> BuildPipelineNodeProps(TrainerName tra`
`83`	`83`	`return BuildLightGbmPipelineNodeProps(sweepParams);`
`84`	`84`	`}`
`85`	`85`
`86`		`- return sweepParams.ToDictionary(p => p.Name, p => (object)p.RawValue);`
	`86`	`+ return sweepParams.ToDictionary(p => p.Name, p => (object)p.ProcessedValue());`
`87`	`87`	`}`
`88`	`88`
`89`	`89`	`private static IDictionary<string, object> BuildLightGbmPipelineNodeProps(IEnumerable<SweepableParam> sweepParams)`
`90`	`90`	`{`
`91`	`91`	`var treeBoosterParams = sweepParams.Where(p => _lightGbmTreeBoosterParamNames.Contains(p.Name));`
`92`	`92`	`var parentArgParams = sweepParams.Except(treeBoosterParams);`
`93`	`93`
`94`		`- var treeBoosterProps = treeBoosterParams.ToDictionary(p => p.Name, p => (object)p.RawValue);`
	`94`	`+ var treeBoosterProps = treeBoosterParams.ToDictionary(p => p.Name, p => (object)p.ProcessedValue());`
`95`	`95`	`var treeBoosterCustomProp = new CustomProperty("Microsoft.ML.LightGBM.TreeBooster", treeBoosterProps);`
`96`	`96`
`97`		`- var props = parentArgParams.ToDictionary(p => p.Name, p => (object)p.RawValue);`
	`97`	`+ var props = parentArgParams.ToDictionary(p => p.Name, p => (object)p.ProcessedValue());`
`98`	`98`	`props[LightGbmTreeBoosterPropName] = treeBoosterCustomProp;`
`99`	`99`
`100`	`100`	`return props;`