Exit if perfect model produced (dotnet#220)

daholste · web-flow · commit e52d7f0a5bef · 2019-02-26T11:07:18.000-08:00
diff --git a/src/Microsoft.ML.Auto/Experiment/Experiment.cs b/src/Microsoft.ML.Auto/Experiment/Experiment.cs
@@ -114,6 +114,13 @@ public List<RunResult<T>> Execute()
                 var iterationResult = runResult.ToIterationResult();
                 ReportProgress(iterationResult);
                 iterationResults.Add(iterationResult);
+
+                // if model is perfect, break
+                if (_metricsAgent.IsModelPerfect(iterationResult.Metrics))
+                {
+                    break;
+                }
+
             } while (_history.Count < _experimentSettings.MaxModels &&
                     !_experimentSettings.CancellationToken.IsCancellationRequested &&
                     stopwatch.Elapsed.TotalSeconds < _experimentSettings.MaxInferenceTimeInSeconds);
diff --git a/src/Microsoft.ML.Auto/Experiment/MetricsAgents/BinaryMetricsAgent.cs b/src/Microsoft.ML.Auto/Experiment/MetricsAgents/BinaryMetricsAgent.cs
@@ -2,7 +2,6 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
-using System;
 using Microsoft.ML.Data;
 
 namespace Microsoft.ML.Auto
@@ -36,10 +35,39 @@ public double GetScore(BinaryClassificationMetrics metrics)
                     return metrics.PositivePrecision;
                 case BinaryClassificationMetric.PositiveRecall:
                     return metrics.PositiveRecall;
+                default:
+                    throw MetricsAgentUtil.BuildMetricNotSupportedException(_optimizingMetric);
+            }
+        }
+
+        public bool IsModelPerfect(BinaryClassificationMetrics metrics)
+        {
+            if (metrics == null)
+            {
+                return false;
             }
 
-            // never expected to reach here
-            throw new NotSupportedException($"{_optimizingMetric} is not a supported sweep metric");
+            switch (_optimizingMetric)
+            {
+                case BinaryClassificationMetric.Accuracy:
+                    return metrics.Accuracy == 1;
+                case BinaryClassificationMetric.Auc:
+                    return metrics.Auc == 1;
+                case BinaryClassificationMetric.Auprc:
+                    return metrics.Auprc == 1;
+                case BinaryClassificationMetric.F1Score:
+                    return metrics.F1Score == 1;
+                case BinaryClassificationMetric.NegativePrecision:
+                    return metrics.NegativePrecision == 1;
+                case BinaryClassificationMetric.NegativeRecall:
+                    return metrics.NegativeRecall == 1;
+                case BinaryClassificationMetric.PositivePrecision:
+                    return metrics.PositivePrecision == 1;
+                case BinaryClassificationMetric.PositiveRecall:
+                    return metrics.PositiveRecall == 1;
+                default:
+                    throw MetricsAgentUtil.BuildMetricNotSupportedException(_optimizingMetric);
+            }
         }
     }
 }
diff --git a/src/Microsoft.ML.Auto/Experiment/MetricsAgents/IMetricsAgent.cs b/src/Microsoft.ML.Auto/Experiment/MetricsAgents/IMetricsAgent.cs
@@ -7,5 +7,7 @@ namespace Microsoft.ML.Auto
     internal interface IMetricsAgent<T>
     {
         double GetScore(T metrics);
+
+        bool IsModelPerfect(T metrics);
     }
 }
diff --git a/src/Microsoft.ML.Auto/Experiment/MetricsAgents/MetricsAgentUtil.cs b/src/Microsoft.ML.Auto/Experiment/MetricsAgents/MetricsAgentUtil.cs
@@ -0,0 +1,16 @@
+﻿// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+
+namespace Microsoft.ML.Auto
+{
+    internal static class MetricsAgentUtil
+    {
+        public static NotSupportedException BuildMetricNotSupportedException<T>(T optimizingMetric)
+        {
+            return new NotSupportedException($"{optimizingMetric} is not a supported sweep metric");
+        }
+    }
+}
diff --git a/src/Microsoft.ML.Auto/Experiment/MetricsAgents/MultiMetricsAgent.cs b/src/Microsoft.ML.Auto/Experiment/MetricsAgents/MultiMetricsAgent.cs
@@ -2,7 +2,6 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
-using System;
 using Microsoft.ML.Data;
 
 namespace Microsoft.ML.Auto
@@ -30,10 +29,33 @@ public double GetScore(MultiClassClassifierMetrics metrics)
                     return metrics.LogLossReduction;
                 case MulticlassClassificationMetric.TopKAccuracy:
                     return metrics.TopKAccuracy;
+                default:
+                    throw MetricsAgentUtil.BuildMetricNotSupportedException(_optimizingMetric);
             }
+        }
 
-            // never expected to reach here
-            throw new NotSupportedException($"{_optimizingMetric} is not a supported sweep metric");
+        public bool IsModelPerfect(MultiClassClassifierMetrics metrics)
+        {
+            if (metrics == null)
+            {
+                return false;
+            }
+
+            switch (_optimizingMetric)
+            {
+                case MulticlassClassificationMetric.AccuracyMacro:
+                    return metrics.AccuracyMacro == 1;
+                case MulticlassClassificationMetric.AccuracyMicro:
+                    return metrics.AccuracyMicro == 1;
+                case MulticlassClassificationMetric.LogLoss:
+                    return metrics.LogLoss == 0;
+                case MulticlassClassificationMetric.LogLossReduction:
+                    return metrics.LogLossReduction == 1;
+                case MulticlassClassificationMetric.TopKAccuracy:
+                    return metrics.TopKAccuracy == 1;
+                default:
+                    throw MetricsAgentUtil.BuildMetricNotSupportedException(_optimizingMetric);
+            }
         }
     }
-}
+}
diff --git a/src/Microsoft.ML.Auto/Experiment/MetricsAgents/RegressionMetricsAgent.cs b/src/Microsoft.ML.Auto/Experiment/MetricsAgents/RegressionMetricsAgent.cs
@@ -2,7 +2,6 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
-using System;
 using Microsoft.ML.Data;
 
 namespace Microsoft.ML.Auto
@@ -28,10 +27,31 @@ public double GetScore(RegressionMetrics metrics)
                     return metrics.Rms;
                 case RegressionMetric.RSquared:
                     return metrics.RSquared;
+                default:
+                    throw MetricsAgentUtil.BuildMetricNotSupportedException(_optimizingMetric);
+            }
+        }
+
+        public bool IsModelPerfect(RegressionMetrics metrics)
+        {
+            if (metrics == null)
+            {
+                return false;
             }
 
-            // never expected to reach here
-            throw new NotSupportedException($"{_optimizingMetric} is not a supported sweep metric");
+            switch (_optimizingMetric)
+            {
+                case RegressionMetric.L1:
+                    return metrics.L1 == 0;
+                case RegressionMetric.L2:
+                    return metrics.L2 == 0;
+                case RegressionMetric.Rms:
+                    return metrics.Rms == 0;
+                case RegressionMetric.RSquared:
+                    return metrics.RSquared == 1;
+                default:
+                    throw MetricsAgentUtil.BuildMetricNotSupportedException(_optimizingMetric);
+            }
         }
     }
 }
diff --git a/src/Test/MetricsAgentsTests.cs b/src/Test/MetricsAgentsTests.cs
@@ -0,0 +1,167 @@
+﻿// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Reflection;
+using Microsoft.ML.Data;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+namespace Microsoft.ML.Auto.Test
+{
+    [TestClass]
+    public class MetricsAgentsTests
+    {
+        [TestMethod]
+        public void BinaryMetricsGetScoreTest()
+        {
+            var metrics = CreateInstance<BinaryClassificationMetrics>(0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8);
+            Assert.AreEqual(0.1, GetScore(metrics, BinaryClassificationMetric.Auc));
+            Assert.AreEqual(0.2, GetScore(metrics, BinaryClassificationMetric.Accuracy));
+            Assert.AreEqual(0.3, GetScore(metrics, BinaryClassificationMetric.PositivePrecision));
+            Assert.AreEqual(0.4, GetScore(metrics, BinaryClassificationMetric.PositiveRecall));
+            Assert.AreEqual(0.5, GetScore(metrics, BinaryClassificationMetric.NegativePrecision));
+            Assert.AreEqual(0.6, GetScore(metrics, BinaryClassificationMetric.NegativeRecall));
+            Assert.AreEqual(0.7, GetScore(metrics, BinaryClassificationMetric.F1Score));
+            Assert.AreEqual(0.8, GetScore(metrics, BinaryClassificationMetric.Auprc));
+        }
+
+        [TestMethod]
+        public void BinaryMetricsNonPerfectTest()
+        {
+            var metrics = CreateInstance<BinaryClassificationMetrics>(0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8);
+            Assert.AreEqual(false, IsPerfectModel(metrics, BinaryClassificationMetric.Accuracy));
+            Assert.AreEqual(false, IsPerfectModel(metrics, BinaryClassificationMetric.Auc));
+            Assert.AreEqual(false, IsPerfectModel(metrics, BinaryClassificationMetric.Auprc));
+            Assert.AreEqual(false, IsPerfectModel(metrics, BinaryClassificationMetric.F1Score));
+            Assert.AreEqual(false, IsPerfectModel(metrics, BinaryClassificationMetric.NegativePrecision));
+            Assert.AreEqual(false, IsPerfectModel(metrics, BinaryClassificationMetric.NegativeRecall));
+            Assert.AreEqual(false, IsPerfectModel(metrics, BinaryClassificationMetric.PositivePrecision));
+            Assert.AreEqual(false, IsPerfectModel(metrics, BinaryClassificationMetric.PositiveRecall));
+        }
+
+        [TestMethod]
+        public void BinaryMetricsPerfectTest()
+        {
+            var metrics = CreateInstance<BinaryClassificationMetrics>(1, 1, 1, 1, 1, 1, 1, 1);
+            Assert.AreEqual(true, IsPerfectModel(metrics, BinaryClassificationMetric.Accuracy));
+            Assert.AreEqual(true, IsPerfectModel(metrics, BinaryClassificationMetric.Auc));
+            Assert.AreEqual(true, IsPerfectModel(metrics, BinaryClassificationMetric.Auprc));
+            Assert.AreEqual(true, IsPerfectModel(metrics, BinaryClassificationMetric.F1Score));
+            Assert.AreEqual(true, IsPerfectModel(metrics, BinaryClassificationMetric.NegativePrecision));
+            Assert.AreEqual(true, IsPerfectModel(metrics, BinaryClassificationMetric.NegativeRecall));
+            Assert.AreEqual(true, IsPerfectModel(metrics, BinaryClassificationMetric.PositivePrecision));
+            Assert.AreEqual(true, IsPerfectModel(metrics, BinaryClassificationMetric.PositiveRecall));
+        }
+
+        [TestMethod]
+        public void MulticlassMetricsGetScoreTest()
+        {
+            var metrics = CreateInstance<MultiClassClassifierMetrics>(0.1, 0.2, 0.3, 0.4, 0, 0.5, new double[] {});
+            Assert.AreEqual(0.1, GetScore(metrics, MulticlassClassificationMetric.AccuracyMicro));
+            Assert.AreEqual(0.2, GetScore(metrics, MulticlassClassificationMetric.AccuracyMacro));
+            Assert.AreEqual(0.3, GetScore(metrics, MulticlassClassificationMetric.LogLoss));
+            Assert.AreEqual(0.4, GetScore(metrics, MulticlassClassificationMetric.LogLossReduction));
+            Assert.AreEqual(0.5, GetScore(metrics, MulticlassClassificationMetric.TopKAccuracy));
+        }
+
+        [TestMethod]
+        public void MulticlassMetricsNonPerfectTest()
+        {
+            var metrics = CreateInstance<MultiClassClassifierMetrics>(0.1, 0.2, 0.3, 0.4, 0, 0.5, new double[] { });
+            Assert.AreEqual(false, IsPerfectModel(metrics, MulticlassClassificationMetric.AccuracyMacro));
+            Assert.AreEqual(false, IsPerfectModel(metrics, MulticlassClassificationMetric.AccuracyMicro));
+            Assert.AreEqual(false, IsPerfectModel(metrics, MulticlassClassificationMetric.LogLoss));
+            Assert.AreEqual(false, IsPerfectModel(metrics, MulticlassClassificationMetric.LogLossReduction));
+            Assert.AreEqual(false, IsPerfectModel(metrics, MulticlassClassificationMetric.TopKAccuracy));
+        }
+
+        [TestMethod]
+        public void MulticlassMetricsPerfectTest()
+        {
+            var metrics = CreateInstance<MultiClassClassifierMetrics>(1, 1, 0, 1, 0, 1, new double[] { });
+            Assert.AreEqual(true, IsPerfectModel(metrics, MulticlassClassificationMetric.AccuracyMicro));
+            Assert.AreEqual(true, IsPerfectModel(metrics, MulticlassClassificationMetric.AccuracyMacro));
+            Assert.AreEqual(true, IsPerfectModel(metrics, MulticlassClassificationMetric.LogLoss));
+            Assert.AreEqual(true, IsPerfectModel(metrics, MulticlassClassificationMetric.LogLossReduction));
+            Assert.AreEqual(true, IsPerfectModel(metrics, MulticlassClassificationMetric.TopKAccuracy));
+        }
+
+        [TestMethod]
+        public void RegressionMetricsGetScoreTest()
+        {
+            var metrics = CreateInstance<RegressionMetrics>(0.2, 0.3, 0.4, 0.5, 0.6);
+            Assert.AreEqual(0.2, GetScore(metrics, RegressionMetric.L1));
+            Assert.AreEqual(0.3, GetScore(metrics, RegressionMetric.L2));
+            Assert.AreEqual(0.4, GetScore(metrics, RegressionMetric.Rms));
+            Assert.AreEqual(0.6, GetScore(metrics, RegressionMetric.RSquared));
+        }
+
+        [TestMethod]
+        public void RegressionMetricsNonPerfectTest()
+        {
+            var metrics = CreateInstance<RegressionMetrics>(0.2, 0.3, 0.4, 0.5, 0.6);
+            Assert.AreEqual(false, IsPerfectModel(metrics, RegressionMetric.L1));
+            Assert.AreEqual(false, IsPerfectModel(metrics, RegressionMetric.L2));
+            Assert.AreEqual(false, IsPerfectModel(metrics, RegressionMetric.Rms));
+            Assert.AreEqual(false, IsPerfectModel(metrics, RegressionMetric.RSquared));
+        }
+
+        [TestMethod]
+        public void RegressionMetricsPerfectTest()
+        {
+            var metrics = CreateInstance<RegressionMetrics>(0, 0, 0, 0, 1);
+            Assert.AreEqual(true, IsPerfectModel(metrics, RegressionMetric.L1));
+            Assert.AreEqual(true, IsPerfectModel(metrics, RegressionMetric.L2));
+            Assert.AreEqual(true, IsPerfectModel(metrics, RegressionMetric.Rms));
+            Assert.AreEqual(true, IsPerfectModel(metrics, RegressionMetric.RSquared));
+        }
+
+        [TestMethod]
+        [ExpectedException(typeof(NotSupportedException))]
+        public void ThrowNotSupportedMetricException()
+        {
+            throw MetricsAgentUtil.BuildMetricNotSupportedException(BinaryClassificationMetric.Accuracy);
+        }
+
+        private static T CreateInstance<T>(params object[] args)
+        {
+            var type = typeof(T);
+            var instance = type.Assembly.CreateInstance(
+                type.FullName, false,
+                BindingFlags.Instance | BindingFlags.NonPublic,
+                null, args, null, null);
+            return (T)instance;
+        }
+
+        private static double GetScore(BinaryClassificationMetrics metrics, BinaryClassificationMetric metric)
+        {
+            return new BinaryMetricsAgent(metric).GetScore(metrics);
+        }
+
+        private static double GetScore(MultiClassClassifierMetrics metrics, MulticlassClassificationMetric metric)
+        {
+            return new MultiMetricsAgent(metric).GetScore(metrics);
+        }
+
+        private static double GetScore(RegressionMetrics metrics, RegressionMetric metric)
+        {
+            return new RegressionMetricsAgent(metric).GetScore(metrics);
+        }
+
+        private static bool IsPerfectModel(BinaryClassificationMetrics metrics, BinaryClassificationMetric metric)
+        {
+            return new BinaryMetricsAgent(metric).IsModelPerfect(metrics);
+        }
+
+        private static bool IsPerfectModel(MultiClassClassifierMetrics metrics, MulticlassClassificationMetric metric)
+        {
+            return new MultiMetricsAgent(metric).IsModelPerfect(metrics);
+        }
+
+        private static bool IsPerfectModel(RegressionMetrics metrics, RegressionMetric metric)
+        {
+            return new RegressionMetricsAgent(metric).IsModelPerfect(metrics);
+        }
+    }
+}

Original file line number	Diff line number	Diff line change
`@@ -7,5 +7,7 @@ namespace Microsoft.ML.Auto`
`7`	`7`	`internal interface IMetricsAgent<T>`
`8`	`8`	`{`
`9`	`9`	`double GetScore(T metrics);`
	`10`	`+`
	`11`	`+ bool IsModelPerfect(T metrics);`
`10`	`12`	`}`
`11`	`13`	`}`