Null reference exception fix for finding best model when some runs have failed (dotnet#239)

Author: daholste

src/Microsoft.ML.Auto/API/BinaryClassificationExperiment.cs

@@ -98,8 +98,7 @@ public static class BinaryExperimentResultExtensions
         public static RunResult<BinaryClassificationMetrics> Best(this IEnumerable<RunResult<BinaryClassificationMetrics>> results, BinaryClassificationMetric metric = BinaryClassificationMetric.Accuracy)
         {
             var metricsAgent = new BinaryMetricsAgent(metric);
-            double maxScore = results.Select(r => metricsAgent.GetScore(r.Metrics)).Max();
-            return results.First(r => metricsAgent.GetScore(r.Metrics) == maxScore);
+            return RunResultUtil.GetBestRunResult(results, metricsAgent);
         }
     }
 }

src/Microsoft.ML.Auto/API/MulticlassClassificationExperiment.cs

@@ -96,8 +96,7 @@ public static class MulticlassExperimentResultExtensions
         public static RunResult<MultiClassClassifierMetrics> Best(this IEnumerable<RunResult<MultiClassClassifierMetrics>> results, MulticlassClassificationMetric metric = MulticlassClassificationMetric.AccuracyMicro)
         {
             var metricsAgent = new MultiMetricsAgent(metric);
-            double maxScore = results.Select(r => metricsAgent.GetScore(r.Metrics)).Max();
-            return results.First(r => metricsAgent.GetScore(r.Metrics) == maxScore);
+            return RunResultUtil.GetBestRunResult(results, metricsAgent);
         }
     }
 }

src/Microsoft.ML.Auto/API/RegressionExperiment.cs

@@ -93,8 +93,7 @@ public static class RegressionExperimentResultExtensions
         public static RunResult<RegressionMetrics> Best(this IEnumerable<RunResult<RegressionMetrics>> results, RegressionMetric metric = RegressionMetric.RSquared)
         {
             var metricsAgent = new RegressionMetricsAgent(metric);
-            double maxScore = results.Select(r => metricsAgent.GetScore(r.Metrics)).Max();
-            return results.First(r => metricsAgent.GetScore(r.Metrics) == maxScore);
+            return RunResultUtil.GetBestRunResult(results, metricsAgent);
         }
     }
 }

src/Microsoft.ML.Auto/Utils/RunResultUtil.cs

@@ -0,0 +1,21 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Collections.Generic;
+using System.Linq;
+
+namespace Microsoft.ML.Auto
+{
+    internal class RunResultUtil
+    {
+        public static RunResult<T> GetBestRunResult<T>(IEnumerable<RunResult<T>> results,
+            IMetricsAgent<T> metricsAgent)
+        {
+            results = results.Where(r => r.Metrics != null);
+            if (!results.Any()) { return null; }
+            double maxScore = results.Select(r => metricsAgent.GetScore(r.Metrics)).Max();
+            return results.First(r => metricsAgent.GetScore(r.Metrics) == maxScore);
+        }
+    }
+}

src/Test/MetricsAgentsTests.cs

@@ -3,7 +3,6 @@
 // See the LICENSE file in the project root for more information.
 
 using System;
-using System.Reflection;
 using Microsoft.ML.Data;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
 
@@ -15,7 +14,7 @@ public class MetricsAgentsTests
         [TestMethod]
         public void BinaryMetricsGetScoreTest()
         {
-            var metrics = CreateInstance<BinaryClassificationMetrics>(0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8);
+            var metrics = MetricsUtil.CreateBinaryClassificationMetrics(0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8);
             Assert.AreEqual(0.1, GetScore(metrics, BinaryClassificationMetric.Auc));
             Assert.AreEqual(0.2, GetScore(metrics, BinaryClassificationMetric.Accuracy));
             Assert.AreEqual(0.3, GetScore(metrics, BinaryClassificationMetric.PositivePrecision));
@@ -29,7 +28,7 @@ public void BinaryMetricsGetScoreTest()
         [TestMethod]
         public void BinaryMetricsNonPerfectTest()
         {
-            var metrics = CreateInstance<BinaryClassificationMetrics>(0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8);
+            var metrics = MetricsUtil.CreateBinaryClassificationMetrics(0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8);
             Assert.AreEqual(false, IsPerfectModel(metrics, BinaryClassificationMetric.Accuracy));
             Assert.AreEqual(false, IsPerfectModel(metrics, BinaryClassificationMetric.Auc));
             Assert.AreEqual(false, IsPerfectModel(metrics, BinaryClassificationMetric.Auprc));
@@ -43,7 +42,7 @@ public void BinaryMetricsNonPerfectTest()
         [TestMethod]
         public void BinaryMetricsPerfectTest()
         {
-            var metrics = CreateInstance<BinaryClassificationMetrics>(1, 1, 1, 1, 1, 1, 1, 1);
+            var metrics = MetricsUtil.CreateBinaryClassificationMetrics(1, 1, 1, 1, 1, 1, 1, 1);
             Assert.AreEqual(true, IsPerfectModel(metrics, BinaryClassificationMetric.Accuracy));
             Assert.AreEqual(true, IsPerfectModel(metrics, BinaryClassificationMetric.Auc));
             Assert.AreEqual(true, IsPerfectModel(metrics, BinaryClassificationMetric.Auprc));
@@ -57,7 +56,7 @@ public void BinaryMetricsPerfectTest()
         [TestMethod]
         public void MulticlassMetricsGetScoreTest()
         {
-            var metrics = CreateInstance<MultiClassClassifierMetrics>(0.1, 0.2, 0.3, 0.4, 0, 0.5, new double[] {});
+            var metrics = MetricsUtil.CreateMulticlassClassificationMetrics(0.1, 0.2, 0.3, 0.4, 0, 0.5, new double[] {});
             Assert.AreEqual(0.1, GetScore(metrics, MulticlassClassificationMetric.AccuracyMicro));
             Assert.AreEqual(0.2, GetScore(metrics, MulticlassClassificationMetric.AccuracyMacro));
             Assert.AreEqual(0.3, GetScore(metrics, MulticlassClassificationMetric.LogLoss));
@@ -68,7 +67,7 @@ public void MulticlassMetricsGetScoreTest()
         [TestMethod]
         public void MulticlassMetricsNonPerfectTest()
         {
-            var metrics = CreateInstance<MultiClassClassifierMetrics>(0.1, 0.2, 0.3, 0.4, 0, 0.5, new double[] { });
+            var metrics = MetricsUtil.CreateMulticlassClassificationMetrics(0.1, 0.2, 0.3, 0.4, 0, 0.5, new double[] { });
             Assert.AreEqual(false, IsPerfectModel(metrics, MulticlassClassificationMetric.AccuracyMacro));
             Assert.AreEqual(false, IsPerfectModel(metrics, MulticlassClassificationMetric.AccuracyMicro));
             Assert.AreEqual(false, IsPerfectModel(metrics, MulticlassClassificationMetric.LogLoss));
@@ -79,7 +78,7 @@ public void MulticlassMetricsNonPerfectTest()
         [TestMethod]
         public void MulticlassMetricsPerfectTest()
         {
-            var metrics = CreateInstance<MultiClassClassifierMetrics>(1, 1, 0, 1, 0, 1, new double[] { });
+            var metrics = MetricsUtil.CreateMulticlassClassificationMetrics(1, 1, 0, 1, 0, 1, new double[] { });
             Assert.AreEqual(true, IsPerfectModel(metrics, MulticlassClassificationMetric.AccuracyMicro));
             Assert.AreEqual(true, IsPerfectModel(metrics, MulticlassClassificationMetric.AccuracyMacro));
             Assert.AreEqual(true, IsPerfectModel(metrics, MulticlassClassificationMetric.LogLoss));
@@ -90,7 +89,7 @@ public void MulticlassMetricsPerfectTest()
         [TestMethod]
         public void RegressionMetricsGetScoreTest()
         {
-            var metrics = CreateInstance<RegressionMetrics>(0.2, 0.3, 0.4, 0.5, 0.6);
+            var metrics = MetricsUtil.CreateRegressionMetrics(0.2, 0.3, 0.4, 0.5, 0.6);
             Assert.AreEqual(0.2, GetScore(metrics, RegressionMetric.L1));
             Assert.AreEqual(0.3, GetScore(metrics, RegressionMetric.L2));
             Assert.AreEqual(0.4, GetScore(metrics, RegressionMetric.Rms));
@@ -100,7 +99,7 @@ public void RegressionMetricsGetScoreTest()
         [TestMethod]
         public void RegressionMetricsNonPerfectTest()
         {
-            var metrics = CreateInstance<RegressionMetrics>(0.2, 0.3, 0.4, 0.5, 0.6);
+            var metrics = MetricsUtil.CreateRegressionMetrics(0.2, 0.3, 0.4, 0.5, 0.6);
             Assert.AreEqual(false, IsPerfectModel(metrics, RegressionMetric.L1));
             Assert.AreEqual(false, IsPerfectModel(metrics, RegressionMetric.L2));
             Assert.AreEqual(false, IsPerfectModel(metrics, RegressionMetric.Rms));
@@ -110,7 +109,7 @@ public void RegressionMetricsNonPerfectTest()
         [TestMethod]
         public void RegressionMetricsPerfectTest()
         {
-            var metrics = CreateInstance<RegressionMetrics>(0, 0, 0, 0, 1);
+            var metrics = MetricsUtil.CreateRegressionMetrics(0, 0, 0, 0, 1);
             Assert.AreEqual(true, IsPerfectModel(metrics, RegressionMetric.L1));
             Assert.AreEqual(true, IsPerfectModel(metrics, RegressionMetric.L2));
             Assert.AreEqual(true, IsPerfectModel(metrics, RegressionMetric.Rms));
@@ -122,17 +121,7 @@ public void RegressionMetricsPerfectTest()
         public void ThrowNotSupportedMetricException()
         {
             throw MetricsAgentUtil.BuildMetricNotSupportedException(BinaryClassificationMetric.Accuracy);
-        }
-
-        private static T CreateInstance<T>(params object[] args)
-        {
-            var type = typeof(T);
-            var instance = type.Assembly.CreateInstance(
-                type.FullName, false,
-                BindingFlags.Instance | BindingFlags.NonPublic,
-                null, args, null, null);
-            return (T)instance;
-        }
+        }        
 
         private static double GetScore(BinaryClassificationMetrics metrics, BinaryClassificationMetric metric)
         {

src/Test/MetricsUtil.cs

@@ -0,0 +1,49 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Reflection;
+using Microsoft.ML.Data;
+
+namespace Microsoft.ML.Auto.Test
+{
+    internal static class MetricsUtil
+    {
+        public static BinaryClassificationMetrics CreateBinaryClassificationMetrics(
+            double auc, double accuracy, double positivePrecision,
+            double positiveRecall, double negativePrecision,
+            double negativeRecall, double f1Score, double auprc)
+        {
+            return CreateInstance<BinaryClassificationMetrics>(auc, accuracy,
+                positivePrecision, positiveRecall, negativePrecision,
+                negativeRecall, f1Score, auprc);
+        }
+
+        public static MultiClassClassifierMetrics CreateMulticlassClassificationMetrics(
+            double accuracyMicro, double accuracyMacro, double logLoss,
+            double logLossReduction, int topK, double topKAccuracy,
+            double[] perClassLogLoss)
+        {
+            return CreateInstance<MultiClassClassifierMetrics>(accuracyMicro,
+                accuracyMacro, logLoss, logLossReduction, topK,
+                topKAccuracy, perClassLogLoss);
+        }
+
+        public static RegressionMetrics CreateRegressionMetrics(double l1, 
+            double l2, double rms, double lossFn, double rSquared)
+        {
+            return CreateInstance<RegressionMetrics>(l1, l2,
+                rms, lossFn, rSquared);
+        }
+
+        private static T CreateInstance<T>(params object[] args)
+        {
+            var type = typeof(T);
+            var instance = type.Assembly.CreateInstance(
+                type.FullName, false,
+                BindingFlags.Instance | BindingFlags.NonPublic,
+                null, args, null, null);
+            return (T)instance;
+        }
+    }
+}

src/Test/RunResultTests.cs

@@ -0,0 +1,47 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Collections.Generic;
+using Microsoft.ML.Data;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+namespace Microsoft.ML.Auto.Test
+{
+    [TestClass]
+    public class RunResultTests
+    {
+        [TestMethod]
+        public void FindBestResultWithSomeNullMetrics()
+        {
+            var metrics1 = MetricsUtil.CreateRegressionMetrics(0.2, 0.2, 0.2, 0.2, 0.2);
+            var metrics2 = MetricsUtil.CreateRegressionMetrics(0.3, 0.3, 0.3, 0.3, 0.3);
+            var metrics3 = MetricsUtil.CreateRegressionMetrics(0.1, 0.1, 0.1, 0.1, 0.1);
+
+            var runResults = new List<RunResult<RegressionMetrics>>()
+            {
+                new RunResult<RegressionMetrics>(null, null, null, null, 0, 0),
+                new RunResult<RegressionMetrics>(null, metrics1, null, null, 0, 0),
+                new RunResult<RegressionMetrics>(null, metrics2, null, null, 0, 0),
+                new RunResult<RegressionMetrics>(null, metrics3, null, null, 0, 0),
+            };
+
+            var metricsAgent = new RegressionMetricsAgent(RegressionMetric.RSquared);
+            var bestResult = RunResultUtil.GetBestRunResult(runResults, metricsAgent);
+            Assert.AreEqual(0.3, bestResult.Metrics.RSquared);
+        }
+
+        [TestMethod]
+        public void FindBestResultWithAllNullMetrics()
+        {
+            var runResults = new List<RunResult<RegressionMetrics>>()
+            {
+                new RunResult<RegressionMetrics>(null, null, null, null, 0, 0),
+            };
+
+            var metricsAgent = new RegressionMetricsAgent(RegressionMetric.RSquared);
+            var bestResult = RunResultUtil.GetBestRunResult(runResults, metricsAgent);
+            Assert.AreEqual(null, bestResult);
+        }
+    }
+}