Merge branch 'master' of https://github.com/dotnet/machinelearning into cleanup

Author: codemzs

Samples/UCI/readme.md

@@ -152,7 +152,7 @@ public Experiment CreateTrainTestExperiment(IDataView trainData, IDataView testD
                     },
                 Outputs =
                     {
-                        Model = finalOutput
+                        PredictorModel = finalOutput
                     },
                 PipelineId = UniqueId.ToString("N"),
                 Kind = MacroUtils.TrainerKindApiValue<Models.MacroUtilsTrainerKinds>(trainerKind),
@@ -189,7 +189,7 @@ public Models.TrainTestEvaluator.Output AddAsTrainTest(Var<IDataView> trainData,
                     },
                 Outputs =
                     {
-                        Model = finalOutput
+                        PredictorModel = finalOutput
                     },
                 TrainingData = trainData,
                 TestingData = testData,

src/Microsoft.ML.PipelineInference/PipelinePattern.cs

@@ -52,6 +52,8 @@ public void SetInput(IHostEnvironment environment, Experiment experiment)
                 experiment.SetInput(_dataViewEntryPoint.Data, _dataView);
             }
 
+            public Var<IDataView> GetInputData() => null;
+
             public abstract IDataView GetDataView(IHostEnvironment environment);
         }
 

src/Microsoft.ML/CSharpApi.cs

@@ -14,6 +14,12 @@ namespace Microsoft.ML
     public interface ILearningPipelineItem
     {
         ILearningPipelineStep ApplyStep(ILearningPipelineStep previousStep, Experiment experiment);
+
+        /// <summary>
+        /// Returns the place holder for input IDataView object for the node in the execution graph.
+        /// </summary>
+        /// <returns></returns>
+        Var<IDataView> GetInputData();
     }
 
     /// <summary>

src/Microsoft.ML/Data/CollectionDataSource.cs

@@ -4,7 +4,6 @@
 
 using Microsoft.ML.Runtime;
 using Microsoft.ML.Runtime.Data;
-using Microsoft.ML.Runtime.EntryPoints;
 using Microsoft.ML.Transforms;
 
 namespace Microsoft.ML.Models
@@ -66,7 +65,11 @@ public BinaryClassificationMetrics Evaluate(PredictionModel model, ILearningPipe
                     throw environment.Except($"Could not find ConfusionMatrix in the results returned in {nameof(BinaryClassificationEvaluator)} Evaluate.");
                 }
 
-                return BinaryClassificationMetrics.FromMetrics(environment, overallMetrics, confusionMatrix);
+                var metric = BinaryClassificationMetrics.FromMetrics(environment, overallMetrics, confusionMatrix);
+
+                Contracts.Check(metric.Count == 1, $"Exactly one metric set was expected but found {metric.Count} metrics");
+
+                return metric[0];
             }
         }
     }

src/Microsoft.ML/ILearningPipelineItem.cs

@@ -6,6 +6,7 @@
 using Microsoft.ML.Runtime.Api;
 using Microsoft.ML.Runtime.Data;
 using System;
+using System.Collections.Generic;
 
 namespace Microsoft.ML.Models
 {
@@ -18,41 +19,50 @@ private BinaryClassificationMetrics()
         {
         }
 
-        internal static BinaryClassificationMetrics FromMetrics(IHostEnvironment env, IDataView overallMetrics, IDataView confusionMatrix)
+        internal static List<BinaryClassificationMetrics> FromMetrics(IHostEnvironment env, IDataView overallMetrics, IDataView confusionMatrix, int confusionMatriceStartIndex = 0)
         {
             Contracts.AssertValue(env);
             env.AssertValue(overallMetrics);
             env.AssertValue(confusionMatrix);
 
             var metricsEnumerable = overallMetrics.AsEnumerable<SerializationClass>(env, true, ignoreMissingColumns: true);
-            var enumerator = metricsEnumerable.GetEnumerator();
-            if (!enumerator.MoveNext())
+            if (!metricsEnumerable.GetEnumerator().MoveNext())
             {
                 throw env.Except("The overall RegressionMetrics didn't have any rows.");
             }
 
-            SerializationClass metrics = enumerator.Current;
+            List<BinaryClassificationMetrics> metrics = new List<BinaryClassificationMetrics>();
+            var confusionMatrices = ConfusionMatrix.Create(env, confusionMatrix).GetEnumerator();
 
-            if (enumerator.MoveNext())
+            int Index = 0;
+            foreach(var metric in metricsEnumerable)
             {
-                throw env.Except("The overall RegressionMetrics contained more than 1 row.");
+
+                if (Index++ >= confusionMatriceStartIndex && !confusionMatrices.MoveNext())
+                {
+                    throw env.Except("Confusion matrices didn't have enough matrices.");
+                }
+
+                metrics.Add(
+                    new BinaryClassificationMetrics()
+                    {
+                        Auc = metric.Auc,
+                        Accuracy = metric.Accuracy,
+                        PositivePrecision = metric.PositivePrecision,
+                        PositiveRecall = metric.PositiveRecall,
+                        NegativePrecision = metric.NegativePrecision,
+                        NegativeRecall = metric.NegativeRecall,
+                        LogLoss = metric.LogLoss,
+                        LogLossReduction = metric.LogLossReduction,
+                        Entropy = metric.Entropy,
+                        F1Score = metric.F1Score,
+                        Auprc = metric.Auprc,
+                        ConfusionMatrix = confusionMatrices.Current,
+                    });
+
             }
 
-            return new BinaryClassificationMetrics()
-            {
-                Auc = metrics.Auc,
-                Accuracy = metrics.Accuracy,
-                PositivePrecision = metrics.PositivePrecision,
-                PositiveRecall = metrics.PositiveRecall,
-                NegativePrecision = metrics.NegativePrecision,
-                NegativeRecall = metrics.NegativeRecall,
-                LogLoss = metrics.LogLoss,
-                LogLossReduction = metrics.LogLossReduction,
-                Entropy = metrics.Entropy,
-                F1Score = metrics.F1Score,
-                Auprc = metrics.Auprc,
-                ConfusionMatrix = ConfusionMatrix.Create(env, confusionMatrix),
-            };
+            return metrics;
         }
 
         /// <summary>
@@ -155,7 +165,7 @@ internal static BinaryClassificationMetrics FromMetrics(IHostEnvironment env, ID
         /// <summary>
         /// This class contains the public fields necessary to deserialize from IDataView.
         /// </summary>
-        private class SerializationClass
+        private sealed class SerializationClass
         {
 #pragma warning disable 649 // never assigned
             [ColumnName(BinaryClassifierEvaluator.Auc)]

src/Microsoft.ML/Models/BinaryClassificationEvaluator.cs

@@ -66,7 +66,11 @@ public ClassificationMetrics Evaluate(PredictionModel model, ILearningPipelineLo
                     throw environment.Except($"Could not find ConfusionMatrix in the results returned in {nameof(BinaryClassificationEvaluator)} Evaluate.");
                 }
 
-                return ClassificationMetrics.FromMetrics(environment, overallMetrics, confusionMatrix);
+                var metric = ClassificationMetrics.FromMetrics(environment, overallMetrics, confusionMatrix);
+
+                Contracts.Check(metric.Count == 1, $"Exactly one metric set was expected but found {metric.Count} metrics");
+
+                return metric[0];
             }
         }
     }

src/Microsoft.ML/Models/BinaryClassificationMetrics.cs

@@ -5,6 +5,7 @@
 using Microsoft.ML.Runtime;
 using Microsoft.ML.Runtime.Api;
 using Microsoft.ML.Runtime.Data;
+using System.Collections.Generic;
 
 namespace Microsoft.ML.Models
 {
@@ -17,36 +18,45 @@ private ClassificationMetrics()
         {
         }
 
-        internal static ClassificationMetrics FromMetrics(IHostEnvironment env, IDataView overallMetrics, IDataView confusionMatrix)
+        internal static List<ClassificationMetrics> FromMetrics(IHostEnvironment env, IDataView overallMetrics, IDataView confusionMatrix, 
+            int confusionMatriceStartIndex = 0)
         {
             Contracts.AssertValue(env);
             env.AssertValue(overallMetrics);
             env.AssertValue(confusionMatrix);
 
             var metricsEnumerable = overallMetrics.AsEnumerable<SerializationClass>(env, true, ignoreMissingColumns: true);
-            var enumerator = metricsEnumerable.GetEnumerator();
-            if (!enumerator.MoveNext())
+            if (!metricsEnumerable.GetEnumerator().MoveNext())
             {
                 throw env.Except("The overall RegressionMetrics didn't have any rows.");
             }
 
-            SerializationClass metrics = enumerator.Current;
+            List<ClassificationMetrics> metrics = new List<ClassificationMetrics>();
+            var confusionMatrices = ConfusionMatrix.Create(env, confusionMatrix).GetEnumerator();
 
-            if (enumerator.MoveNext())
+            int Index = 0;
+            foreach (var metric in metricsEnumerable)
             {
-                throw env.Except("The overall RegressionMetrics contained more than 1 row.");
+                if (Index++ >= confusionMatriceStartIndex && !confusionMatrices.MoveNext())
+                {
+                    throw env.Except("Confusion matrices didn't have enough matrices.");
+                }
+                
+                metrics.Add(
+                    new ClassificationMetrics()
+                    {
+                        AccuracyMicro = metric.AccuracyMicro,
+                        AccuracyMacro = metric.AccuracyMacro,
+                        LogLoss = metric.LogLoss,
+                        LogLossReduction = metric.LogLossReduction,
+                        TopKAccuracy = metric.TopKAccuracy,
+                        PerClassLogLoss = metric.PerClassLogLoss,
+                        ConfusionMatrix = confusionMatrices.Current
+                    });
+
             }
 
-            return new ClassificationMetrics()
-            {
-                AccuracyMicro = metrics.AccuracyMicro,
-                AccuracyMacro = metrics.AccuracyMacro,
-                LogLoss = metrics.LogLoss,
-                LogLossReduction = metrics.LogLossReduction,
-                TopKAccuracy = metrics.TopKAccuracy,
-                PerClassLogLoss = metrics.PerClassLogLoss,
-                ConfusionMatrix = ConfusionMatrix.Create(env, confusionMatrix)
-            };
+            return metrics;
         }
 
         /// <summary>
@@ -125,7 +135,7 @@ internal static ClassificationMetrics FromMetrics(IHostEnvironment env, IDataVie
         /// <summary>
         /// This class contains the public fields necessary to deserialize from IDataView.
         /// </summary>
-        private class SerializationClass
+        private sealed class SerializationClass
         {
 #pragma warning disable 649 // never assigned
             [ColumnName(MultiClassClassifierEvaluator.AccuracyMicro)]

src/Microsoft.ML/Models/ClassificationEvaluator.cs

@@ -41,7 +41,7 @@ private ConfusionMatrix(double[,] elements, string[] classNames)
             });
         }
 
-        internal static ConfusionMatrix Create(IHostEnvironment env, IDataView confusionMatrix)
+        internal static List<ConfusionMatrix> Create(IHostEnvironment env, IDataView confusionMatrix)
         {
             Contracts.AssertValue(env);
             env.AssertValue(confusionMatrix);
@@ -51,36 +51,44 @@ internal static ConfusionMatrix Create(IHostEnvironment env, IDataView confusion
                 env.Except($"ConfusionMatrix data view did not contain a {nameof(MetricKinds.ColumnNames.Count)} column.");
             }
 
+            IRowCursor cursor = confusionMatrix.GetRowCursor(col => col == countColumn);
+            var slots = default(VBuffer<DvText>);
+            confusionMatrix.Schema.GetMetadata(MetadataUtils.Kinds.SlotNames, countColumn, ref slots);
+            string[] classNames = new string[slots.Count];
+            for (int i = 0; i < slots.Count; i++)
+            {
+                classNames[i] = slots.Values[i].ToString();
+            }
+
             ColumnType type = confusionMatrix.Schema.GetColumnType(countColumn);
             env.Assert(type.IsVector);
-
-            double[,] elements = new double[type.VectorSize, type.VectorSize];
-
-            IRowCursor cursor = confusionMatrix.GetRowCursor(col => col == countColumn);
             ValueGetter<VBuffer<double>> countGetter = cursor.GetGetter<VBuffer<double>>(countColumn);
             VBuffer<double> countValues = default;
-
+            List<ConfusionMatrix> confusionMatrices = new List<ConfusionMatrix>();
+            
             int valuesRowIndex = 0;
+            double[,] elements = null;
             while (cursor.MoveNext())
             {
+                if(valuesRowIndex == 0)
+                    elements = new double[type.VectorSize, type.VectorSize];
+
                 countGetter(ref countValues);
                 for (int i = 0; i < countValues.Length; i++)
                 {
                     elements[valuesRowIndex, i] = countValues.Values[i];
                 }
 
                 valuesRowIndex++;
-            }
 
-            var slots = default(VBuffer<DvText>);
-            confusionMatrix.Schema.GetMetadata(MetadataUtils.Kinds.SlotNames, countColumn, ref slots);
-            string[] classNames = new string[slots.Count];
-            for (int i = 0; i < slots.Count; i++)
-            {
-                classNames[i] = slots.Values[i].ToString();
+                if(valuesRowIndex == type.VectorSize)
+                {
+                    valuesRowIndex = 0;
+                    confusionMatrices.Add(new ConfusionMatrix(elements, classNames));
+                }
             }
 
-            return new ConfusionMatrix(elements, classNames);
+            return confusionMatrices;
         }
 
         /// <summary>