CrossValidator class

Author: Pete Luferenko

test/Microsoft.ML.Tests/Scenarios/Api/CrossValidation.cs

@@ -105,7 +105,12 @@ void New_CrossValidation()
                             ConvergenceTolerance = 1f
                         }, "Features", "Label"));
 
-                (var models, var metrics) = MyHelperExtensions.CrossValidateBinary(env, data, pipeline, "Label", numFolds: 2);
+                var cv = new MyCrossValidation.BinaryCrossValidator(env)
+                {
+                    NumFolds = 2
+                };
+
+                var cvResult = cv.CrossValidate(data, pipeline);
             }
         }
     }

test/Microsoft.ML.Tests/Scenarios/Api/Wrappers.cs

@@ -187,11 +187,11 @@ public ScorerWrapper(IHostEnvironment env, IDataView scorer, TModel trainedModel
         public TModel InnerModel { get; }
     }
 
-    public class BinaryScorerWrapper<TModel>: ScorerWrapper<TModel>
-        where TModel: IPredictor
+    public class BinaryScorerWrapper<TModel> : ScorerWrapper<TModel>
+        where TModel : IPredictor
     {
         public BinaryScorerWrapper(IHostEnvironment env, TModel model, ISchema inputSchema, string featureColumn, BinaryClassifierScorer.Arguments args)
-            :base(env, MakeScorer(env, inputSchema, featureColumn, model, args), model, featureColumn)
+            : base(env, MakeScorer(env, inputSchema, featureColumn, model, args), model, featureColumn)
         {
         }
 
@@ -235,7 +235,7 @@ public SchemaShape GetOutputSchema()
     }
 
     public abstract class TrainerBase<TTransformer, TModel> : IEstimator<TTransformer>
-        where TTransformer: ScorerWrapper<TModel>
+        where TTransformer : ScorerWrapper<TModel>
         where TModel : IPredictor
     {
         protected readonly IHostEnvironment _env;
@@ -414,7 +414,7 @@ public SchemaShape GetOutputSchema(SchemaShape inputSchema)
         }
     }
 
-    public sealed class MySdca : TrainerBase<BinaryScorerWrapper<IPredictor>,IPredictor>
+    public sealed class MySdca : TrainerBase<BinaryScorerWrapper<IPredictor>, IPredictor>
     {
         private readonly LinearClassificationTrainer.Arguments _args;
 
@@ -428,7 +428,7 @@ public MySdca(IHostEnvironment env, LinearClassificationTrainer.Arguments args,
 
         public ITransformer Train(IDataView trainData, IDataView validationData = null) => TrainTransformer(trainData, validationData);
 
-        protected override BinaryScorerWrapper<IPredictor> MakeScorer(IPredictor predictor, RoleMappedData data) 
+        protected override BinaryScorerWrapper<IPredictor> MakeScorer(IPredictor predictor, RoleMappedData data)
             => new BinaryScorerWrapper<IPredictor>(_env, predictor, data.Data.Schema, _featureCol, new BinaryClassifierScorer.Arguments());
     }
 
@@ -512,68 +512,92 @@ public BinaryClassificationMetrics Evaluate(IDataView data, string labelColumn,
         }
     }
 
-    public static class MyHelperExtensions
+    public static class MyCrossValidation
     {
-        public static void SaveAsBinary(this IDataView data, IHostEnvironment env, Stream stream)
+        public sealed class BinaryCrossValidationMetrics
         {
-            var saver = new BinarySaver(env, new BinarySaver.Arguments());
-            using (var ch = env.Start("SaveData"))
-                DataSaverUtils.SaveDataView(ch, saver, data, stream);
+            public readonly ITransformer[] FoldModels;
+            public readonly BinaryClassificationMetrics[] FoldMetrics;
+
+            public BinaryCrossValidationMetrics(ITransformer[] models, BinaryClassificationMetrics[] metrics)
+            {
+                FoldModels = models;
+                FoldMetrics = metrics;
+            }
         }
 
-        public static IDataView FitAndTransform(this IEstimator<ITransformer> est, IDataView data) => est.Fit(data).Transform(data);
+        public sealed class BinaryCrossValidator
+        {
+            private readonly IHostEnvironment _env;
 
-        public static IDataView FitAndRead<TSource>(this IDataReaderEstimator<TSource, IDataReader<TSource>> est, TSource source)
-            => est.Fit(source).Read(source);
+            public int NumFolds { get; set; } = 2;
 
-        public static (ITransformer[] Models, BinaryClassificationMetrics[] Metrics) CrossValidateBinary(IHostEnvironment env, IDataView trainData, IEstimator<ITransformer> estimator,
-            string labelColumn,
-            int numFolds = 2,
-            string stratificationColumn = null,
-            bool cache = false)
-        {
-            var models = new ITransformer[numFolds];
-            var metrics = new BinaryClassificationMetrics[numFolds];
+            public string StratificationColumn { get; set; }
 
-            if (stratificationColumn == null)
+            public string LabelColumn { get; set; } = DefaultColumnNames.Label;
+
+            public BinaryCrossValidator(IHostEnvironment env)
             {
-                stratificationColumn = "StratificationColumn";
-                var random = new GenerateNumberTransform(env, trainData, stratificationColumn);
-                trainData = random;
+                _env = env;
             }
-            else
-                throw new NotImplementedException();
-
-            IDataView cachedTrain = trainData;
-            if (cache)
-                cachedTrain = new CacheDataView(env, trainData, prefetch: null);
 
-            var evaluator = new MyBinaryClassifierEvaluator(env, new BinaryClassifierEvaluator.Arguments() { });
-
-            for (int fold = 0; fold < numFolds; fold++)
+            public BinaryCrossValidationMetrics CrossValidate(IDataView trainData, IEstimator<ITransformer> estimator)
             {
-                var trainFilter = new RangeFilter(env, new RangeFilter.Arguments()
+                var models = new ITransformer[NumFolds];
+                var metrics = new BinaryClassificationMetrics[NumFolds];
+
+                if (StratificationColumn == null)
                 {
-                    Column = stratificationColumn,
-                    Min = (Double)fold / numFolds,
-                    Max = (Double)(fold + 1) / numFolds,
-                    Complement = true
-                }, cachedTrain);
-                var testFilter = new RangeFilter(env, new RangeFilter.Arguments()
+                    StratificationColumn = "StratificationColumn";
+                    var random = new GenerateNumberTransform(_env, trainData, StratificationColumn);
+                    trainData = random;
+                }
+                else
+                    throw new NotImplementedException();
+
+                var evaluator = new MyBinaryClassifierEvaluator(_env, new BinaryClassifierEvaluator.Arguments() { });
+
+                for (int fold = 0; fold < NumFolds; fold++)
                 {
-                    Column = stratificationColumn,
-                    Min = (Double)fold / numFolds,
-                    Max = (Double)(fold + 1) / numFolds,
-                    Complement = false
-                }, cachedTrain);
-
-                models[fold] = estimator.Fit(trainFilter);
-                var scoredTest = models[fold].Transform(testFilter);
-                metrics[fold] = evaluator.Evaluate(scoredTest, labelColumn: labelColumn, probabilityColumn: "Probability");
+                    var trainFilter = new RangeFilter(_env, new RangeFilter.Arguments()
+                    {
+                        Column = StratificationColumn,
+                        Min = (Double)fold / NumFolds,
+                        Max = (Double)(fold + 1) / NumFolds,
+                        Complement = true
+                    }, trainData);
+                    var testFilter = new RangeFilter(_env, new RangeFilter.Arguments()
+                    {
+                        Column = StratificationColumn,
+                        Min = (Double)fold / NumFolds,
+                        Max = (Double)(fold + 1) / NumFolds,
+                        Complement = false
+                    }, trainData);
+
+                    models[fold] = estimator.Fit(trainFilter);
+                    var scoredTest = models[fold].Transform(testFilter);
+                    metrics[fold] = evaluator.Evaluate(scoredTest, labelColumn: LabelColumn, probabilityColumn: "Probability");
+                }
+
+                return new BinaryCrossValidationMetrics(models, metrics);
+
             }
+        }
+    }
 
-            return (models, metrics);
+
+    public static class MyHelperExtensions
+    {
+        public static void SaveAsBinary(this IDataView data, IHostEnvironment env, Stream stream)
+        {
+            var saver = new BinarySaver(env, new BinarySaver.Arguments());
+            using (var ch = env.Start("SaveData"))
+                DataSaverUtils.SaveDataView(ch, saver, data, stream);
         }
 
+        public static IDataView FitAndTransform(this IEstimator<ITransformer> est, IDataView data) => est.Fit(data).Transform(data);
+
+        public static IDataView FitAndRead<TSource>(this IDataReaderEstimator<TSource, IDataReader<TSource>> est, TSource source)
+            => est.Fit(source).Read(source);
     }
 }