Added evaluation

Author: Pete Luferenko

test/Microsoft.ML.Tests/Scenarios/Api/Evaluation.cs

@@ -1,7 +1,9 @@
-using Microsoft.ML.Runtime.Data;
+using Microsoft.ML.Runtime.Api;
+using Microsoft.ML.Runtime.Data;
 using Microsoft.ML.Runtime.Learners;
 using Xunit;
 using Microsoft.ML.Models;
+using System.Linq;
 
 namespace Microsoft.ML.Tests.Scenarios.Api
 {
@@ -46,5 +48,34 @@ public void Evaluation()
                 var metrics = BinaryClassificationMetrics.FromMetrics(env, metricsDict["OverallMetrics"], metricsDict["ConfusionMatrix"])[0];
             }
         }
+
+        /// <summary>
+        /// Evaluation: Similar to the simple train scenario, except instead of having some 
+        /// predictive structure, be able to score another "test" data file, run the result 
+        /// through an evaluator and get metrics like AUC, accuracy, PR curves, and whatnot. 
+        /// Getting metrics out of this shoudl be as straightforward and unannoying as possible.
+        /// </summary>
+        [Fact]
+        public void New_Evaluation()
+        {
+            var dataPath = GetDataPath(SentimentDataPath);
+            var testDataPath = GetDataPath(SentimentTestPath);
+
+            using (var env = new TlcEnvironment(seed: 1, conc: 1))
+            {
+                // Pipeline.
+                var pipeline = new MyTextLoader(env, MakeSentimentTextLoaderArgs())
+                    .Append(new MyTextTransform(env, MakeSentimentTextTransformArgs()))
+                    .Append(new MySdca(env, new LinearClassificationTrainer.Arguments { NumThreads = 1 }, "Features", "Label"));
+
+                // Train.
+                var model = pipeline.Fit(new MultiFileSource(dataPath));
+
+                // Evaluate on the test set.
+                var dataEval = model.Read(new MultiFileSource(testDataPath));
+                var evaluator = new MyBinaryClassifierEvaluator(env, new BinaryClassifierEvaluator.Arguments() { });
+                var metrics = evaluator.Evaluate(dataEval, labelColumn: "Label", probabilityColumn: "Probability");
+            }
+        }
     }
 }

test/Microsoft.ML.Tests/Scenarios/Api/SimpleTrainAndPredict.cs

@@ -71,7 +71,6 @@ public void New_SimpleTrainAndPredict()
             {
                 // Pipeline.
                 var pipeline = new MyTextLoader(env, MakeSentimentTextLoaderArgs())
-                    .StartPipe() // Actually optional
                     .Append(new MyTextTransform(env, MakeSentimentTextTransformArgs()))
                     .Append(new MySdca(env, new LinearClassificationTrainer.Arguments { NumThreads = 1 }, "Features", "Label"));
 

test/Microsoft.ML.Tests/Scenarios/Api/Wrappers.cs

@@ -1,4 +1,5 @@
 using Microsoft.ML.Core.Data;
+using Microsoft.ML.Models;
 using Microsoft.ML.Runtime;
 using Microsoft.ML.Runtime.Api;
 using Microsoft.ML.Runtime.Data;
@@ -9,6 +10,8 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
+using System.Linq;
+
 [assembly: LoadableClass(typeof(TransformWrapper), null, typeof(SignatureLoadModel),
     "Transform wrapper", TransformWrapper.LoaderSignature)]
 [assembly: LoadableClass(typeof(LoaderWrapper), null, typeof(SignatureLoadModel),
@@ -163,16 +166,16 @@ public TransformWrapper(IHostEnvironment env, ModelLoadContext ctx)
         public IDataView Transform(IDataView input) => ApplyTransformUtils.ApplyAllTransformsToData(_env, _xf, input);
     }
 
-    public interface IPredictorTransformer<out TModel>: ITransformer
+    public interface IPredictorTransformer<out TModel> : ITransformer
     {
         TModel TrainedModel { get; }
     }
 
-    public class ScorerWrapper<TModel>: TransformWrapper, IPredictorTransformer<TModel>
-        where TModel: IPredictor
+    public class ScorerWrapper<TModel> : TransformWrapper, IPredictorTransformer<TModel>
+        where TModel : IPredictor
     {
         public ScorerWrapper(IHostEnvironment env, IDataView scorer, TModel trainedModel)
-            :base(env, scorer)
+            : base(env, scorer)
         {
             Model = trainedModel;
         }
@@ -206,7 +209,7 @@ public SchemaShape GetOutputSchema()
     }
 
     public abstract class TrainerBase<TModel> : IEstimator<ScorerWrapper<TModel>>
-        where TModel: IPredictor
+        where TModel : IPredictor
     {
         protected readonly IHostEnvironment _env;
         private readonly string _featureCol;
@@ -298,12 +301,12 @@ public MySdca(IHostEnvironment env, LinearClassificationTrainer.Arguments args,
         public ITransformer Train(IDataView trainData, IDataView validationData = null) => TrainTransformer(trainData, validationData);
     }
 
-    public sealed class MyAveragedPerceptron: TrainerBase<IPredictor>
+    public sealed class MyAveragedPerceptron : TrainerBase<IPredictor>
     {
         private readonly AveragedPerceptronTrainer _trainer;
 
         public MyAveragedPerceptron(IHostEnvironment env, AveragedPerceptronTrainer.Arguments args, string featureCol, string labelCol)
-            :base(env, false, featureCol, labelCol)
+            : base(env, false, featureCol, labelCol)
         {
             _trainer = new AveragedPerceptronTrainer(env, args);
         }
@@ -334,5 +337,31 @@ public TDst Predict(TSrc example)
         }
     }
 
+    public sealed class MyBinaryClassifierEvaluator
+    {
+        private readonly IHostEnvironment _env;
+        private readonly BinaryClassifierEvaluator _evaluator;
+
+        public MyBinaryClassifierEvaluator(IHostEnvironment env, BinaryClassifierEvaluator.Arguments args)
+        {
+            _env = env;
+            _evaluator = new BinaryClassifierEvaluator(env, args);
+        }
+
+        public BinaryClassificationMetrics Evaluate(IDataView data, string labelColumn, string probabilityColumn)
+        {
+            var ci = EvaluateUtils.GetScoreColumnInfo(_env, data.Schema, null, "Score", MetadataUtils.Const.ScoreColumnKind.BinaryClassification);
+            var map = new KeyValuePair<RoleMappedSchema.ColumnRole, string>[]
+            {
+                RoleMappedSchema.CreatePair(MetadataUtils.Const.ScoreValueKind.Probability, probabilityColumn),
+                RoleMappedSchema.CreatePair(MetadataUtils.Const.ScoreValueKind.Score, ci.Name)
+            };
+            var rmd = new RoleMappedData(data, labelColumn, "Features", opt: true, custom: map);
+
+            var metricsDict = _evaluator.Evaluate(rmd);
+            return BinaryClassificationMetrics.FromMetrics(_env, metricsDict["OverallMetrics"], metricsDict["ConfusionMatrix"]).Single();
+        }
+    }
+
 
 }