Lowered execution times on some tests.

Implemented new API for initial predictor and validation sets

Author: Pete Luferenko

test/Microsoft.ML.Tests/Scenarios/Api/AutoNormalizationAndCaching.cs

@@ -30,7 +30,8 @@ public void AutoNormalizationAndCaching()
                 // Train.
                 var trainer = new LinearClassificationTrainer(env, new LinearClassificationTrainer.Arguments
                 {
-                    NumThreads = 1
+                    NumThreads = 1,
+                    ConvergenceTolerance = 1f
                 });
 
                 // Auto-caching.

test/Microsoft.ML.Tests/Scenarios/Api/EstimatorPipe.cs

@@ -152,7 +152,7 @@ public CompositeReader<TSource, TNewLastTransformer> Append<TNewLastTransformer>
             return new CompositeReader<TSource, TNewLastTransformer>(Reader, Transformer.Append(transformer));
         }
 
-        public void Save(IHostEnvironment env, Stream outputStream)
+        public void SavePipeline(IHostEnvironment env, Stream outputStream)
         {
             using (var ch = env.Start("Saving model"))
             {
@@ -171,7 +171,7 @@ public void Save(IHostEnvironment env, Stream outputStream)
 
     public static class CompositeReader
     {
-        public static CompositeReader<IMultiStreamSource, ITransformer> LoadModel(IHostEnvironment env, Stream stream)
+        public static CompositeReader<IMultiStreamSource, ITransformer> LoadPipeline(IHostEnvironment env, Stream stream)
         {
             using (var rep = RepositoryReader.Open(stream, env))
             {

test/Microsoft.ML.Tests/Scenarios/Api/TrainSaveModelAndPredict.cs

@@ -93,10 +93,10 @@ public void New_TrainSaveModelAndPredict()
                 {
                     // Save model. 
                     using (var fs = file.CreateWriteStream())
-                        model.Save(env, fs);
+                        model.SavePipeline(env, fs);
 
                     // Load model.
-                    loadedModel = CompositeReader.LoadModel(env, file.OpenReadStream());
+                    loadedModel = CompositeReader.LoadPipeline(env, file.OpenReadStream());
 
                 }
 

test/Microsoft.ML.Tests/Scenarios/Api/TrainWithInitialPredictor.cs

@@ -39,5 +39,39 @@ public void TrainWithInitialPredictor()
                 var finalPredictor = secondTrainer.Train(new TrainContext(trainRoles, initialPredictor: predictor));
             }
         }
+
+        /// <summary>
+        /// Train with initial predictor: Similar to the simple train scenario, but also accept a pre-trained initial model.
+        /// The scenario might be one of the online linear learners that can take advantage of this, e.g., averaged perceptron.
+        /// </summary>
+        [Fact]
+        public void New_TrainWithInitialPredictor()
+        {
+            var dataPath = GetDataPath(SentimentDataPath);
+
+            using (var env = new TlcEnvironment(seed: 1, conc: 1))
+            {
+                // Pipeline.
+                var pipeline = new MyTextLoader(env, MakeSentimentTextLoaderArgs())
+                    .Append(new MyTextTransform(env, MakeSentimentTextTransformArgs()));
+
+                // Train the pipeline, prepare train set.
+                var reader = pipeline.Fit(new MultiFileSource(dataPath));
+                var trainData = reader.Read(new MultiFileSource(dataPath));
+
+
+                // Train the first predictor.
+                var trainer = new MySdca(env, new LinearClassificationTrainer.Arguments
+                {
+                    NumThreads = 1
+                }, "Features", "Label");
+                var firstPredictor = trainer.Fit(trainData);
+
+                // Train the second predictor on the same data.
+                var secondTrainer = new MyAveragedPerceptron(env, new AveragedPerceptronTrainer.Arguments(), "Features", "Label");
+                var finalPredictor = secondTrainer.Train(trainData, firstPredictor);
+            }
+        }
+
     }
 }

test/Microsoft.ML.Tests/Scenarios/Api/TrainWithValidationSet.cs

@@ -32,7 +32,7 @@ public void TrainWithValidationSet()
                 // Apply the same transformations on the validation set.
                 // Sadly, there is no way to easily apply the same loader to different data, so we either have
                 // to create another loader, or to save the loader to model file and then reload.
-                
+
                 // A new one is not always feasible, but this time it is.
                 var validLoader = new TextLoader(env, MakeSentimentTextLoaderArgs(), new MultiFileSource(validationDataPath));
                 var validData = ApplyTransformUtils.ApplyAllTransformsToData(env, trainData, validLoader);
@@ -42,11 +42,41 @@ public void TrainWithValidationSet()
                 var cachedValid = new CacheDataView(env, validData, prefetch: null);
 
                 // Train.
-                var trainer = new FastTreeBinaryClassificationTrainer(env, new FastTreeBinaryClassificationTrainer.Arguments());
+                var trainer = new FastTreeBinaryClassificationTrainer(env, new FastTreeBinaryClassificationTrainer.Arguments
+                {
+                    NumTrees = 3
+                });
                 var trainRoles = new RoleMappedData(cachedTrain, label: "Label", feature: "Features");
-                var validRoles = new RoleMappedData(cachedTrain, label: "Label", feature: "Features");
+                var validRoles = new RoleMappedData(cachedValid, label: "Label", feature: "Features");
                 trainer.Train(new Runtime.TrainContext(trainRoles, validRoles));
             }
         }
+
+        /// <summary>
+        /// Train with validation set: Similar to the simple train scenario, but also support a validation set.
+        /// The learner might be trees with early stopping.
+        /// </summary>
+        [Fact]
+        public void New_TrainWithValidationSet()
+        {
+            var dataPath = GetDataPath(SentimentDataPath);
+            var validationDataPath = GetDataPath(SentimentTestPath);
+
+            using (var env = new TlcEnvironment(seed: 1, conc: 1))
+            {
+                // Pipeline.
+                var pipeline = new MyTextLoader(env, MakeSentimentTextLoaderArgs())
+                    .Append(new MyTextTransform(env, MakeSentimentTextTransformArgs()));
+
+                // Train the pipeline, prepare train and validation set.
+                var reader = pipeline.Fit(new MultiFileSource(dataPath));
+                var trainData = reader.Read(new MultiFileSource(dataPath));
+                var validData = reader.Read(new MultiFileSource(validationDataPath));
+
+                // Train model with validation set.
+                var trainer = new MySdca(env, new Runtime.Learners.LinearClassificationTrainer.Arguments(), "Features", "Label");
+                var transformer = trainer.Train(trainData, validData);
+            }
+        }
     }
 }

test/Microsoft.ML.Tests/Scenarios/Api/Wrappers.cs

@@ -16,6 +16,8 @@
 
 namespace Microsoft.ML.Tests.Scenarios.Api
 {
+    using LinearModel = LinearPredictor;
+
     public sealed class LoaderWrapper : IDataReader<IMultiStreamSource>, ICanSaveModel
     {
         public const string LoaderSignature = "LoaderWrapper";
@@ -161,6 +163,24 @@ public TransformWrapper(IHostEnvironment env, ModelLoadContext ctx)
         public IDataView Transform(IDataView input) => ApplyTransformUtils.ApplyAllTransformsToData(_env, _xf, input);
     }
 
+    public interface IPredictorTransformer<out TModel>: ITransformer
+    {
+        TModel TrainedModel { get; }
+    }
+
+    public class ScorerWrapper<TModel>: TransformWrapper, IPredictorTransformer<TModel>
+        where TModel: IPredictor
+    {
+        public ScorerWrapper(IHostEnvironment env, IDataView scorer, TModel trainedModel)
+            :base(env, scorer)
+        {
+            Model = trainedModel;
+        }
+
+        public TModel TrainedModel => Model;
+
+        public TModel Model { get; }
+    }
 
     public class MyTextLoader : IDataReaderEstimator<IMultiStreamSource, LoaderWrapper>
     {
@@ -185,7 +205,8 @@ public SchemaShape GetOutputSchema()
         }
     }
 
-    public abstract class TrainerBase : IEstimator<TransformWrapper>
+    public abstract class TrainerBase<TModel> : IEstimator<ScorerWrapper<TModel>>
+        where TModel: IPredictor
     {
         protected readonly IHostEnvironment _env;
         private readonly string _featureCol;
@@ -200,25 +221,41 @@ protected TrainerBase(IHostEnvironment env, bool cache, string featureColumn, st
             _labelCol = labelColumn;
         }
 
-        public TransformWrapper Fit(IDataView input)
+        public ScorerWrapper<TModel> Fit(IDataView input)
         {
-            var cached = _cache ? new CacheDataView(_env, input, prefetch: null) : input;
+            return TrainTransformer(input);
+        }
+
+        protected ScorerWrapper<TModel> TrainTransformer(IDataView trainSet, IDataView validationSet = null, IPredictor initPredictor = null)
+        {
+            var cachedTrain = _cache ? new CacheDataView(_env, trainSet, prefetch: null) : trainSet;
+
+            var trainRoles = new RoleMappedData(cachedTrain, label: _labelCol, feature: _featureCol);
+            RoleMappedData validRoles;
 
-            var trainRoles = new RoleMappedData(cached, label: _labelCol, feature: _featureCol);
-            var pred = Train(trainRoles);
+            if (validationSet == null)
+                validRoles = null;
+            else
+            {
+                var cachedValid = _cache ? new CacheDataView(_env, validationSet, prefetch: null) : validationSet;
+                validRoles = new RoleMappedData(cachedValid, label: _labelCol, feature: _featureCol);
+            }
 
-            var emptyData = new EmptyDataView(_env, input.Schema);
+            var pred = TrainCore(new TrainContext(trainRoles, validRoles, initPredictor));
+
+            var emptyData = new EmptyDataView(_env, trainSet.Schema);
             var scoreRoles = new RoleMappedData(emptyData, label: _labelCol, feature: _featureCol);
             IDataScorerTransform scorer = ScoreUtils.GetScorer(pred, scoreRoles, _env, trainRoles.Schema);
-            return new TransformWrapper(_env, scorer);
+            return new ScorerWrapper<TModel>(_env, scorer, pred);
+
         }
 
         public SchemaShape GetOutputSchema(SchemaShape inputSchema)
         {
             throw new NotImplementedException();
         }
 
-        protected abstract IPredictor Train(RoleMappedData data);
+        protected abstract TModel TrainCore(TrainContext trainContext);
     }
 
     public class MyTextTransform : IEstimator<TransformWrapper>
@@ -246,7 +283,7 @@ public SchemaShape GetOutputSchema(SchemaShape inputSchema)
         }
     }
 
-    public sealed class MySdca : TrainerBase
+    public sealed class MySdca : TrainerBase<IPredictor>
     {
         private readonly LinearClassificationTrainer.Arguments _args;
 
@@ -256,7 +293,27 @@ public MySdca(IHostEnvironment env, LinearClassificationTrainer.Arguments args,
             _args = args;
         }
 
-        protected override IPredictor Train(RoleMappedData data) => new LinearClassificationTrainer(_env, _args).Train(data);
+        protected override IPredictor TrainCore(TrainContext context) => new LinearClassificationTrainer(_env, _args).Train(context);
+
+        public ITransformer Train(IDataView trainData, IDataView validationData = null) => TrainTransformer(trainData, validationData);
+    }
+
+    public sealed class MyAveragedPerceptron: TrainerBase<IPredictor>
+    {
+        private readonly AveragedPerceptronTrainer _trainer;
+
+        public MyAveragedPerceptron(IHostEnvironment env, AveragedPerceptronTrainer.Arguments args, string featureCol, string labelCol)
+            :base(env, false, featureCol, labelCol)
+        {
+            _trainer = new AveragedPerceptronTrainer(env, args);
+        }
+
+        protected override IPredictor TrainCore(TrainContext trainContext) => _trainer.Train(trainContext);
+
+        public ITransformer Train(IDataView trainData, IPredictorTransformer<IPredictor> initialPredictor)
+        {
+            return TrainTransformer(trainData, initPredictor: initialPredictor.TrainedModel);
+        }
     }
 
     public sealed class MyPredictionEngine<TSrc, TDst>