Added transformer scope enum for decomposability

Pete Luferenko · Pete Luferenko · commit 660ebeb939aa · 2018-08-13T10:01:09.000-07:00
diff --git a/test/Microsoft.ML.Tests/Scenarios/Api/ApiScenariosTests.cs b/test/Microsoft.ML.Tests/Scenarios/Api/ApiScenariosTests.cs
@@ -21,13 +21,17 @@ public ApiScenariosTests(ITestOutputHelper output) : base(output)
         public const string SentimentDataPath = "wikipedia-detox-250-line-data.tsv";
         public const string SentimentTestPath = "wikipedia-detox-250-line-test.tsv";
 
-        public class IrisData
+        public class IrisData: IrisDataNoLabel
+        {
+            public string Label;
+        }
+
+        public class IrisDataNoLabel
         {
             public float SepalLength;
             public float SepalWidth;
             public float PetalLength;
             public float PetalWidth;
-            public string Label;
         }
 
         public class IrisPrediction
diff --git a/test/Microsoft.ML.Tests/Scenarios/Api/DecomposableTrainAndPredict.cs b/test/Microsoft.ML.Tests/Scenarios/Api/DecomposableTrainAndPredict.cs
@@ -37,7 +37,7 @@ void DecomposableTrainAndPredict()
 
                 var scoreRoles = new RoleMappedData(concat, label: "Label", feature: "Features");
                 IDataScorerTransform scorer = ScoreUtils.GetScorer(predictor, scoreRoles, env, trainRoles.Schema);
-                
+
                 // Cut of term transform from pipeline.
                 var new_scorer = ApplyTransformUtils.ApplyAllTransformsToData(env, scorer, loader, term);
                 var keyToValue = new KeyToValueTransform(env, new_scorer, "PredictedLabel");
@@ -52,5 +52,40 @@ void DecomposableTrainAndPredict()
                 }
             }
         }
+
+        /// <summary>
+        /// Decomposable train and predict: Train on Iris multiclass problem, which will require
+        /// a transform on labels. Be able to reconstitute the pipeline for a prediction only task,
+        /// which will essentially "drop" the transform over labels, while retaining the property
+        /// that the predicted label for this has a key-type, the probability outputs for the classes
+        /// have the class labels as slot names, etc. This should be do-able without ugly compromises like,
+        /// say, injecting a dummy label.
+        /// </summary>
+        [Fact]
+        void New_DecomposableTrainAndPredict()
+        {
+            var dataPath = GetDataPath(IrisDataPath);
+            using (var env = new TlcEnvironment())
+            {
+                var data = new MyTextLoader(env, MakeIrisTextLoaderArgs())
+                    .FitAndRead(new MultiFileSource(dataPath));
+
+                var pipeline = new MyConcatTransform(env, "Features", "SepalLength", "SepalWidth", "PetalLength", "PetalWidth")
+                    .Append(new MyTermTransform(env, "Label"), TransformerScope.TrainTest)
+                    .Append(new MySdcaMulticlass(env, new SdcaMultiClassTrainer.Arguments { MaxIterations = 100, Shuffle = true, NumThreads = 1 }, "Features", "Label"))
+                    .Append(new MyKeyToValueTransform(env, "PredictedLabel"));
+
+                var model = pipeline.Fit(data).GetModelFor(TransformerScope.Scoring);
+                var engine = new MyPredictionEngine<IrisDataNoLabel, IrisPrediction>(env, model);
+
+                var testLoader = new TextLoader(env, MakeIrisTextLoaderArgs(), new MultiFileSource(dataPath));
+                var testData = testLoader.AsEnumerable<IrisData>(env, false);
+                foreach (var input in testData.Take(20))
+                {
+                    var prediction = engine.Predict(input);
+                    Assert.True(prediction.PredictedLabel == input.Label);
+                }
+            }
+        }
     }
 }
diff --git a/test/Microsoft.ML.Tests/Scenarios/Api/EstimatorPipe.cs b/test/Microsoft.ML.Tests/Scenarios/Api/EstimatorPipe.cs
@@ -4,6 +4,7 @@
 using Microsoft.ML.Runtime.Internal.Utilities;
 using Microsoft.ML.Runtime.Model;
 using Microsoft.ML.Tests.Scenarios.Api;
+using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
@@ -17,10 +18,20 @@ public sealed class TransformerChain<TLastTransformer> : ITransformer, ICanSaveM
         where TLastTransformer : class, ITransformer
     {
         private readonly ITransformer[] _transformers;
+        private readonly TransformerScope[] _scopes;
         public readonly TLastTransformer LastTransformer;
 
         private const string TransformDirTemplate = "Transform_{0:000}";
 
+        internal TransformerChain(ITransformer[] transformers, TransformerScope[] scopes)
+        {
+            _transformers = transformers.ToArray();
+            _scopes = scopes.ToArray();
+            LastTransformer = transformers.Last() as TLastTransformer;
+            Contracts.Check(LastTransformer != null);
+            Contracts.Check(transformers.Length == scopes.Length);
+        }
+
         public TransformerChain(params ITransformer[] transformers)
         {
             if (Utils.Size(transformers) == 0)
@@ -31,6 +42,7 @@ public TransformerChain(params ITransformer[] transformers)
             else
             {
                 _transformers = transformers.ToArray();
+                _scopes = transformers.Select(x => TransformerScope.Everything).ToArray();
                 LastTransformer = transformers.Last() as TLastTransformer;
                 Contracts.Check(LastTransformer != null);
             }
@@ -63,11 +75,26 @@ public IEnumerable<ITransformer> GetParts()
             return _transformers;
         }
 
-        public TransformerChain<TNewLast> Append<TNewLast>(TNewLast transformer)
+        public TransformerChain<ITransformer> GetModelFor(TransformerScope scopeFilter)
+        {
+            var xfs = new List<ITransformer>();
+            var scopes = new List<TransformerScope>();
+            for (int i=0; i<_transformers.Length; i++)
+            {
+                if ((_scopes[i] & scopeFilter) != TransformerScope.None)
+                {
+                    xfs.Add(_transformers[i]);
+                    scopes.Add(_scopes[i]);
+                }
+            }
+            return new TransformerChain<ITransformer>(xfs.ToArray(), scopes.ToArray());
+        }
+
+        public TransformerChain<TNewLast> Append<TNewLast>(TNewLast transformer, TransformerScope scope)
             where TNewLast : class, ITransformer
         {
             Contracts.CheckValue(transformer, nameof(transformer));
-            return new TransformerChain<TNewLast>(_transformers.Append(transformer).ToArray());
+            return new TransformerChain<TNewLast>(_transformers.Append(transformer).ToArray(), _scopes.Append(scope).ToArray());
         }
 
         public void Save(ModelSaveContext ctx)
@@ -79,6 +106,7 @@ public void Save(ModelSaveContext ctx)
 
             for (int i = 0; i < _transformers.Length; i++)
             {
+                ctx.Writer.Write((int)_scopes[i]);
                 var dirName = string.Format(TransformDirTemplate, i);
                 ctx.SaveModel(_transformers[i], dirName);
             }
@@ -88,8 +116,10 @@ internal TransformerChain(IHostEnvironment env, ModelLoadContext ctx)
         {
             int len = ctx.Reader.ReadInt32();
             _transformers = new ITransformer[len];
+            _scopes = new TransformerScope[len];
             for (int i = 0; i < len; i++)
             {
+                _scopes[i] = (TransformerScope)(ctx.Reader.ReadInt32());
                 var dirName = string.Format(TransformDirTemplate, i);
                 ctx.LoadModel<ITransformer, SignatureLoadModel>(env, out _transformers[i], dirName);
             }
@@ -146,12 +176,6 @@ public ISchema GetOutputSchema()
             return s;
         }
 
-        public CompositeReader<TSource, TNewLastTransformer> Append<TNewLastTransformer>(TNewLastTransformer transformer)
-            where TNewLastTransformer : class, ITransformer
-        {
-            return new CompositeReader<TSource, TNewLastTransformer>(Reader, Transformer.Append(transformer));
-        }
-
         public void SavePipeline(IHostEnvironment env, Stream outputStream)
         {
             using (var ch = env.Start("Saving model"))
@@ -182,26 +206,40 @@ public static CompositeReader<IMultiStreamSource, ITransformer> LoadPipeline(IHo
         }
     }
 
+    [Flags]
+    public enum TransformerScope
+    {
+        None = 0,
+        Training = 1 << 0,
+        Testing = 1 << 1,
+        Scoring = 1 << 2,
+        TrainTest = Training | Testing,
+        Everything = Training | Testing | Scoring
+    }
+
     public sealed class EstimatorChain<TLastTransformer> : IEstimator<TransformerChain<TLastTransformer>>
         where TLastTransformer : class, ITransformer
     {
+        private readonly TransformerScope[] _scopes;
+
         private readonly IEstimator<ITransformer>[] _estimators;
         public readonly IEstimator<TLastTransformer> LastEstimator;
 
-        public EstimatorChain(params IEstimator<ITransformer>[] estimators)
+        private EstimatorChain(IEstimator<ITransformer>[] estimators, TransformerScope[] scopes)
         {
-            Contracts.CheckValueOrNull(estimators);
-            if (Utils.Size(estimators) == 0)
-            {
-                _estimators = new IEstimator<ITransformer>[0];
-                LastEstimator = null;
-            }
-            else
-            {
-                _estimators = estimators;
-                LastEstimator = estimators.Last() as IEstimator<TLastTransformer>;
-                Contracts.Check(LastEstimator != null);
-            }
+            _estimators = estimators;
+            _scopes = scopes;
+            LastEstimator = estimators.Last() as IEstimator<TLastTransformer>;
+
+            Contracts.Check(LastEstimator != null);
+            Contracts.Check(Utils.Size(estimators) == Utils.Size(scopes));
+        }
+
+        public EstimatorChain()
+        {
+            _estimators = new IEstimator<ITransformer>[0];
+            LastEstimator = null;
+            _scopes = new TransformerScope[0];
         }
 
         public TransformerChain<TLastTransformer> Fit(IDataView input)
@@ -215,7 +253,7 @@ public TransformerChain<TLastTransformer> Fit(IDataView input)
                 dv = xfs[i].Transform(dv);
             }
 
-            return new TransformerChain<TLastTransformer>(xfs);
+            return new TransformerChain<TLastTransformer>(xfs, _scopes);
         }
 
         public SchemaShape GetOutputSchema(SchemaShape inputSchema)
@@ -230,11 +268,11 @@ public SchemaShape GetOutputSchema(SchemaShape inputSchema)
             return s;
         }
 
-        public EstimatorChain<TNewTrans> Append<TNewTrans>(IEstimator<TNewTrans> estimator)
+        public EstimatorChain<TNewTrans> Append<TNewTrans>(IEstimator<TNewTrans> estimator, TransformerScope scope = TransformerScope.Everything)
             where TNewTrans : class, ITransformer
         {
             Contracts.CheckValue(estimator, nameof(estimator));
-            return new EstimatorChain<TNewTrans>(_estimators.Append(estimator).ToArray());
+            return new EstimatorChain<TNewTrans>(_estimators.Append(estimator).ToArray(), _scopes.Append(scope).ToArray());
         }
     }
 
@@ -282,16 +320,18 @@ public CompositeReaderEstimator<TSource, TNewTrans> Append<TNewTrans>(IEstimator
 
     public static class LearningPipelineExtensions
     {
-        public static CompositeReaderEstimator<TSource, ITransformer> StartPipe<TSource>(this IDataReaderEstimator<TSource, IDataReader<TSource>> start)
-        {
-            return new CompositeReaderEstimator<TSource, ITransformer>(start);
-        }
-
         public static CompositeReaderEstimator<TSource, TTrans> Append<TSource, TTrans>(
             this IDataReaderEstimator<TSource, IDataReader<TSource>> start, IEstimator<TTrans> estimator)
             where TTrans : class, ITransformer
         {
             return new CompositeReaderEstimator<TSource, ITransformer>(start).Append(estimator);
         }
+
+        public static EstimatorChain<TTrans> Append<TTrans>(
+            this IEstimator<ITransformer> start, IEstimator<TTrans> estimator, TransformerScope scope = TransformerScope.Everything)
+            where TTrans : class, ITransformer
+        {
+            return new EstimatorChain<ITransformer>().Append(start).Append(estimator, scope);
+        }
     }
 }
diff --git a/test/Microsoft.ML.Tests/Scenarios/Api/Wrappers.cs b/test/Microsoft.ML.Tests/Scenarios/Api/Wrappers.cs
@@ -297,6 +297,87 @@ public SchemaShape GetOutputSchema(SchemaShape inputSchema)
         }
     }
 
+    public class MyTermTransform: IEstimator<TransformWrapper>
+    {
+        private readonly IHostEnvironment _env;
+        private readonly string _column;
+        private readonly string _srcColumn;
+
+        public MyTermTransform(IHostEnvironment env, string column, string srcColumn = null)
+        {
+            _env = env;
+            _column = column;
+            _srcColumn = srcColumn;
+        }
+
+        public TransformWrapper Fit(IDataView input)
+        {
+            var xf = new TermTransform(_env, input, _column, _srcColumn);
+            var empty = new EmptyDataView(_env, input.Schema);
+            var chunk = ApplyTransformUtils.ApplyAllTransformsToData(_env, xf, empty, input);
+            return new TransformWrapper(_env, chunk);
+        }
+
+        public SchemaShape GetOutputSchema(SchemaShape inputSchema)
+        {
+            throw new NotImplementedException();
+        }
+    }
+
+    public class MyConcatTransform: IEstimator<TransformWrapper>
+    {
+        private readonly IHostEnvironment _env;
+        private readonly string _name;
+        private readonly string[] _source;
+
+        public MyConcatTransform(IHostEnvironment env, string name, params string[] source)
+        {
+            _env = env;
+            _name = name;
+            _source = source;
+        }
+
+        public TransformWrapper Fit(IDataView input)
+        {
+            var xf = new ConcatTransform(_env, input, _name, _source);
+            var empty = new EmptyDataView(_env, input.Schema);
+            var chunk = ApplyTransformUtils.ApplyAllTransformsToData(_env, xf, empty, input);
+            return new TransformWrapper(_env, chunk);
+        }
+
+        public SchemaShape GetOutputSchema(SchemaShape inputSchema)
+        {
+            throw new NotImplementedException();
+        }
+    }
+
+    public class MyKeyToValueTransform: IEstimator<TransformWrapper>
+    {
+        private readonly IHostEnvironment _env;
+        private readonly string _name;
+        private readonly string _source;
+
+        public MyKeyToValueTransform(IHostEnvironment env, string name, string source = null)
+        {
+            _env = env;
+            _name = name;
+            _source = source;
+        }
+
+        public TransformWrapper Fit(IDataView input)
+        {
+            var xf = new KeyToValueTransform(_env, input, _name, _source);
+            var empty = new EmptyDataView(_env, input.Schema);
+            var chunk = ApplyTransformUtils.ApplyAllTransformsToData(_env, xf, empty, input);
+            return new TransformWrapper(_env, chunk);
+        }
+
+        public SchemaShape GetOutputSchema(SchemaShape inputSchema)
+        {
+            throw new NotImplementedException();
+        }
+    }
+
     public sealed class MySdca : TrainerBase<IPredictor>
     {
         private readonly LinearClassificationTrainer.Arguments _args;
@@ -312,6 +393,19 @@ public MySdca(IHostEnvironment env, LinearClassificationTrainer.Arguments args,
         public ITransformer Train(IDataView trainData, IDataView validationData = null) => TrainTransformer(trainData, validationData);
     }
 
+    public sealed class MySdcaMulticlass: TrainerBase<IPredictor>
+    {
+        private readonly SdcaMultiClassTrainer.Arguments _args;
+
+        public MySdcaMulticlass(IHostEnvironment env, SdcaMultiClassTrainer.Arguments args, string featureCol, string labelCol)
+            : base(env, true, true, featureCol, labelCol)
+        {
+            _args = args;
+        }
+
+        protected override IPredictor TrainCore(TrainContext context) => new SdcaMultiClassTrainer(_env, _args).Train(context);
+    }
+
     public sealed class MyAveragedPerceptron : TrainerBase<IPredictor>
     {
         private readonly AveragedPerceptronTrainer _trainer;
@@ -382,5 +476,10 @@ public static void SaveAsBinary(this IDataView data, IHostEnvironment env, Strea
             using (var ch = env.Start("SaveData"))
                 DataSaverUtils.SaveDataView(ch, saver, data, stream);
         }
+
+        public static IDataView FitAndTransform(this IEstimator<ITransformer> est, IDataView data) => est.Fit(data).Transform(data);
+
+        public static IDataView FitAndRead<TSource>(this IDataReaderEstimator<TSource, IDataReader<TSource>> est, TSource source)
+            => est.Fit(source).Read(source);
     }
 }
