merge with master

Author: Ivan Matantsev

src/Microsoft.ML.Core/Data/IEstimator.cs

@@ -45,12 +45,6 @@ public Column(string name, VectorKind vecKind, DataKind itemKind, bool isKey, st
                 IsKey = isKey;
                 MetadataKinds = metadataKinds;
             }
-
-            public Column CloneWithNewName(string newName)
-            {
-                Contracts.Check(newName != Name, "Should be different name");
-                return new Column(newName, Kind, ItemKind, IsKey, MetadataKinds);
-            }
         }
 
         public SchemaShape(Column[] columns)

src/Microsoft.ML.Data/DataLoadSave/CompositeDataReader.cs

@@ -0,0 +1,111 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using Microsoft.ML.Core.Data;
+using Microsoft.ML.Runtime.Model;
+using System.IO;
+
+namespace Microsoft.ML.Runtime.Data
+{
+    /// <summary>
+    /// This class represents a data reader that applies a transformer chain after reading.
+    /// It also has methods to save itself to a repository.
+    /// </summary>
+    public sealed class CompositeDataReader<TSource, TLastTransformer> : IDataReader<TSource>
+        where TLastTransformer : class, ITransformer
+    {
+        /// <summary>
+        /// The underlying data reader.
+        /// </summary>
+        public readonly IDataReader<TSource> Reader;
+        /// <summary>
+        /// The chain of transformers (possibly empty) that are applied to data upon reading.
+        /// </summary>
+        public readonly TransformerChain<TLastTransformer> Transformer;
+
+        public CompositeDataReader(IDataReader<TSource> reader, TransformerChain<TLastTransformer> transformerChain = null)
+        {
+            Contracts.CheckValue(reader, nameof(reader));
+            Contracts.CheckValueOrNull(transformerChain);
+
+            Reader = reader;
+            Transformer = transformerChain ?? new TransformerChain<TLastTransformer>();
+        }
+
+        public IDataView Read(TSource input)
+        {
+            var idv = Reader.Read(input);
+            idv = Transformer.Transform(idv);
+            return idv;
+        }
+
+        public ISchema GetOutputSchema()
+        {
+            var s = Reader.GetOutputSchema();
+            return Transformer.GetOutputSchema(s);
+        }
+
+        /// <summary>
+        /// Append a new transformer to the end.
+        /// </summary>
+        /// <returns>The new composite data reader</returns>
+        public CompositeDataReader<TSource, TNewLast> AppendTransformer<TNewLast>(TNewLast transformer)
+            where TNewLast : class, ITransformer
+        {
+            Contracts.CheckValue(transformer, nameof(transformer));
+
+            return new CompositeDataReader<TSource, TNewLast>(Reader, Transformer.Append(transformer));
+        }
+
+        /// <summary>
+        /// Save the contents to a stream, as a "model file".
+        /// </summary>
+        public void SaveTo(IHostEnvironment env, Stream outputStream)
+        {
+            Contracts.CheckValue(env, nameof(env));
+            env.CheckValue(outputStream, nameof(outputStream));
+
+            env.Check(outputStream.CanWrite && outputStream.CanSeek, "Need a writable and seekable stream to save");
+            using (var ch = env.Start("Saving pipeline"))
+            {
+                using (var rep = RepositoryWriter.CreateNew(outputStream, ch))
+                {
+                    ch.Trace("Saving data reader");
+                    ModelSaveContext.SaveModel(rep, Reader, "Reader");
+
+                    ch.Trace("Saving transformer chain");
+                    ModelSaveContext.SaveModel(rep, Transformer, TransformerChain.LoaderSignature);
+                    rep.Commit();
+                }
+            }
+        }
+    }
+
+    /// <summary>
+    /// Utility class to facilitate loading from a stream.
+    /// </summary>
+    public static class CompositeDataReader
+    {
+        /// <summary>
+        /// Load the pipeline from stream.
+        /// </summary>
+        public static CompositeDataReader<IMultiStreamSource, ITransformer> LoadFrom(IHostEnvironment env, Stream stream)
+        {
+            Contracts.CheckValue(env, nameof(env));
+            env.CheckValue(stream, nameof(stream));
+
+            env.Check(stream.CanRead && stream.CanSeek, "Need a readable and seekable stream to load");
+            using (var rep = RepositoryReader.Open(stream, env))
+            using (var ch = env.Start("Loading pipeline"))
+            {
+                ch.Trace("Loading data reader");
+                ModelLoadContext.LoadModel<IDataReader<IMultiStreamSource>, SignatureLoadModel>(env, out var reader, rep, "Reader");
+
+                ch.Trace("Loader transformer chain");
+                ModelLoadContext.LoadModel<TransformerChain<ITransformer>, SignatureLoadModel>(env, out var transformerChain, rep, TransformerChain.LoaderSignature);
+                return new CompositeDataReader<IMultiStreamSource, ITransformer>(reader, transformerChain);
+            }
+        }
+    }
+}

src/Microsoft.ML.Data/DataLoadSave/CompositeReaderEstimator.cs

@@ -0,0 +1,59 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using Microsoft.ML.Core.Data;
+
+namespace Microsoft.ML.Runtime.Data
+{
+    /// <summary>
+    /// An estimator class for composite data reader.
+    /// It can be used to build a 'trainable smart data reader', although this pattern is not very common.
+    /// </summary>
+    public sealed class CompositeReaderEstimator<TSource, TLastTransformer> : IDataReaderEstimator<TSource, CompositeDataReader<TSource, TLastTransformer>>
+        where TLastTransformer : class, ITransformer
+    {
+        private readonly IDataReaderEstimator<TSource, IDataReader<TSource>> _start;
+        private readonly EstimatorChain<TLastTransformer> _estimatorChain;
+
+        public CompositeReaderEstimator(IDataReaderEstimator<TSource, IDataReader<TSource>> start, EstimatorChain<TLastTransformer> estimatorChain = null)
+        {
+            Contracts.CheckValue(start, nameof(start));
+            Contracts.CheckValueOrNull(estimatorChain);
+
+            _start = start;
+            _estimatorChain = estimatorChain ?? new EstimatorChain<TLastTransformer>();
+
+            // REVIEW: enforce that estimator chain can read the reader's schema.
+            // Right now it throws.
+            // GetOutputSchema();
+        }
+
+        public CompositeDataReader<TSource, TLastTransformer> Fit(TSource input)
+        {
+            var start = _start.Fit(input);
+            var idv = start.Read(input);
+
+            var xfChain = _estimatorChain.Fit(idv);
+            return new CompositeDataReader<TSource, TLastTransformer>(start, xfChain);
+        }
+
+        public SchemaShape GetOutputSchema()
+        {
+            var shape = _start.GetOutputSchema();
+            return _estimatorChain.GetOutputSchema(shape);
+        }
+
+        /// <summary>
+        /// Append another estimator to the end.
+        /// </summary>
+        public CompositeReaderEstimator<TSource, TNewTrans> Append<TNewTrans>(IEstimator<TNewTrans> estimator)
+            where TNewTrans : class, ITransformer
+        {
+            Contracts.CheckValue(estimator, nameof(estimator));
+
+            return new CompositeReaderEstimator<TSource, TNewTrans>(_start, _estimatorChain.Append(estimator));
+        }
+    }
+
+}

src/Microsoft.ML.Data/DataLoadSave/EstimatorChain.cs

@@ -0,0 +1,78 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using Microsoft.ML.Core.Data;
+using Microsoft.ML.Runtime.Internal.Utilities;
+using System.Linq;
+
+namespace Microsoft.ML.Runtime.Data
+{
+    /// <summary>
+    /// Represents a chain (potentially empty) of estimators that end with a <typeparamref name="TLastTransformer"/>.
+    /// If the chain is empty, <typeparamref name="TLastTransformer"/> is always <see cref="ITransformer"/>.
+    /// </summary>
+    public sealed class EstimatorChain<TLastTransformer> : IEstimator<TransformerChain<TLastTransformer>>
+        where TLastTransformer : class, ITransformer
+    {
+        private readonly TransformerScope[] _scopes;
+        private readonly IEstimator<ITransformer>[] _estimators;
+        public readonly IEstimator<TLastTransformer> LastEstimator;
+
+        private EstimatorChain(IEstimator<ITransformer>[] estimators, TransformerScope[] scopes)
+        {
+            Contracts.AssertValueOrNull(estimators);
+            Contracts.AssertValueOrNull(scopes);
+            Contracts.Assert(Utils.Size(estimators) == Utils.Size(scopes));
+
+            _estimators = estimators ?? new IEstimator<ITransformer>[0];
+            _scopes = scopes ?? new TransformerScope[0];
+            LastEstimator = estimators.LastOrDefault() as IEstimator<TLastTransformer>;
+
+            Contracts.Assert((_estimators.Length > 0) == (LastEstimator != null));
+        }
+
+        /// <summary>
+        /// Create an empty estimator chain.
+        /// </summary>
+        public EstimatorChain()
+        {
+            _estimators = new IEstimator<ITransformer>[0];
+            _scopes = new TransformerScope[0];
+            LastEstimator = null;
+        }
+
+        public TransformerChain<TLastTransformer> Fit(IDataView input)
+        {
+            // REVIEW: before fitting, run schema propagation.
+            // Currently, it throws.
+            // GetOutputSchema(SchemaShape.Create(input.Schema);
+
+            IDataView current = input;
+            var xfs = new ITransformer[_estimators.Length];
+            for (int i = 0; i < _estimators.Length; i++)
+            {
+                var est = _estimators[i];
+                xfs[i] = est.Fit(current);
+                current = xfs[i].Transform(current);
+            }
+
+            return new TransformerChain<TLastTransformer>(xfs, _scopes);
+        }
+
+        public SchemaShape GetOutputSchema(SchemaShape inputSchema)
+        {
+            var s = inputSchema;
+            foreach (var est in _estimators)
+                s = est.GetOutputSchema(s);
+            return s;
+        }
+
+        public EstimatorChain<TNewTrans> Append<TNewTrans>(IEstimator<TNewTrans> estimator, TransformerScope scope = TransformerScope.Everything)
+            where TNewTrans : class, ITransformer
+        {
+            Contracts.CheckValue(estimator, nameof(estimator));
+            return new EstimatorChain<TNewTrans>(_estimators.Append(estimator).ToArray(), _scopes.Append(scope).ToArray());
+        }
+    }
+}

src/Microsoft.ML.Data/DataLoadSave/EstimatorExtensions.cs

@@ -0,0 +1,65 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using Microsoft.ML.Core.Data;
+
+namespace Microsoft.ML.Runtime.Data
+{
+    /// <summary>
+    /// Extension methods that allow chaining estimator and transformer pipes together.
+    /// </summary>
+    public static class LearningPipelineExtensions
+    {
+        /// <summary>
+        /// Create a composite reader estimator by appending an estimator to a reader estimator.
+        /// </summary>
+        public static CompositeReaderEstimator<TSource, TTrans> Append<TSource, TTrans>(
+            this IDataReaderEstimator<TSource, IDataReader<TSource>> start, IEstimator<TTrans> estimator)
+            where TTrans : class, ITransformer
+        {
+            Contracts.CheckValue(start, nameof(start));
+            Contracts.CheckValue(estimator, nameof(estimator));
+
+            return new CompositeReaderEstimator<TSource, ITransformer>(start).Append(estimator);
+        }
+
+        /// <summary>
+        /// Create an estimator chain by appending an estimator to an estimator.
+        /// </summary>
+        public static EstimatorChain<TTrans> Append<TTrans>(
+            this IEstimator<ITransformer> start, IEstimator<TTrans> estimator,
+            TransformerScope scope = TransformerScope.Everything)
+            where TTrans : class, ITransformer
+        {
+            Contracts.CheckValue(start, nameof(start));
+            Contracts.CheckValue(estimator, nameof(estimator));
+
+            return new EstimatorChain<ITransformer>().Append(start).Append(estimator, scope);
+        }
+
+        /// <summary>
+        /// Create a composite reader by appending a transformer to a data reader.
+        /// </summary>
+        public static CompositeDataReader<TSource, TTrans> Append<TSource, TTrans>(this IDataReader<TSource> reader, TTrans transformer)
+            where TTrans : class, ITransformer
+        {
+            Contracts.CheckValue(reader, nameof(reader));
+            Contracts.CheckValue(transformer, nameof(transformer));
+
+            return new CompositeDataReader<TSource, ITransformer>(reader).AppendTransformer(transformer);
+        }
+
+        /// <summary>
+        /// Create a transformer chain by appending a transformer to a transformer.
+        /// </summary>
+        public static TransformerChain<TTrans> Append<TTrans>(this ITransformer start, TTrans transformer)
+            where TTrans : class, ITransformer
+        {
+            Contracts.CheckValue(start, nameof(start));
+            Contracts.CheckValue(transformer, nameof(transformer));
+
+            return new TransformerChain<TTrans>(start, transformer);
+        }
+    }
+}