Concat transformer

Author: Pete Luferenko

src/Microsoft.ML.Data/DataView/RowToRowMapperTransform.cs

@@ -134,15 +134,15 @@ public sealed class RowToRowMapperTransform : RowToRowTransformBase, IRowToRowMa
     {
         private sealed class Bindings : ColumnBindingsBase
         {
-            private readonly RowToRowMapperTransform _parent;
+            private readonly IRowMapper _mapper;
             public readonly RowMapperColumnInfo[] OutputColInfos;
 
-            public Bindings(ISchema inputSchema, RowToRowMapperTransform parent)
-                : base(inputSchema, true, Contracts.CheckRef(parent, nameof(parent))._mapper.GetOutputColumns().Select(info => info.Name).ToArray())
+            public Bindings(ISchema inputSchema, IRowMapper mapper)
+                : base(inputSchema, true, Contracts.CheckRef(mapper, nameof(mapper)).GetOutputColumns().Select(info => info.Name).ToArray())
             {
-                Contracts.AssertValue(parent);
-                _parent = parent;
-                OutputColInfos = _parent._mapper.GetOutputColumns().ToArray();
+                Contracts.AssertValue(mapper);
+                _mapper = mapper;
+                OutputColInfos = _mapper.GetOutputColumns().ToArray();
             }
 
             protected override ColumnType GetColumnTypeCore(int iinfo)
@@ -168,7 +168,7 @@ public bool[] GetActive(Func<int, bool> predicate, out Func<int, bool> predicate
                 var predicateOut = GetActiveOutputColumns(active);
 
                 // Now map those to active input columns.
-                var predicateIn = _parent._mapper.GetDependencies(predicateOut);
+                var predicateIn = _mapper.GetDependencies(predicateOut);
 
                 // Combine the two sets of input columns.
                 predicateInput =
@@ -255,7 +255,14 @@ public RowToRowMapperTransform(IHostEnvironment env, IDataView input, IRowMapper
         {
             Contracts.CheckValue(mapper, nameof(mapper));
             _mapper = mapper;
-            _bindings = new Bindings(input.Schema, this);
+            _bindings = new Bindings(input.Schema, mapper);
+        }
+
+        public static ISchema GetOutputSchema(ISchema inputSchema, IRowMapper mapper)
+        {
+            Contracts.CheckValue(inputSchema, nameof(inputSchema));
+            Contracts.CheckValue(mapper, nameof(mapper));
+            return new Bindings(inputSchema, mapper);
         }
 
         private RowToRowMapperTransform(IHost host, ModelLoadContext ctx, IDataView input)
@@ -265,7 +272,7 @@ private RowToRowMapperTransform(IHost host, ModelLoadContext ctx, IDataView inpu
             // _mapper
 
             ctx.LoadModel<IRowMapper, SignatureLoadRowMapper>(host, out _mapper, "Mapper", input.Schema);
-            _bindings = new Bindings(input.Schema, this);
+            _bindings = new Bindings(input.Schema, _mapper);
         }
 
         public static RowToRowMapperTransform Create(IHostEnvironment env, ModelLoadContext ctx, IDataView input)

src/Microsoft.ML.Data/EntryPoints/SchemaManipulation.cs

@@ -23,7 +23,7 @@ public static CommonOutputs.TransformOutput ConcatColumns(IHostEnvironment env,
             host.CheckValue(input, nameof(input));
             EntryPointUtils.CheckInputArgs(host, input);
 
-            var xf = new ConcatTransform(env, input, input.Data);
+            var xf = ConcatTransform.Create(env, input, input.Data);
             return new CommonOutputs.TransformOutput { Model = new TransformModel(env, xf, input.Data), OutputData = xf };
         }
 

src/Microsoft.ML.Data/Transforms/ConcatEstimator.cs

@@ -4,18 +4,11 @@
 
 using Microsoft.ML.Core.Data;
 using Microsoft.ML.Data.StaticPipe.Runtime;
-using Microsoft.ML.Runtime;
-using Microsoft.ML.Runtime.Data;
-using Microsoft.ML.Runtime.Data.IO;
 using Microsoft.ML.Runtime.Internal.Utilities;
-using Microsoft.ML.Runtime.Model;
 using System;
 using System.Collections.Generic;
 using System.Linq;
 
-[assembly: LoadableClass(typeof(ConcatTransformer), null, typeof(SignatureLoadModel),
-    "Concat Transformer Wrapper", ConcatTransformer.LoaderSignature)]
-
 namespace Microsoft.ML.Runtime.Data
 {
     public sealed class ConcatEstimator : IEstimator<ITransformer>
@@ -41,11 +34,7 @@ public ConcatEstimator(IHostEnvironment env, string name, params string[] source
         public ITransformer Fit(IDataView input)
         {
             _host.CheckValue(input, nameof(input));
-
-            var xf = new ConcatTransform(_host, input, _name, _source);
-            var empty = new EmptyDataView(_host, input.Schema);
-            var chunk = ApplyTransformUtils.ApplyAllTransformsToData(_host, xf, empty, input);
-            return new ConcatTransformer(_host, chunk);
+            return new ConcatTransform(_host, _name, _source);
         }
 
         private bool HasCategoricals(SchemaShape.Column col)
@@ -123,90 +112,6 @@ public SchemaShape GetOutputSchema(SchemaShape inputSchema)
         }
     }
 
-    // REVIEW: Note that the presence of this thing is a temporary measure only.
-    // If it is cleaned up by code complete so much the better, but if not we will
-    // have to wait a little bit.
-    internal sealed class ConcatTransformer : ITransformer, ICanSaveModel
-    {
-        public const string LoaderSignature = "ConcatTransformWrapper";
-        private const string TransformDirTemplate = "Step_{0:000}";
-
-        private readonly IHostEnvironment _env;
-        private readonly IDataView _xf;
-
-        internal ConcatTransformer(IHostEnvironment env, IDataView xf)
-        {
-            _env = env;
-            _xf = xf;
-        }
-
-        public ISchema GetOutputSchema(ISchema inputSchema)
-        {
-            var dv = new EmptyDataView(_env, inputSchema);
-            var output = ApplyTransformUtils.ApplyAllTransformsToData(_env, _xf, dv);
-            return output.Schema;
-        }
-
-        public void Save(ModelSaveContext ctx)
-        {
-            ctx.CheckAtModel();
-            ctx.SetVersionInfo(GetVersionInfo());
-
-            var dataPipe = _xf;
-            var transforms = new List<IDataTransform>();
-            while (dataPipe is IDataTransform xf)
-            {
-                // REVIEW: a malicious user could construct a loop in the Source chain, that would
-                // cause this method to iterate forever (and throw something when the list overflows). There's
-                // no way to insulate from ALL malicious behavior.
-                transforms.Add(xf);
-                dataPipe = xf.Source;
-                Contracts.AssertValue(dataPipe);
-            }
-            transforms.Reverse();
-
-            ctx.SaveSubModel("Loader", c => BinaryLoader.SaveInstance(_env, c, dataPipe.Schema));
-
-            ctx.Writer.Write(transforms.Count);
-            for (int i = 0; i < transforms.Count; i++)
-            {
-                var dirName = string.Format(TransformDirTemplate, i);
-                ctx.SaveModel(transforms[i], dirName);
-            }
-        }
-
-        private static VersionInfo GetVersionInfo()
-        {
-            return new VersionInfo(
-                modelSignature: "CCATWRPR",
-                verWrittenCur: 0x00010001, // Initial
-                verReadableCur: 0x00010001,
-                verWeCanReadBack: 0x00010001,
-                loaderSignature: LoaderSignature);
-        }
-
-        public ConcatTransformer(IHostEnvironment env, ModelLoadContext ctx)
-        {
-            ctx.CheckAtModel(GetVersionInfo());
-            int n = ctx.Reader.ReadInt32();
-
-            ctx.LoadModel<IDataLoader, SignatureLoadDataLoader>(env, out var loader, "Loader", new MultiFileSource(null));
-
-            IDataView data = loader;
-            for (int i = 0; i < n; i++)
-            {
-                var dirName = string.Format(TransformDirTemplate, i);
-                ctx.LoadModel<IDataTransform, SignatureLoadDataTransform>(env, out var xf, dirName, data);
-                data = xf;
-            }
-
-            _env = env;
-            _xf = data;
-        }
-
-        public IDataView Transform(IDataView input) => ApplyTransformUtils.ApplyAllTransformsToData(_env, _xf, input);
-    }
-
     /// <summary>
     /// The extension methods and implementation support for concatenating columns together.
     /// </summary>