Implementing copy column estimator

src/Microsoft.ML.Core/Data/IEstimator.cs

@@ -45,6 +45,12 @@ public Column(string name, VectorKind vecKind, DataKind itemKind, bool isKey, st
                 IsKey = isKey;
                 MetadataKinds = metadataKinds;
             }
+
+            public Column CloneWithNewName(string newName)
+            {
+                Contracts.Check(newName != Name, "Should be different name");
+                return new Column(newName, Kind, ItemKind, IsKey, MetadataKinds);
+            }
         }
 
         public SchemaShape(Column[] columns)

src/Microsoft.ML.Data/Transforms/CopyColumnsTransform.cs

@@ -3,8 +3,11 @@
 // See the LICENSE file in the project root for more information.
 
 using System;
+using System.Collections.Generic;
+using System.Linq;
 using System.Reflection;
 using System.Text;
+using Microsoft.ML.Core.Data;
 using Microsoft.ML.Runtime;
 using Microsoft.ML.Runtime.CommandLine;
 using Microsoft.ML.Runtime.Data;
@@ -18,6 +21,9 @@
 [assembly: LoadableClass(CopyColumnsTransform.Summary, typeof(CopyColumnsTransform), null, typeof(SignatureLoadDataTransform),
     CopyColumnsTransform.UserName, CopyColumnsTransform.LoaderSignature)]
 
+[assembly: LoadableClass(CopyColumnsTransform.Summary, typeof(CopyColumnsTransformer), null, typeof(SignatureLoadModel),
+    CopyColumnsTransform.UserName, CopyColumnsTransformer.LoaderSignature)]
+
 namespace Microsoft.ML.Runtime.Data
 {
     public sealed class CopyColumnsTransform : OneToOneTransformBase
@@ -72,7 +78,7 @@ private static VersionInfo GetVersionInfo()
         /// <param name="name">Name of the output column.</param>
         /// <param name="source">Name of the column to be copied.</param>
         public CopyColumnsTransform(IHostEnvironment env, IDataView input, string name, string source)
-            : this(env, new Arguments(){ Column = new[] { new Column() { Source = source, Name = name }}}, input)
+            : this(env, new Arguments() { Column = new[] { new Column() { Source = source, Name = name } } }, input)
         {
         }
 
@@ -163,4 +169,189 @@ protected override Delegate GetGetterCore(IChannel ch, IRow input, int iinfo, ou
             return (Delegate)methodInfo.Invoke(input, new object[] { col });
         }
     }
+
+    public sealed class CopyColumnsEstimator : IEstimator<CopyColumnsTransformer>
+    {
+        private readonly (string Source, string Name)[] _columns;
+        private readonly IHost _host;
+
+        public CopyColumnsEstimator(IHostEnvironment env, string input, string output)
+        {
+            Contracts.CheckNonWhiteSpace(input, nameof(input));
+            Contracts.CheckNonWhiteSpace(output, nameof(output));
+            _columns = new (string, string)[1] { (input, output) };
+            _host = env.Register("CopyColumnsEstimator");
+        }
+
+        public CopyColumnsEstimator(IHostEnvironment env, (string Source, string Name)[] columns)
+        {
+            Contracts.CheckValue(columns, nameof(columns));
+            var newNames = new HashSet<string>();
+            foreach (var column in columns)
+            {
+                if (newNames.Contains(column.Name))
+                    throw Contracts.ExceptUserArg(nameof(columns), $"New column {column.Name} specified multiple times");
+                newNames.Add(column.Name);
+            }
+            _columns = columns;
+            _host = env.Register("CopyColumnsEstimator");
+        }
+
+        public CopyColumnsTransformer Fit(IDataView input)
+        {
+            // invoke schema validation.
+            GetOutputSchema(SchemaShape.Create(input.Schema));
+            return new CopyColumnsTransformer(_host, _columns);
+        }
+
+        public SchemaShape GetOutputSchema(SchemaShape inputSchema)
+        {
+            Contracts.CheckValue(inputSchema, nameof(inputSchema));
+            Contracts.CheckValue(inputSchema.Columns, nameof(inputSchema.Columns));
+            var originDic = inputSchema.Columns.ToDictionary(x => x.Name);
+            var resultDic = inputSchema.Columns.ToDictionary(x => x.Name);
+            foreach ((string source, string name) pair in _columns)
+            {
+                if (originDic.ContainsKey(pair.source))
+                {
+                    var col = originDic[pair.source].CloneWithNewName(pair.name);
+                    resultDic[pair.name] = col;
+                }
+                else
+                {
+                    throw Contracts.ExceptUserArg(nameof(inputSchema), $"{pair.source} not found in {nameof(inputSchema)}");
+                }
+            }
+            var result = new SchemaShape(originDic.Values.ToArray());
+            return result;
+        }
+    }
+
+    public sealed class CopyColumnsTransformer : ITransformer, ICanSaveModel
+    {
+        private readonly (string Source, string Name)[] _columns;
+        private readonly IHost _host;
+
+        private class CopyColumnsRowMapper : IRowMapper
+        {
+            private readonly ISchema _schema;
+            private readonly HashSet<int> _originalColumnSources;
+            private (string source, string name)[] _columns;
+
+            public CopyColumnsRowMapper(ISchema schema, (string source, string name)[] columns)
+            {
+                _schema = schema;
+                _columns = columns;
+                _originalColumnSources = new HashSet<int>();
+                var sources = new HashSet<string>();
+                foreach (var source in columns.Select(x => x.source))
+                    sources.Add(source);
+                for (int i = 0; i < _schema.ColumnCount; i++)
+                    if (sources.Contains(_schema.GetColumnName(i)))
+                        _originalColumnSources.Add(i);
+
+            }
+
+            public Delegate[] CreateGetters(IRow input, Func<int, bool> activeOutput, out Action disposer)
+            {
+                var result = new Delegate[_columns.Length];
+                int i = 0;
+                foreach (var column in _columns)
+                {
+                    // validate activeOutput.
+                    input.Schema.TryGetColumnIndex(column.source, out int colIndex);
+                    var type = input.Schema.GetColumnType(colIndex);
+
+                    result[i++] = Utils.MarshalInvoke(MakeGetter<int>, type.RawType, input, colIndex);
+                }
+                disposer = null;
+                return result;
+            }
+
+            private Delegate MakeGetter<T>(IRow row, int src) => row.GetGetter<T>(src);
+
+            public Func<int, bool> GetDependencies(Func<int, bool> activeOutput) => (col) => (activeOutput(col) && _originalColumnSources.Contains(col));
+
+            public RowMapperColumnInfo[] GetOutputColumns()
+            {
+                var result = new RowMapperColumnInfo[_columns.Length];
+                // Do I need return all columns or only output???
+                for (int i = 0; i < _columns.Length; i++)
+                {
+                    _schema.TryGetColumnIndex(_columns[i].source, out int colIndex);
+                    // how to get metadata?
+                    result[i] = new RowMapperColumnInfo(_columns[i].name, _schema.GetColumnType(colIndex), null);
+                }
+                return result;
+            }
+
+            public void Save(ModelSaveContext ctx)
+            {
+                throw new NotImplementedException();
+            }
+        }
+        public CopyColumnsTransformer(IHostEnvironment env, (string Source, string Name)[] columns)
+        {
+            _columns = columns;
+            _host = env.Register("CopyColumnsTransformer");
+        }
+
+        public ISchema GetOutputSchema(ISchema inputSchema)
+        {
+            //Validate schema.
+            return Transform(new EmptyDataView(_host, inputSchema)).Schema;
+        }
+
+        private static VersionInfo GetVersionInfo()
+        {
+            return new VersionInfo(
+                modelSignature: "COPYCOLT",
+                verWrittenCur: 0x00010001, // Initial
+                verReadableCur: 0x00010001,
+                verWeCanReadBack: 0x00010001,
+                loaderSignature: LoaderSignature);
+        }
+
+        public const string LoaderSignature = "CopyTransform";
+
+        public void Save(ModelSaveContext ctx)
+        {
+            _host.CheckValue(ctx, nameof(ctx));
+            ctx.CheckAtModel();
+            ctx.SetVersionInfo(GetVersionInfo());
+
+            // *** Binary format ***
+            // int: number of added columns
+            // for each added column
+            //   int: id of output column name
+            //   int: id of input column name
+            ctx.Writer.Write(_columns.Length);
+            foreach (var column in _columns)
+            {
+                ctx.SaveNonEmptyString(column.Name);
+                ctx.SaveNonEmptyString(column.Source);
+            }
+        }
+        public static CopyColumnsTransformer Create(IHostEnvironment env, ModelLoadContext ctx)
+        {
+            Contracts.CheckValue(env, nameof(env));
+            env.CheckValue(ctx, nameof(ctx));
+            ctx.CheckAtModel(GetVersionInfo());
+            var lenght = ctx.Reader.ReadInt32();
+            var columns = new (string Source, string Name)[lenght];
+            for (int i = 0; i < lenght; i++)
+            {
+                columns[i].Name = ctx.LoadNonEmptyString();
+                columns[i].Source = ctx.LoadNonEmptyString();
+            }
+            return new CopyColumnsTransformer(env, columns);
+        }
+
+        public IDataView Transform(IDataView input)
+        {
+            IRowMapper mapper = new CopyColumnsRowMapper(input.Schema, _columns);
+            return new RowToRowMapperTransform(_host, input, mapper);
+        }
+
+    }
 }

test/Microsoft.ML.Tests/CopyColumnEstimatorTests.cs

@@ -0,0 +1,128 @@
+using Microsoft.ML.Runtime.Api;
+using Microsoft.ML.Runtime.Data;
+using System.Collections.Generic;
+using System.IO;
+using Xunit;
+
+namespace Microsoft.ML.Tests
+{
+    public class CopyColumnEstimatorTests
+    {
+        class TestClass
+        {
+            public int A;
+            public int B;
+            public int C;
+        }
+        class TestClassXY
+        {
+            public int X;
+            public int Y;
+        }
+
+
+        [Fact]
+        void TestWorking()
+        {
+            var data = new List<TestClass>() { new TestClass() { A = 1, B = 2, C = 3, }, new TestClass() { A = 4, B = 5, C = 6 } };
+            using (var env = new TlcEnvironment())
+            {
+                var dataView = ComponentCreation.CreateDataView(env, data);
+                var est = new CopyColumnsEstimator(env, new[] { ("A", "D"), ("B", "E") });
+                var transformer = est.Fit(dataView);
+                var result = transformer.Transform(dataView);
+                ValidateCopyColumnTransformer(result);
+            }
+        }
+
+        private void ValidateCopyColumnTransformer(IDataView result)
+        {
+            using (var cursor = result.GetRowCursor(x => true))
+            {
+                DvInt4 avalue = 0;
+                DvInt4 bvalue = 0;
+                DvInt4 dvalue = 0;
+                DvInt4 evalue = 0;
+                var aGetter = cursor.GetGetter<DvInt4>(0);
+                var bGetter = cursor.GetGetter<DvInt4>(1);
+                var dGetter = cursor.GetGetter<DvInt4>(3);
+                var eGetter = cursor.GetGetter<DvInt4>(4);
+                while (cursor.MoveNext())
+                {
+                    aGetter(ref avalue);
+                    bGetter(ref bvalue);
+                    dGetter(ref dvalue);
+                    eGetter(ref evalue);
+                    Assert.Equal(avalue, dvalue);
+                    Assert.Equal(bvalue, evalue);
+                }
+            }
+        }
+
+        [Fact]
+        void TestBadOriginalSchema()
+        {
+            var data = new List<TestClass>() { new TestClass() { A = 1, B = 2, C = 3, }, new TestClass() { A = 4, B = 5, C = 6 } };
+            using (var env = new TlcEnvironment())
+            {
+                var dataView = ComponentCreation.CreateDataView(env, data);
+                var est = new CopyColumnsEstimator(env, new[] { ("D", "A"), ("B", "E") });
+                bool failed = false;
+                try
+                {
+                    var transformer = est.Fit(dataView);
+                }
+                catch
+                {
+                    failed = true;
+                }
+                Assert.True(failed);
+            }
+        }
+
+        [Fact]
+        void TestBadTransformSchmea()
+        {
+            var data = new List<TestClass>() { new TestClass() { A = 1, B = 2, C = 3, }, new TestClass() { A = 4, B = 5, C = 6 } };
+            var xydata = new List<TestClassXY>() { new TestClassXY() { X = 10, Y = 100 }, new TestClassXY() { X = -1, Y = -100 } };
+            using (var env = new TlcEnvironment())
+            {
+                var dataView = ComponentCreation.CreateDataView(env, data);
+                var xyDataView = ComponentCreation.CreateDataView(env, xydata);
+                var est = new CopyColumnsEstimator(env, new[] { ("A", "D"), ("B", "E") });
+                var transformer = est.Fit(dataView);
+                bool failed = false;
+                try
+                {
+                    var result = transformer.Transform(xyDataView);
+                }
+                catch
+                {
+                    failed = true;
+                }
+                Assert.True(failed);
+            }
+        }
+
+        [Fact]
+        void TestSavingAndLoading()
+        {
+            var data = new List<TestClass>() { new TestClass() { A = 1, B = 2, C = 3, }, new TestClass() { A = 4, B = 5, C = 6 } };
+            using (var env = new TlcEnvironment())
+            {
+                var dataView = ComponentCreation.CreateDataView(env, data);
+                var est = new CopyColumnsEstimator(env, new[] { ("A", "D"), ("B", "E") });
+                var transformer = est.Fit(dataView);
+                using (var ms = new MemoryStream())
+                {
+                    transformer.SaveTo(env, ms);
+                    ms.Position = 0;
+                    var loadedTransformer = TransformerChain.LoadFrom(env, ms);
+                    var result = loadedTransformer.Transform(dataView);
+                    ValidateCopyColumnTransformer(result);
+                }
+
+            }
+        }
+    }
+}