Term transformer implementation

src/Microsoft.ML.Data/DataView/RowToRowMapperTransform.cs

@@ -9,6 +9,8 @@
 using Microsoft.ML.Runtime;
 using Microsoft.ML.Runtime.Data;
 using Microsoft.ML.Runtime.Model;
+using Microsoft.ML.Runtime.Model.Onnx;
+using Microsoft.ML.Runtime.Model.Pfa;
 
 [assembly: LoadableClass(typeof(RowToRowMapperTransform), null, typeof(SignatureLoadDataTransform),
     "", RowToRowMapperTransform.LoaderSignature)]
@@ -110,7 +112,7 @@ public Dictionary<string, MetadataInfo> Infos()
     /// It does so with the help of an <see cref="IRowMapper"/>, that is given a schema in its constructor, and has methods
     /// to get the dependencies on input columns and the getters for the output columns, given an active set of output columns.
     /// </summary>
-    public sealed class RowToRowMapperTransform : RowToRowTransformBase
+    public sealed class RowToRowMapperTransform : RowToRowTransformBase, IRowToRowMapper, ITransformCanSaveOnnx, ITransformCanSavePfa
     {
         private sealed class Bindings : ColumnBindingsBase
         {
@@ -209,6 +211,10 @@ private static VersionInfo GetVersionInfo()
 
         public override ISchema Schema { get { return _bindings; } }
 
+        public bool CanSaveOnnx => _mapper is ISaveAsOnnx onnxMapper ? true : false;
+
+        public bool CanSavePfa => _mapper is ISaveAsOnnx pfaMapper ? true : false;
+
         public RowToRowMapperTransform(IHostEnvironment env, IDataView input, IRowMapper mapper)
             : base(env, RegistrationName, input)
         {
@@ -318,6 +324,96 @@ public override IRowCursor[] GetRowCursorSet(out IRowCursorConsolidator consolid
             return cursors;
         }
 
+        public void SaveAsOnnx(OnnxContext ctx)
+        {
+            if (_mapper is ISaveAsOnnx onnx)
+                onnx.SaveAsOnnx(ctx);
+        }
+
+        public void SaveAsPfa(BoundPfaContext ctx)
+        {
+            if (_mapper is ISaveAsPfa onnx)
+                onnx.SaveAsPfa(ctx);
+        }
+
+        public Func<int, bool> GetDependencies(Func<int, bool> predicate)
+        {
+            Func<int, bool> predicateInput;
+            var active = _bindings.GetActive(predicate, out predicateInput);
+            return predicateInput;
+        }
+
+        public IRow GetRow(IRow input, Func<int, bool> active, out Action disposer)
+        {
+            Host.CheckValue(input, nameof(input));
+            Host.CheckValue(active, nameof(active));
+            Host.Check(input.Schema == Source.Schema, "Schema of input row must be the same as the schema the mapper is bound to");
+
+            disposer = null;
+            using (var ch = Host.Start("GetEntireRow"))
+            {
+                Action disp;
+                var activeArr = new bool[Schema.ColumnCount];
+                for (int i = 0; i < Schema.ColumnCount; i++)
+                    activeArr[i] = active(i);
+                var pred = _bindings.GetActiveOutputColumns(activeArr);
+                var getters = _mapper.CreateGetters(input, pred, out disp);
+                disposer += disp;
+                ch.Done();
+                return new Row(input, this, Schema, getters);
+            }
+        }
+
+        private sealed class Row : IRow
+        {
+            private readonly IRow _input;
+            private readonly Delegate[] _getters;
+
+            private readonly RowToRowMapperTransform _parent;
+
+            public long Batch { get { return _input.Batch; } }
+
+            public long Position { get { return _input.Position; } }
+
+            public ISchema Schema { get; }
+
+            public Row(IRow input, RowToRowMapperTransform parent, ISchema schema, Delegate[] getters)
+            {
+                _input = input;
+                _parent = parent;
+                Schema = schema;
+                _getters = getters;
+            }
+
+            public ValueGetter<TValue> GetGetter<TValue>(int col)
+            {
+                bool isSrc;
+                int index = _parent._bindings.MapColumnIndex(out isSrc, col);
+                if (isSrc)
+                    return _input.GetGetter<TValue>(index);
+
+                Contracts.Assert(_getters[index] != null);
+                var fn = _getters[index] as ValueGetter<TValue>;
+                if (fn == null)
+                    throw Contracts.Except("Invalid TValue in GetGetter: '{0}'", typeof(TValue));
+                return fn;
+            }
+
+            public ValueGetter<UInt128> GetIdGetter()
+            {
+                return _input.GetIdGetter();
+            }
+
+            public bool IsColumnActive(int col)
+            {
+                bool isSrc;
+                int index = _parent._bindings.MapColumnIndex(out isSrc, col);
+                if (isSrc)
+                    return _input.IsColumnActive((index));
+                return _getters[index] != null;
+            }
+        }
+
         private sealed class RowCursor : SynchronizedCursorBase<IRowCursor>, IRowCursor
         {
             private readonly Delegate[] _getters;

src/Microsoft.ML.Data/Model/Onnx/ICanSaveOnnx.cs

@@ -19,9 +19,9 @@ public interface ICanSaveOnnx
     }
 
     /// <summary>
-    /// This data model component is savable as ONNX.
+    /// This component know how to save himself in ONNX format.
     /// </summary>
-    public interface ITransformCanSaveOnnx: ICanSaveOnnx, IDataTransform
+    public interface ISaveAsOnnx
     {
         /// <summary>
         /// Save as ONNX.
@@ -30,6 +30,13 @@ public interface ITransformCanSaveOnnx: ICanSaveOnnx, IDataTransform
         void SaveAsOnnx(OnnxContext ctx);
     }
 
+    /// <summary>
+    /// This data model component is savable as ONNX.
+    /// </summary>
+    public interface ITransformCanSaveOnnx : ICanSaveOnnx, ISaveAsOnnx, IDataTransform
+    {
+    }
+
     /// <summary>
     /// This <see cref="ISchemaBindableMapper"/> is savable in ONNX. Note that this is
     /// typically called within an <see cref="IDataScorerTransform"/> that is wrapping

src/Microsoft.ML.Data/Model/Pfa/ICanSavePfa.cs

@@ -20,9 +20,9 @@ public interface ICanSavePfa
     }
 
     /// <summary>
-    /// This data model component is savable as PFA. See http://dmg.org/pfa/ .
+    /// This component know how to save himself in Pfa format.
     /// </summary>
-    public interface ITransformCanSavePfa : ICanSavePfa, IDataTransform
+    public interface ISaveAsPfa
     {
         /// <summary>
         /// Save as PFA. For any columns that are output, this interface should use
@@ -34,6 +34,14 @@ public interface ITransformCanSavePfa : ICanSavePfa, IDataTransform
         void SaveAsPfa(BoundPfaContext ctx);
     }
 
+    /// <summary>
+    /// This data model component is savable as PFA. See http://dmg.org/pfa/ .
+    /// </summary>
+    public interface ITransformCanSavePfa : ICanSavePfa, ISaveAsPfa, IDataTransform
+    {
+
+    }
+
     /// <summary>
     /// This <see cref="ISchemaBindableMapper"/> is savable as a PFA. Note that this is
     /// typically called within an <see cref="IDataScorerTransform"/> that is wrapping

src/Microsoft.ML.Data/Transforms/CopyColumnsTransform.cs

@@ -217,7 +217,7 @@ internal sealed class CopyColumnsRowMapper : IRowMapper
     {
         private readonly ISchema _schema;
         private readonly Dictionary<int, int> _colNewToOldMapping;
-        private (string Source, string Name)[] _columns;
+        private readonly (string Source, string Name)[] _columns;
         private readonly IHost _host;
         public const string LoaderSignature = "CopyColumnsRowMapper";
 

src/Microsoft.ML.Data/Transforms/TermEstimatorcs.cs

@@ -0,0 +1,73 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using Microsoft.ML.Core.Data;
+using System.Collections.Generic;
+using System.Linq;
+using static Microsoft.ML.Runtime.Data.TermTransform;
+
+namespace Microsoft.ML.Runtime.Data
+{
+    public sealed class TermEstimator : IEstimator<TermTransform>
+    {
+        private readonly int _maxNumTerms;
+        private readonly SortOrder _sort;
+        private readonly Column[] _columns;
+        private readonly IHost _host;
+
+        public TermEstimator(IHostEnvironment env, string name, string source = null, int maxNumTerms = Defaults.MaxNumTerms, SortOrder sort = Defaults.Sort) :
+           this(env, maxNumTerms, sort, new Column { Name = name, Source = source ?? name })
+        {
+        }
+
+        public TermEstimator(IHostEnvironment env, int maxNumTerms = Defaults.MaxNumTerms, SortOrder sort = Defaults.Sort, params Column[] columns)
+        {
+            Contracts.CheckValue(env, nameof(env));
+            _host = env.Register(nameof(TermEstimator));
+            var newNames = new HashSet<string>();
+            foreach (var column in columns)
+            {
+                if (newNames.Contains(column.Name))
+                    throw Contracts.ExceptUserArg(nameof(columns), $"New column {column.Name} specified multiple times");
+                newNames.Add(column.Name);
+            }
+            _columns = columns;
+            _maxNumTerms = maxNumTerms;
+            _sort = sort;
+        }
+
+        public TermTransform Fit(IDataView input)
+        {
+            // Invoke schema validation.
+            GetOutputSchema(SchemaShape.Create(input.Schema));
+            var args = new Arguments
+            {
+                Column = _columns,
+                MaxNumTerms = _maxNumTerms,
+                Sort = _sort
+            };
+            return new TermTransform(_host, args, input);
+        }
+
+        public SchemaShape GetOutputSchema(SchemaShape inputSchema)
+        {
+            _host.CheckValue(inputSchema, nameof(inputSchema));
+            var resultDic = inputSchema.Columns.ToDictionary(x => x.Name);
+            foreach (var column in _columns)
+            {
+                var originalColumn = inputSchema.FindColumn(column.Source);
+                if (originalColumn != null)
+                {
+                    var col = new SchemaShape.Column(column.Name, originalColumn.Kind, DataKind.U4, true, originalColumn.MetadataKinds);
+                    resultDic[column.Name] = col;
+                }
+                else
+                {
+                    throw _host.ExceptParam(nameof(inputSchema), $"{column.Source} not found in {nameof(inputSchema)}");
+                }
+            }
+            return new SchemaShape(resultDic.Values.ToArray());
+        }
+    }
+}