Fix zero based key input from C# classes for matrix factorization (#1507)

This PR makes 0-based key types loaded properly by modifying the getter for C# classes. The previous framework doesn't map raw key value to the actual ordinal number; for example, 0/1/2 in 0-based key should be mapped to 1/2/3 in ML.NET's type system while the previous system maps 0/1/2 to just 0/1/2 if they are read from C# classes. Note that out-of-range values should be mapped to 0. In addition, matrix factorization module is improved for a better code quality.

Author: wschin

src/Microsoft.ML.Api/DataViewConstructionUtils.cs

@@ -207,7 +207,16 @@ private Delegate CreateGetter(ColumnType colType, InternalSchemaDefinition.Colum
                         Host.Assert(colType.RawType == Nullable.GetUnderlyingType(outputType));
                     else
                         Host.Assert(colType.RawType == outputType);
-                    del = CreateDirectGetterDelegate<int>;
+
+                    if (!colType.IsKey)
+                        del = CreateDirectGetterDelegate<int>;
+                    else
+                    {
+                        var keyRawType = colType.RawType;
+                        Host.Assert(colType.AsKey.Contiguous);
+                        Func<Delegate, ColumnType, Delegate> delForKey = CreateKeyGetterDelegate<uint>;
+                        return Utils.MarshalInvoke(delForKey, keyRawType, peek, colType);
+                    }
                 }
                 else
                 {
@@ -288,6 +297,38 @@ private Delegate CreateDirectGetterDelegate<TDst>(Delegate peekDel)
                     peek(GetCurrentRowObject(), Position, ref dst));
             }
 
+            private Delegate CreateKeyGetterDelegate<TDst>(Delegate peekDel, ColumnType colType)
+            {
+                // Make sure the function is dealing with key.
+                Host.Check(colType.IsKey);
+                // Following equations work only with contiguous key type.
+                Host.Check(colType.AsKey.Contiguous);
+                // Following equations work only with unsigned integers.
+                Host.Check(typeof(TDst) == typeof(ulong) || typeof(TDst) == typeof(uint) ||
+                    typeof(TDst) == typeof(byte) || typeof(TDst) == typeof(bool));
+
+                // Convert delegate function to a function which can fetch the underlying value.
+                var peek = peekDel as Peek<TRow, TDst>;
+                Host.AssertValue(peek);
+
+                TDst rawKeyValue = default;
+                ulong key = 0; // the raw key value as ulong
+                ulong min = colType.AsKey.Min;
+                ulong max = min + (ulong)colType.AsKey.Count - 1;
+                ulong result = 0; // the result as ulong
+                ValueGetter<TDst> getter = (ref TDst dst) =>
+                {
+                    peek(GetCurrentRowObject(), Position, ref rawKeyValue);
+                    key = (ulong)Convert.ChangeType(rawKeyValue, typeof(ulong));
+                    if (min <= key && key <= max)
+                        result = key - min + 1;
+                    else
+                        result = 0;
+                    dst = (TDst)Convert.ChangeType(result, typeof(TDst));
+                };
+                return getter;
+            }
+
             protected abstract TRow GetCurrentRowObject();
 
             public bool IsColumnActive(int col)

src/Microsoft.ML.Recommender/MatrixFactorizationPredictor.cs

@@ -13,7 +13,6 @@
 using Microsoft.ML.Runtime.Model;
 using Microsoft.ML.Runtime.Recommender;
 using Microsoft.ML.Runtime.Recommender.Internal;
-using Microsoft.ML.Trainers;
 using Microsoft.ML.Trainers.Recommender;
 
 [assembly: LoadableClass(typeof(MatrixFactorizationPredictor), null, typeof(SignatureLoadModel), "Matrix Factorization Predictor Executor", MatrixFactorizationPredictor.LoaderSignature)]
@@ -347,9 +346,12 @@ private Delegate[] CreateGetter(IRow input, bool[] active)
                 var getters = new Delegate[1];
                 if (active[0])
                 {
+                    // First check if expected columns are ok and then create getters to acccess those columns' values.
                     CheckInputSchema(input.Schema, _matrixColumnIndexColumnIndex, _matrixRowIndexCololumnIndex);
-                    var matrixColumnIndexGetter = input.GetGetter<uint>(_matrixColumnIndexColumnIndex);
-                    var matrixRowIndexGetter = input.GetGetter<uint>(_matrixRowIndexCololumnIndex);
+                    var matrixColumnIndexGetter = RowCursorUtils.GetGetterAs<uint>(NumberType.U4, input, _matrixColumnIndexColumnIndex);
+                    var matrixRowIndexGetter = RowCursorUtils.GetGetterAs<uint>(NumberType.U4, input, _matrixRowIndexCololumnIndex);
+
+                    // Assign the getter of the prediction score. It maps a pair of matrix column index and matrix row index to a scalar.
                     getters[0] = _parent.GetGetter(matrixColumnIndexGetter, matrixRowIndexGetter);
                 }
                 return getters;

src/Microsoft.ML.Recommender/MatrixFactorizationTrainer.cs

@@ -89,44 +89,50 @@ public sealed class MatrixFactorizationTrainer : TrainerBase<MatrixFactorization
     {
         public sealed class Arguments
         {
-            [Argument(ArgumentType.AtMostOnce, HelpText = "Regularization parameter")]
+            [Argument(ArgumentType.AtMostOnce, HelpText = "Regularization parameter. " +
+                "It's the weight of factor matrices' norms in the objective function minimized by matrix factorization's algorithm. " +
+                "A small value could cause over-fitting.")]
             [TGUI(SuggestedSweeps = "0.01,0.05,0.1,0.5,1")]
             [TlcModule.SweepableDiscreteParam("Lambda", new object[] { 0.01f, 0.05f, 0.1f, 0.5f, 1f })]
-            public Double Lambda = 0.1;
+            public double Lambda = 0.1;
 
-            [Argument(ArgumentType.AtMostOnce, HelpText = "Latent space dimension")]
+            [Argument(ArgumentType.AtMostOnce, HelpText = "Latent space dimension (denoted by k). If the factorized matrix is m-by-n, " +
+                "two factor matrices found by matrix factorization are m-by-k and k-by-n, respectively. " +
+                "This value is also known as the rank of matrix factorization because k is generally much smaller than m and n.")]
             [TGUI(SuggestedSweeps = "8,16,64,128")]
             [TlcModule.SweepableDiscreteParam("K", new object[] { 8, 16, 64, 128 })]
             public int K = 8;
 
-            [Argument(ArgumentType.AtMostOnce, HelpText = "Training iterations", ShortName = "iter")]
+            [Argument(ArgumentType.AtMostOnce, HelpText = "Training iterations; that is, the times that the training algorithm iterates through the whole training data once.", ShortName = "iter")]
             [TGUI(SuggestedSweeps = "10,20,40")]
             [TlcModule.SweepableDiscreteParam("NumIterations", new object[] { 10, 20, 40 })]
             public int NumIterations = 20;
 
-            [Argument(ArgumentType.AtMostOnce, HelpText = "Initial learning rate")]
+            [Argument(ArgumentType.AtMostOnce, HelpText = "Initial learning rate. It specifies the speed of the training algorithm. " +
+                "Small value may increase the number of iterations needed to achieve a reasonable result. Large value may lead to numerical difficulty such as a infinity value.")]
             [TGUI(SuggestedSweeps = "0.001,0.01,0.1")]
             [TlcModule.SweepableDiscreteParam("Eta", new object[] { 0.001f, 0.01f, 0.1f })]
-            public Double Eta = 0.1;
+            public double Eta = 0.1;
 
-            [Argument(ArgumentType.AtMostOnce, HelpText = "Number of threads", ShortName = "t")]
+            [Argument(ArgumentType.AtMostOnce, HelpText = "Number of threads can be used in the training procedure.", ShortName = "t")]
             public int? NumThreads;
 
-            [Argument(ArgumentType.AtMostOnce, HelpText = "Suppress writing additional information to output")]
+            [Argument(ArgumentType.AtMostOnce, HelpText = "Suppress writing additional information to output.")]
             public bool Quiet;
 
-            [Argument(ArgumentType.AtMostOnce, HelpText = "Force the matrix factorization P and Q to be non-negative", ShortName = "nn")]
+            [Argument(ArgumentType.AtMostOnce, HelpText = "Force the factor matrices to be non-negative.", ShortName = "nn")]
             public bool NonNegative;
         };
 
         internal const string Summary = "From pairs of row/column indices and a value of a matrix, this trains a predictor capable of filling in unknown entries of the matrix, "
-            + "utilizing a low-rank matrix factorization. This technique is often used in recommender system, where the row and column indices indicate users and items, "
-            + "and the value of the matrix is some rating. ";
+            + "using a low-rank matrix factorization. This technique is often used in recommender system, where the row and column indices indicate users and items, "
+            + "and the values of the matrix are ratings. ";
 
-        private readonly Double _lambda;
+        // LIBMF's parameter
+        private readonly double _lambda;
         private readonly int _k;
         private readonly int _iter;
-        private readonly Double _eta;
+        private readonly double _eta;
         private readonly int _threads;
         private readonly bool _quiet;
         private readonly bool _doNmf;
@@ -135,16 +141,28 @@ public sealed class Arguments
         public const string LoadNameValue = "MatrixFactorization";
 
         /// <summary>
-        /// The row, column, and label columns that the trainer expects. This module uses tuples of (row index, column index, label value) to specify a matrix.
+        /// The row index, column index, and label columns needed to specify the training matrix. This trainer uses tuples of (row index, column index, label value) to specify a matrix.
         /// For example, a 2-by-2 matrix
         ///   [9, 4]
         ///   [8, 7]
         /// can be encoded as tuples (0, 0, 9), (0, 1, 4), (1, 0, 8), and (1, 1, 7). It means that the row/column/label column contains [0, 0, 1, 1]/
         /// [0, 1, 0, 1]/[9, 4, 8, 7].
         /// </summary>
-        public readonly SchemaShape.Column MatrixColumnIndexColumn; // column indices of the training matrix
-        public readonly SchemaShape.Column MatrixRowIndexColumn; // row indices of the training matrix
-        public readonly SchemaShape.Column LabelColumn;
+
+        /// <summary>
+        /// The name of variable (i.e., Column in a <see cref="IDataView"/> type system) used be as matrix's column index.
+        /// </summary>
+        public readonly string MatrixColumnIndexName;
+
+        /// <summary>
+        /// The name of variable (i.e., column in a <see cref="IDataView"/> type system) used as matrix's row index.
+        /// </summary>
+        public readonly string MatrixRowIndexName;
+
+        /// <summary>
+        /// The name variable (i.e., column in a <see cref="IDataView"/> type system) used as matrix's element value.
+        /// </summary>
+        public readonly string LabelName;
 
         /// <summary>
         /// The <see cref="TrainerInfo"/> contains general parameters for this trainer.
@@ -155,7 +173,7 @@ public sealed class Arguments
         /// Extra information the trainer can use. For example, its validation set (if not null) can be use to evaluate the
         /// training progress made at each training iteration.
         /// </summary>
-        public readonly TrainerEstimatorContext Context;
+        private readonly TrainerEstimatorContext _context;
 
         /// <summary>
         /// Legacy constructor initializing a new instance of <see cref="MatrixFactorizationTrainer"/> through the legacy
@@ -209,11 +227,11 @@ public MatrixFactorizationTrainer(IHostEnvironment env, string labelColumn, stri
             _doNmf = args.NonNegative;
 
             Info = new TrainerInfo(normalization: false, caching: false);
-            Context = context;
+            _context = context;
 
-            LabelColumn = new SchemaShape.Column(labelColumn, SchemaShape.Column.VectorKind.Scalar, NumberType.R4, false);
-            MatrixColumnIndexColumn = new SchemaShape.Column(matrixColumnIndexColumnName, SchemaShape.Column.VectorKind.Scalar, NumberType.U4, true);
-            MatrixRowIndexColumn = new SchemaShape.Column(matrixRowIndexColumnName, SchemaShape.Column.VectorKind.Scalar, NumberType.U4, true);
+            LabelName = labelColumn;
+            MatrixColumnIndexName = matrixColumnIndexColumnName;
+            MatrixRowIndexName = matrixRowIndexColumnName;
         }
 
         /// <summary>
@@ -270,22 +288,21 @@ private MatrixFactorizationPredictor TrainCore(IChannel ch, RoleMappedData data,
             int rowCount = matrixRowIndexColInfo.Type.KeyCount;
             ch.Assert(rowCount > 0);
             ch.Assert(colCount > 0);
-            // Checks for equality on the validation set ensure it is correct here.
 
+            // Checks for equality on the validation set ensure it is correct here.
             using (var cursor = data.Data.GetRowCursor(c => c == matrixColumnIndexColInfo.Index || c == matrixRowIndexColInfo.Index || c == data.Schema.Label.Index))
             {
                 // LibMF works only over single precision floats, but we want to be able to consume either.
-                ValueGetter<Single> labGetter = RowCursorUtils.GetGetterAs<Single>(NumberType.R4, cursor, data.Schema.Label.Index);
-                var matrixColumnIndexGetter = cursor.GetGetter<uint>(matrixColumnIndexColInfo.Index);
-                var matrixRowIndexGetter = cursor.GetGetter<uint>(matrixRowIndexColInfo.Index);
+                var labGetter = RowCursorUtils.GetGetterAs<float>(NumberType.R4, cursor, data.Schema.Label.Index);
+                var matrixColumnIndexGetter = RowCursorUtils.GetGetterAs<uint>(NumberType.U4, cursor, matrixColumnIndexColInfo.Index);
+                var matrixRowIndexGetter = RowCursorUtils.GetGetterAs<uint>(NumberType.U4, cursor, matrixRowIndexColInfo.Index);
 
                 if (validData == null)
                 {
                     // Have the trainer do its work.
                     using (var buffer = PrepareBuffer())
                     {
-                        buffer.Train(ch, rowCount, colCount,
-                            cursor, labGetter, matrixRowIndexGetter, matrixColumnIndexGetter);
+                        buffer.Train(ch, rowCount, colCount, cursor, labGetter, matrixRowIndexGetter, matrixColumnIndexGetter);
                         predictor = new MatrixFactorizationPredictor(Host, buffer, matrixColumnIndexColInfo.Type.AsKey, matrixRowIndexColInfo.Type.AsKey);
                     }
                 }
@@ -294,16 +311,16 @@ private MatrixFactorizationPredictor TrainCore(IChannel ch, RoleMappedData data,
                     using (var validCursor = validData.Data.GetRowCursor(
                         c => c == validMatrixColumnIndexColInfo.Index || c == validMatrixRowIndexColInfo.Index || c == validData.Schema.Label.Index))
                     {
-                        ValueGetter<Single> validLabGetter = RowCursorUtils.GetGetterAs<Single>(NumberType.R4, validCursor, validData.Schema.Label.Index);
-                        var validXGetter = validCursor.GetGetter<uint>(validMatrixColumnIndexColInfo.Index);
-                        var validYGetter = validCursor.GetGetter<uint>(validMatrixRowIndexColInfo.Index);
+                        ValueGetter<float> validLabelGetter = RowCursorUtils.GetGetterAs<float>(NumberType.R4, validCursor, validData.Schema.Label.Index);
+                        var validMatrixColumnIndexGetter = RowCursorUtils.GetGetterAs<uint>(NumberType.U4, validCursor, validMatrixColumnIndexColInfo.Index);
+                        var validMatrixRowIndexGetter = RowCursorUtils.GetGetterAs<uint>(NumberType.U4, validCursor, validMatrixRowIndexColInfo.Index);
 
                         // Have the trainer do its work.
                         using (var buffer = PrepareBuffer())
                         {
                             buffer.TrainWithValidation(ch, rowCount, colCount,
                                 cursor, labGetter, matrixRowIndexGetter, matrixColumnIndexGetter,
-                                validCursor, validLabGetter, validYGetter, validXGetter);
+                                validCursor, validLabelGetter, validMatrixRowIndexGetter, validMatrixColumnIndexGetter);
                             predictor = new MatrixFactorizationPredictor(Host, buffer, matrixColumnIndexColInfo.Type.AsKey, matrixRowIndexColInfo.Type.AsKey);
                         }
                     }
@@ -328,20 +345,20 @@ public MatrixFactorizationPredictionTransformer Fit(IDataView input)
             MatrixFactorizationPredictor model = null;
 
             var roles = new List<KeyValuePair<RoleMappedSchema.ColumnRole, string>>();
-            roles.Add(new KeyValuePair<RoleMappedSchema.ColumnRole, string>(RoleMappedSchema.ColumnRole.Label, LabelColumn.Name));
-            roles.Add(new KeyValuePair<RoleMappedSchema.ColumnRole, string>(RecommenderUtils.MatrixColumnIndexKind.Value, MatrixColumnIndexColumn.Name));
-            roles.Add(new KeyValuePair<RoleMappedSchema.ColumnRole, string>(RecommenderUtils.MatrixRowIndexKind.Value, MatrixRowIndexColumn.Name));
+            roles.Add(new KeyValuePair<RoleMappedSchema.ColumnRole, string>(RoleMappedSchema.ColumnRole.Label, LabelName));
+            roles.Add(new KeyValuePair<RoleMappedSchema.ColumnRole, string>(RecommenderUtils.MatrixColumnIndexKind.Value, MatrixColumnIndexName));
+            roles.Add(new KeyValuePair<RoleMappedSchema.ColumnRole, string>(RecommenderUtils.MatrixRowIndexKind.Value, MatrixRowIndexName));
 
             var trainingData = new RoleMappedData(input, roles);
-            var validData = Context == null ? null : new RoleMappedData(Context.ValidationSet, roles);
+            var validData = _context == null ? null : new RoleMappedData(_context.ValidationSet, roles);
 
             using (var ch = Host.Start("Training"))
             using (var pch = Host.StartProgressChannel("Training"))
             {
                 model = TrainCore(ch, trainingData, validData);
             }
 
-            return new MatrixFactorizationPredictionTransformer(Host, model, input.Schema, MatrixColumnIndexColumn.Name, MatrixRowIndexColumn.Name);
+            return new MatrixFactorizationPredictionTransformer(Host, model, input.Schema, MatrixColumnIndexName, MatrixRowIndexName);
         }
 
         public SchemaShape GetOutputSchema(SchemaShape inputSchema)
@@ -357,13 +374,15 @@ void CheckColumnsCompatible(SchemaShape.Column cachedColumn, string expectedColu
                     throw Host.Except($"{expectedColumnName} column '{cachedColumn.Name}' is not compatible");
             }
 
-            // In prediction phase, no label column is expected.
-            if (LabelColumn != null)
-                CheckColumnsCompatible(LabelColumn, LabelColumn.Name);
+            // Check if label column is good.
+            var labelColumn = new SchemaShape.Column(LabelName, SchemaShape.Column.VectorKind.Scalar, NumberType.R4, false);
+            CheckColumnsCompatible(labelColumn, LabelName);
 
-            // In both of training and prediction phases, we need columns of user ID and column ID.
-            CheckColumnsCompatible(MatrixColumnIndexColumn, MatrixColumnIndexColumn.Name);
-            CheckColumnsCompatible(MatrixRowIndexColumn, MatrixRowIndexColumn.Name);
+            // Check if columns of matrix's row and column indexes are good. Note that column of IDataView and column of matrix are two different things.
+            var matrixColumnIndexColumn = new SchemaShape.Column(MatrixColumnIndexName, SchemaShape.Column.VectorKind.Scalar, NumberType.U4, true);
+            var matrixRowIndexColumn = new SchemaShape.Column(MatrixRowIndexName, SchemaShape.Column.VectorKind.Scalar, NumberType.U4, true);
+            CheckColumnsCompatible(matrixColumnIndexColumn, MatrixColumnIndexName);
+            CheckColumnsCompatible(matrixRowIndexColumn, MatrixRowIndexName);
 
             // Input columns just pass through so that output column dictionary contains all input columns.
             var outColumns = inputSchema.Columns.ToDictionary(x => x.Name);
@@ -377,7 +396,7 @@ void CheckColumnsCompatible(SchemaShape.Column cachedColumn, string expectedColu
 
         private SchemaShape.Column[] GetOutputColumnsCore(SchemaShape inputSchema)
         {
-            bool success = inputSchema.TryFindColumn(LabelColumn.Name, out var labelCol);
+            bool success = inputSchema.TryFindColumn(LabelName, out var labelCol);
             Contracts.Assert(success);
 
             return new[]