ValueMapper: added more tests with 'TestEstimatorCore' and fixed a bug in ValueType. (#2416)

Author: zeahmed

src/Microsoft.ML.Data/Transforms/ConversionsExtensionsCatalog.cs

@@ -176,6 +176,33 @@ public static ValueMappingEstimator<TInputType, TOutputType> ValueMap<TInputType
             params (string outputColumnName, string inputColumnName)[] columns)
             => new ValueMappingEstimator<TInputType, TOutputType>(CatalogUtils.GetEnvironment(catalog), keys, values, columns);
 
+        /// <summary>
+        /// <see cref="ValueMappingEstimator"/>
+        /// </summary>
+        /// <typeparam name="TInputType">The key type.</typeparam>
+        /// <typeparam name="TOutputType">The value type.</typeparam>
+        /// <param name="catalog">The categorical transform's catalog</param>
+        /// <param name="keys">The list of keys to use for the mapping. The mapping is 1-1 with values. This list must be the same length as values and
+        /// cannot contain duplicate keys.</param>
+        /// <param name="values">The list of values (an array) to pair with the keys for the mapping. This list must be equal to the same length as keys.</param>
+        /// <param name="columns">The columns to apply this transform on.</param>
+        /// <returns>An instance of the ValueMappingEstimator</returns>
+        /// <example>
+        /// <format type="text/markdown">
+        /// <![CDATA[
+        ///  [!code-csharp[ValueMappingEstimator](~/../docs/samples/docs/samples/Microsoft.ML.Samples/Dynamic/ValueMapping.cs)]
+        ///  [!code-csharp[ValueMappingEstimator](~/../docs/samples/docs/samples/Microsoft.ML.Samples/Dynamic/ValueMappingStringToKeyType.cs)]
+        ///  [!code-csharp[ValueMappingEstimator](~/../docs/samples/docs/samples/Microsoft.ML.Samples/Dynamic/ValueMappingFloatToString.cs)]
+        ///  [!code-csharp[ValueMappingEstimator](~/../docs/samples/docs/samples/Microsoft.ML.Samples/Dynamic/ValueMappingStringToArray.cs)]
+        /// ]]></format>
+        /// </example>
+        public static ValueMappingEstimator<TInputType, TOutputType> ValueMap<TInputType, TOutputType>(
+            this TransformsCatalog.ConversionTransforms catalog,
+            IEnumerable<TInputType> keys,
+            IEnumerable<TOutputType[]> values,
+            params (string outputColumnName, string inputColumnName)[] columns)
+            => new ValueMappingEstimator<TInputType, TOutputType>(CatalogUtils.GetEnvironment(catalog), keys, values, columns);
+
         /// <summary>
         /// <see cref="ValueMappingEstimator"/>
         /// </summary>

src/Microsoft.ML.Data/Transforms/ValueMapping.cs

@@ -63,10 +63,23 @@ public override SchemaShape GetOutputSchema(SchemaShape inputSchema)
             Host.CheckValue(inputSchema, nameof(inputSchema));
 
             var resultDic = inputSchema.ToDictionary(x => x.Name);
-            var vectorKind = Transformer.ValueColumnType is VectorType ? SchemaShape.Column.VectorKind.Vector : SchemaShape.Column.VectorKind.Scalar;
+
+            // Decide the output VectorKind for the columns
+            // based on the value type of the dictionary
+            var vectorKind = SchemaShape.Column.VectorKind.Scalar;
+            if (Transformer.ValueColumnType is VectorType)
+            {
+                vectorKind = SchemaShape.Column.VectorKind.Vector;
+                if(Transformer.ValueColumnType.GetVectorSize() == 0)
+                    vectorKind = SchemaShape.Column.VectorKind.VariableVector;
+            }
+
+            // Set the data type of the output column
+            // if the output VectorKind is a vector or variable vector then
+            // this is the data type of items stored in the vector.
             var isKey = Transformer.ValueColumnType is KeyType;
             var columnType = (isKey) ? NumberType.U4 :
-                                    Transformer.ValueColumnType;
+                                    Transformer.ValueColumnType.GetItemType();
             var metadataShape = SchemaShape.Create(Transformer.ValueColumnMetadata.Schema);
             foreach (var (outputColumnName, inputColumnName) in _columns)
             {
@@ -78,6 +91,9 @@ public override SchemaShape GetOutputSchema(SchemaShape inputSchema)
                 {
                     if (Transformer.ValueColumnType is VectorType)
                         throw Host.ExceptNotSupp("Column '{0}' cannot be mapped to values when the column and the map values are both vector type.", inputColumnName);
+                    // if input to the estimator is of vector type then output should always be vector.
+                    // The transformer maps each item in input vector to the values in the dictionary
+                    // producing a vector as output.
                     vectorKind = originalColumn.Kind;
                 }
                 // Create the Value column

test/Microsoft.ML.Tests/Transformers/ValueMappingTests.cs

@@ -6,6 +6,7 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using Microsoft.Data.DataView;
 using Microsoft.ML.Core.Data;
 using Microsoft.ML.Data;
 using Microsoft.ML.Model;
@@ -159,6 +160,13 @@ public void ValueMapVectorValueTest()
                 new int[] {400, 500, 600, 700 }};
 
             var estimator = new ValueMappingEstimator<string, int>(Env, keys, values, new[] { ("D", "A"), ("E", "B"), ("F", "C") });
+            var schema = estimator.GetOutputSchema(SchemaShape.Create(dataView.Schema));
+            foreach (var name in new[] { "D", "E", "F" })
+            {
+                Assert.True(schema.TryFindColumn(name, out var originalColumn));
+                Assert.Equal(SchemaShape.Column.VectorKind.VariableVector, originalColumn.Kind);
+            }
+
             var t = estimator.Fit(dataView);
 
             var result = t.Transform(dataView);
@@ -509,6 +517,42 @@ public void ValueMappingWorkout()
             TestEstimatorCore(est, validFitInput: dataView, invalidInput: badDataView);
         }
 
+        [Fact]
+        public void ValueMappingValueTypeIsVectorWorkout()
+        {
+            var data = new[] { new TestClass() { A = "bar", B = "test", C = "foo" } };
+            var dataView = ML.Data.ReadFromEnumerable(data);
+            var badData = new[] { new TestWrong() { A = "bar", B = 1.2f } };
+            var badDataView = ML.Data.ReadFromEnumerable(badData);
+
+            var keys = new List<string>() { "foo", "bar", "test" };
+            var values = new List<int[]>() {
+                new int[] {2, 3, 4 },
+                new int[] {100, 200 },
+                new int[] {400, 500, 600, 700 }};
+
+            // Workout on value mapping
+            var est = ML.Transforms.Conversion.ValueMap(keys, values, new[] { ("D", "A"), ("E", "B"), ("F", "C") });
+            TestEstimatorCore(est, validFitInput: dataView, invalidInput: badDataView);
+        }
+
+        [Fact]
+        public void ValueMappingInputIsVectorWorkout()
+        {
+            var data = new[] { new TestClass() { B = "bar test foo" } };
+            var dataView = ML.Data.ReadFromEnumerable(data);
+
+            var badData = new[] { new TestWrong() { B = 1.2f } };
+            var badDataView = ML.Data.ReadFromEnumerable(badData);
+
+            var keys = new List<ReadOnlyMemory<char>>() { "foo".AsMemory(), "bar".AsMemory(), "test".AsMemory(), "wahoo".AsMemory() };
+            var values = new List<int>() { 1, 2, 3, 4 };
+
+            var est = ML.Transforms.Text.TokenizeWords("TokenizeB", "B")
+                .Append(ML.Transforms.Conversion.ValueMap(keys, values, new[] { ("VecB", "TokenizeB") }));
+            TestEstimatorCore(est, validFitInput: dataView, invalidInput: badDataView);
+        }
+
         [Fact]
         void TestCommandLine()
         {