tests, somewhat pigsty

Author: Ivan Matantsev

src/Microsoft.ML.Data/Transforms/TermTransform.cs

@@ -175,7 +175,7 @@ public ColumnInfo(string input, string output, int maxNumTerms = TermEstimator.D
             public readonly string[] Term;
             public readonly bool TextKeyValues;
 
-            protected string Terms { get; set; }
+            protected internal string Terms { get; set; }
         }
 
         public const string Summary = "Converts input values (words, numbers, etc.) to index in a dictionary.";

src/Microsoft.ML.Transforms/CategoricalStaticExtensions.cs

@@ -0,0 +1,80 @@
+<#@ template debug="false" hostspecific="false" language="C#" #>
+<#@ assembly name="System.Core" #>
+<#@ import namespace="System.Linq" #>
+<#@ import namespace="System.Text" #>
+<#@ import namespace="System.Collections.Generic" #>
+<#@ output extension=".cs" #>
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using Microsoft.ML.Data.StaticPipe.Runtime;
+
+namespace Microsoft.ML.Runtime.Data
+{
+    public static partial class CategoricalStaticExtensions
+    {
+        // Do not edit this file directly. Rather, it is generated out of TermStaticExtensions.tt.
+<#
+// Let's skip the time-based types for now.
+foreach (string typeName in new string[] { "string", "float", "double", "sbyte", "short", "int", "long", "byte", "ushort", "uint", "ulong", "bool" }) {
+bool startRegionBlock = true;
+#>
+
+        #region For <#=typeName#> inputs.
+<#
+foreach (bool inputIsKey in new bool[] { false, true }) {
+foreach (string arityName in new string[] { "Scalar", "Vector", "VarVector" }) {
+string onFitType = typeName == "string" ? "ReadOnlyMemory<char>" : typeName;
+bool omitInputArity = arityName == "Scalar" && inputIsKey;
+bool isNumeric = typeName != "string" && typeName != "bool";
+bool isScalar = arityName == "Scalar";
+bool isVarVector = arityName == "VarVector";
+
+if (!startRegionBlock) { // Put lines between the declarations to make them look pretty, but not after the region tag.
+#>
+
+<#}
+startRegionBlock = false;
+#>
+        /// <summary>
+        /// Map values to a key-value representation, where the key type's values are those values observed in the input
+        /// during fitting. During transformation, any values unobserved during fitting will map to the missing key.
+<#
+if (typeName == "string") { #>
+        /// Because the empty string is never entered into the dictionary, it will always map to the missing key.
+<# }
+if (typeName == "float" || typeName == "double") { #>
+        /// Because <c>NaN</c> floating point values are never entered into the dictionary, and they will always map to the missing key.
+<# }
+if (!isScalar && isNumeric && !inputIsKey) { #>
+        /// Zero is considered a valid value and so will be entered into the dictionary if observed. The potential perf
+        /// implication in that case is that sparse input numeric vectors will map to dense output key vectors.
+<# }
+if (inputIsKey) { #>
+        /// We are inputting a key type with values, and in that case the dictionary is considered to be built over the
+        /// values of the keys, rather than the keys themselves. This also mean the key-values learned for the output
+        /// will be a subset of the key-values in the input.
+<# }
+
+#>
+        /// </summary>
+        /// <param name="input">The input column.</param>
+        /// <param name="outputKind"></param>
+        /// <param name="order">The ordering policy for what order values will appear in the enumerated set.</param>
+        /// <param name="maxItems">The maximum number of items.</param>
+
+        /// <returns>The key-valued column.</returns>
+        public static <#=isVarVector?"Var":""#>Vector<float> OneHotEncoding<#=inputIsKey?"<T>":""#>(this <#=omitInputArity?"":arityName+"<"#><#=inputIsKey?"Key<T, ":""#><#=typeName#>><#=inputIsKey&&!isScalar?">":""#> input,
+                OneHotOutputKind outputKind = DefOut, KeyValueOrder order = DefSort, int maxItems = DefMax)
+            => new Impl<#=arityName#><<#=typeName#>>(Contracts.CheckRef(input, nameof(input)), new Config(order, maxItems, outputKind));
+<#
+} }
+#>
+        #endregion
+<#
+}
+#>
+    }
+}

src/Microsoft.ML.Transforms/CategoricalStaticExtensions.tt

@@ -15,6 +15,7 @@
 using Microsoft.ML.Runtime.Internal.Utilities;
 using Microsoft.ML.Runtime.Internal.Internallearn;
 using Microsoft.ML.Core.Data;
+using Microsoft.ML.Data.StaticPipe.Runtime;
 
 [assembly: LoadableClass(CategoricalTransform.Summary, typeof(IDataTransform), typeof(CategoricalTransform), typeof(CategoricalTransform.Arguments), typeof(SignatureDataTransform),
     CategoricalTransform.UserName, "CategoricalTransform", "CatTransform", "Categorical", "Cat")]
@@ -119,7 +120,7 @@ public Arguments()
 
         public const string UserName = "Categorical Transform";
 
-        public static IDataView Create(IHostEnvironment env, Arguments args, IDataView input)
+        public static IDataTransform Create(IHostEnvironment env, Arguments args, IDataView input)
         {
             Contracts.CheckValue(env, nameof(env));
             var h = env.Register("Categorical");
@@ -140,7 +141,7 @@ public static IDataView Create(IHostEnvironment env, Arguments args, IDataView i
                 col.SetTerms(column.Terms);
                 columns.Add(col);
             }
-            return Create(env, input, columns.ToArray());
+            return Create(env, input, columns.ToArray()) as IDataTransform;
         }
 
         public static IDataView Create(IHostEnvironment env, IDataView input, params CategoricalEstimator.ColumnInfo[] columns)
@@ -306,4 +307,109 @@ public static CommonOutputs.TransformOutput KeyToText(IHostEnvironment env, KeyT
             return new CommonOutputs.TransformOutput { Model = new TransformModel(env, xf, input.Data), OutputData = xf };
         }
     }
+
+    public enum OneHotOutputKind : byte
+    {
+        /// <summary>
+        /// Output is a bag (multi-set) vector
+        /// </summary>
+        Bag = 1,
+
+        /// <summary>
+        /// Output is an indicator vector
+        /// </summary>
+        Ind = 2,
+
+        /// <summary>
+        /// Output is a key value
+        /// </summary>
+        Key = 3,
+
+        /// <summary>
+        /// Output is binary encoded
+        /// </summary>
+        Bin = 4,
+    }
+
+    public static partial class CategoricalStaticExtensions
+    {
+        // I am not certain I see a good way to cover the distinct types beyond complete enumeration.
+        // Raw generics would allow illegal possible inputs, e.g., Scalar<Bitmap>. So, this is a partial
+        // class, and all the public facing extension methods for each possible type are in a T4 generated result.
+
+        private const KeyValueOrder DefSort = (KeyValueOrder)TermEstimator.Defaults.Sort;
+        private const int DefMax = TermEstimator.Defaults.MaxNumTerms;
+        private const OneHotOutputKind DefOut = (OneHotOutputKind)CategoricalEstimator.Defaults.OutKind;
+
+        private struct Config
+        {
+            public readonly KeyValueOrder Order;
+            public readonly int Max;
+            public readonly OneHotOutputKind OutputKind;
+
+            public Config(KeyValueOrder order, int max, OneHotOutputKind outputKind)
+            {
+                Order = order;
+                Max = max;
+                OutputKind = outputKind;
+            }
+        }
+
+        private interface IOneHotCol
+        {
+            PipelineColumn Input { get; }
+            Config Config { get; }
+        }
+
+        private sealed class ImplScalar<T> :Vector<float>, IOneHotCol
+        {
+            public PipelineColumn Input { get; }
+            public Config Config { get; }
+            public ImplScalar(PipelineColumn input, Config config) : base(Rec.Inst, input)
+            {
+                Input = input;
+                Config = config;
+            }
+        }
+
+        private sealed class ImplVector<T> : Vector<float>, IOneHotCol
+        {
+            public PipelineColumn Input { get; }
+            public Config Config { get; }
+            public ImplVector(PipelineColumn input, Config config) : base(Rec.Inst, input)
+            {
+                Input = input;
+                Config = config;
+            }
+        }
+
+        private sealed class ImplVarVector<T> : VarVector<float>, IOneHotCol
+        {
+            public PipelineColumn Input { get; }
+            public Config Config { get; }
+            public ImplVarVector(PipelineColumn input, Config config) : base(Rec.Inst, input)
+            {
+                Input = input;
+                Config = config;
+            }
+        }
+
+        private sealed class Rec : EstimatorReconciler
+        {
+            public static readonly Rec Inst = new Rec();
+
+            public override IEstimator<ITransformer> Reconcile(IHostEnvironment env, PipelineColumn[] toOutput,
+                IReadOnlyDictionary<PipelineColumn, string> inputNames, IReadOnlyDictionary<PipelineColumn, string> outputNames, IReadOnlyCollection<string> usedNames)
+            {
+                var infos = new CategoricalEstimator.ColumnInfo[toOutput.Length];
+                for (int i = 0; i < toOutput.Length; ++i)
+                {
+                    var tcol = (IOneHotCol)toOutput[i];
+                    infos[i] = new CategoricalEstimator.ColumnInfo(inputNames[tcol.Input], outputNames[toOutput[i]], (CategoricalTransform.OutputKind)tcol.Config.OutputKind,
+                        tcol.Config.Max, (TermTransform.SortOrder)tcol.Config.Order);
+                }
+                return new CategoricalEstimator(env, infos);
+            }
+        }
+    }
 }

src/Microsoft.ML.Transforms/CategoricalTransform.cs

@@ -51,6 +51,11 @@
   </ItemGroup>
 
   <ItemGroup>
+    <Compile Update="CategoricalStaticExtensions.cs">
+      <DesignTime>True</DesignTime>
+      <AutoGen>True</AutoGen>
+      <DependentUpon>CategoricalStaticExtensions.tt</DependentUpon>
+    </Compile>
     <Compile Update="Properties\Resources.Designer.cs">
       <DesignTime>True</DesignTime>
       <AutoGen>True</AutoGen>
@@ -65,4 +70,15 @@
     </EmbeddedResource>
   </ItemGroup>
 
+  <ItemGroup>
+    <None Update="CategoricalStaticExtensions.tt">
+      <Generator>TextTemplatingFileGenerator</Generator>
+      <LastGenOutput>CategoricalStaticExtensions.cs</LastGenOutput>
+    </None>
+  </ItemGroup>
+
+  <ItemGroup>
+    <Service Include="{508349b6-6b84-4df5-91f0-309beebad82d}" />
+  </ItemGroup>
+
 </Project>

src/Microsoft.ML.Transforms/Microsoft.ML.Transforms.csproj

@@ -0,0 +1,13 @@
+#@ TextLoader{
+#@   header+
+#@   sep=tab
+#@   col=A:R4:0-9
+#@   col=B:R4:10-49
+#@   col=C:R4:50-59
+#@   col=D:R4:60-99
+#@ }
+5	3	6	4	8	1	2	7	10	9	[0].5	[0].1	[0].4	[0].3	[0].6	[0].8	[0].10	[0].2	[0].7	[0].9	[1].5	[1].1	[1].4	[1].3	[1].6	[1].8	[1].10	[1].2	[1].7	[1].9	[2].5	[2].1	[2].4	[2].3	[2].6	[2].8	[2].10	[2].2	[2].7	[2].9	[3].5	[3].1	[3].4	[3].3	[3].6	[3].8	[3].10	[3].2	[3].7	[3].9	5	3	6	4	8	1	2	7	10	9	[0].5	[0].1	[0].4	[0].3	[0].6	[0].8	[0].10	[0].2	[0].7	[0].9	[1].5	[1].1	[1].4	[1].3	[1].6	[1].8	[1].10	[1].2	[1].7	[1].9	[2].5	[2].1	[2].4	[2].3	[2].6	[2].8	[2].10	[2].2	[2].7	[2].9	[3].5	[3].1	[3].4	[3].3	[3].6	[3].8	[3].10	[3].2	[3].7	[3].9
+100	0:1	10:1	21:1	31:1	41:1	50:1	60:1	71:1	81:1	91:1
+100	0:1	10:1	22:1	32:1	40:1	50:1	60:1	72:1	82:1	90:1
+100	1:1	13:1	21:1	31:1	41:1	51:1	63:1	71:1	81:1	91:1
+100	2:1	14:1	25:1	35:1	41:1	52:1	64:1	75:1	85:1	91:1

test/BaselineOutput/SingleDebug/Categorical/featurized.tsv

@@ -0,0 +1,13 @@
+#@ TextLoader{
+#@   header+
+#@   sep=tab
+#@   col=A:R4:0-9
+#@   col=B:R4:10-49
+#@   col=C:R4:50-59
+#@   col=D:R4:60-99
+#@ }
+5	3	6	4	8	1	2	7	10	9	[0].5	[0].1	[0].4	[0].3	[0].6	[0].8	[0].10	[0].2	[0].7	[0].9	[1].5	[1].1	[1].4	[1].3	[1].6	[1].8	[1].10	[1].2	[1].7	[1].9	[2].5	[2].1	[2].4	[2].3	[2].6	[2].8	[2].10	[2].2	[2].7	[2].9	[3].5	[3].1	[3].4	[3].3	[3].6	[3].8	[3].10	[3].2	[3].7	[3].9	5	3	6	4	8	1	2	7	10	9	[0].5	[0].1	[0].4	[0].3	[0].6	[0].8	[0].10	[0].2	[0].7	[0].9	[1].5	[1].1	[1].4	[1].3	[1].6	[1].8	[1].10	[1].2	[1].7	[1].9	[2].5	[2].1	[2].4	[2].3	[2].6	[2].8	[2].10	[2].2	[2].7	[2].9	[3].5	[3].1	[3].4	[3].3	[3].6	[3].8	[3].10	[3].2	[3].7	[3].9
+100	0:1	10:1	21:1	31:1	41:1	50:1	60:1	71:1	81:1	91:1
+100	0:1	10:1	22:1	32:1	40:1	50:1	60:1	72:1	82:1	90:1
+100	1:1	13:1	21:1	31:1	41:1	51:1	63:1	71:1	81:1	91:1
+100	2:1	14:1	25:1	35:1	41:1	52:1	64:1	75:1	85:1	91:1