adding some trainer extensions on the StandardLearners catalog. Correcting namespace, and names

docs/code/MlNetCookBook.md

@@ -1132,7 +1132,7 @@ var learningPipeline = reader.MakeNewEstimator()
         BagOfBigrams: r.Message.NormalizeText().ToBagofHashedWords(ngramLength: 2, allLengths: false),
 
         // NLP pipeline 3: bag of tri-character sequences with TF-IDF weighting.
-        BagOfTrichar: r.Message.TokenizeIntoCharacters().ToNgrams(ngramLength: 3, weighting: NgramCountingEstimator.WeightingCriteria.TfIdf),
+        BagOfTrichar: r.Message.TokenizeIntoCharacters().ToNgrams(ngramLength: 3, weighting: NgramExtractingEstimator.WeightingCriteria.TfIdf),
 
         // NLP pipeline 4: word embeddings.
         Embeddings: r.Message.NormalizeText().TokenizeText().WordEmbeddings(WordEmbeddingsExtractorTransformer.PretrainedModelKind.GloVeTwitter25D)
@@ -1186,8 +1186,8 @@ var dynamicPipeline =
 
     // NLP pipeline 3: bag of tri-character sequences with TF-IDF weighting.
     .Append(mlContext.Transforms.Text.TokenizeCharacters("Message", "MessageChars"))
-    .Append(new NgramCountingEstimator(mlContext, "MessageChars", "BagOfTrichar",
-                ngramLength: 3, weighting: NgramTokenizingTransformer.WeightingCriteria.TfIdf))
+    .Append(new NgramExtractingEstimator(mlContext, "MessageChars", "BagOfTrichar",
+                ngramLength: 3, weighting: NgramExtractingEstimator.WeightingCriteria.TfIdf))
 
     // NLP pipeline 4: word embeddings.
     .Append(mlContext.Transforms.Text.TokenizeWords("NormalizedMessage", "TokenizedMessage"))
@@ -1297,7 +1297,7 @@ var dynamicPipeline =
     // Concatenate all the features together into one column 'Features'.
     mlContext.Transforms.Concatenate("Features", "SepalLength", "SepalWidth", "PetalLength", "PetalWidth")
     // Note that the label is text, so it needs to be converted to key.
-    .Append(mlContext.Transforms.Categorical.MapValueToKey("Label"), TransformerScope.TrainTest)
+    .Append(mlContext.Transforms.Conversions.MapValueToKey("Label"), TransformerScope.TrainTest)
     // Use the multi-class SDCA model to predict the label using features.
     .Append(mlContext.MulticlassClassification.Trainers.StochasticDualCoordinateAscent());
 

docs/samples/Microsoft.ML.Samples/Dynamic/MatrixFactorization.cs

@@ -4,9 +4,6 @@
 using System;
 using System.Collections.Generic;
 
-// NOTE: WHEN ADDING TO THE FILE, ALWAYS APPEND TO THE END OF IT. 
-// If you change the existinc content, check that the files referencing it in the XML documentation are still correct, as they reference
-// line by line. 
 namespace Microsoft.ML.Samples.Dynamic
 {
     public class MatrixFactorizationExample
@@ -67,7 +64,7 @@ public static void MatrixFactorizationInMemoryData()
             // Create a matrix factorization trainer which may consume "Value" as the training label, "MatrixColumnIndex" as the
             // matrix's column index, and "MatrixRowIndex" as the matrix's row index. Here nameof(...) is used to extract field
             // names' in MatrixElement class.
-            var pipeline = new MatrixFactorizationTrainer(mlContext, 
+            var pipeline = mlContext.Recommendation().Trainers.MatrixFactorization( 
                 nameof(MatrixElement.MatrixColumnIndex),
                 nameof(MatrixElement.MatrixRowIndex),
                 nameof(MatrixElement.Value),
@@ -85,7 +82,7 @@ public static void MatrixFactorizationInMemoryData()
             var prediction = model.Transform(dataView);
 
             // Calculate regression matrices for the prediction result.
-            var metrics = mlContext.Regression.Evaluate(prediction,
+            var metrics = mlContext.Recommendation().Evaluate(prediction,
                 label: nameof(MatrixElement.Value), score: nameof(MatrixElementForScore.Score));
 
             // Print out some metrics for checking the model's quality.

docs/samples/Microsoft.ML.Samples/Microsoft.ML.Samples.csproj

@@ -16,6 +16,7 @@
 
     <NativeAssemblyReference Include="CpuMathNative" />
     <NativeAssemblyReference Include="FastTreeNative" />
+    <NativeAssemblyReference Include="MatrixFactorizationNative" />
 
     <ProjectReference Include="..\..\..\src\Microsoft.ML.Analyzer\Microsoft.ML.Analyzer.csproj">
       <ReferenceOutputAssembly>false</ReferenceOutputAssembly>

docs/samples/Microsoft.ML.Samples/Program.cs

@@ -6,7 +6,7 @@ internal static class Program
     {
         static void Main(string[] args)
         {
-            NormalizerExample.Normalizer();
+            MatrixFactorizationExample.MatrixFactorizationInMemoryData();
         }
     }
 }

src/Microsoft.ML.Data/EntryPoints/SchemaManipulation.cs

@@ -37,14 +37,14 @@ public static CommonOutputs.TransformOutput SelectColumns(IHostEnvironment env,
             return new CommonOutputs.TransformOutput { Model = new TransformModel(env, xf, input.Data), OutputData = xf };
         }
 
-        [TlcModule.EntryPoint(Name = "Transforms.ColumnCopier", Desc = "Duplicates columns from the dataset", UserName = ColumnsCopyingTransformer.UserName, ShortName = ColumnsCopyingTransformer.ShortName)]
-        public static CommonOutputs.TransformOutput CopyColumns(IHostEnvironment env, ColumnsCopyingTransformer.Arguments input)
+        [TlcModule.EntryPoint(Name = "Transforms.ColumnCopier", Desc = "Duplicates columns from the dataset", UserName = ColumnCopyingTransformer.UserName, ShortName = ColumnCopyingTransformer.ShortName)]
+        public static CommonOutputs.TransformOutput CopyColumns(IHostEnvironment env, ColumnCopyingTransformer.Arguments input)
         {
             Contracts.CheckValue(env, nameof(env));
             var host = env.Register("CopyColumns");
             host.CheckValue(input, nameof(input));
             EntryPointUtils.CheckInputArgs(host, input);
-            var xf = ColumnsCopyingTransformer.Create(env, input, input.Data);
+            var xf = ColumnCopyingTransformer.Create(env, input, input.Data);
             return new CommonOutputs.TransformOutput { Model = new TransformModel(env, xf, input.Data), OutputData = xf };
         }
     }

src/Microsoft.ML.Data/EntryPoints/ScoreColumnSelector.cs

@@ -101,7 +101,7 @@ public static CommonOutputs.TransformOutput RenameBinaryPredictionScoreColumns(I
                         copyCols.Add((source, name));
                     }
 
-                    var copyColumn = new ColumnsCopyingTransformer(env, copyCols.ToArray()).Transform(input.Data);
+                    var copyColumn = new ColumnCopyingTransformer(env, copyCols.ToArray()).Transform(input.Data);
                     var dropColumn = ColumnSelectingTransformer.CreateDrop(env, copyColumn, copyCols.Select(c => c.Source).ToArray());
                     return new CommonOutputs.TransformOutput { Model = new TransformModel(env, dropColumn, input.Data), OutputData = dropColumn };
                 }

src/Microsoft.ML.Data/Evaluators/AnomalyDetectionEvaluator.cs

@@ -724,7 +724,7 @@ protected override void PrintFoldResultsCore(IChannel ch, Dictionary<string, IDa
             colsToKeep.Add(AnomalyDetectionEvaluator.OverallMetrics.ThreshAtNumPos);
             colsToKeep.Add(BinaryClassifierEvaluator.Auc);
 
-            overall = new ColumnsCopyingTransformer(Host, cols).Transform(overall);
+            overall = new ColumnCopyingTransformer(Host, cols).Transform(overall);
             IDataView fold = ColumnSelectingTransformer.CreateKeep(Host, overall, colsToKeep.ToArray());
 
             string weightedFold;

src/Microsoft.ML.Data/Evaluators/BinaryClassifierEvaluator.cs

@@ -1356,7 +1356,7 @@ protected override void PrintFoldResultsCore(IChannel ch, Dictionary<string, IDa
             if (fold.Schema.TryGetColumnIndex(MetricKinds.ColumnNames.StratVal, out index))
                 colsToKeep.Add(MetricKinds.ColumnNames.StratVal);
 
-            fold = new ColumnsCopyingTransformer(Host, cols).Transform(fold);
+            fold = new ColumnCopyingTransformer(Host, cols).Transform(fold);
 
             // Select the columns that are specified in the Copy
             fold = ColumnSelectingTransformer.CreateKeep(Host, fold, colsToKeep.ToArray());

src/Microsoft.ML.Data/Evaluators/MamlEvaluator.cs

@@ -245,7 +245,7 @@ private IDataView WrapPerInstance(RoleMappedData perInst)
             foreach (var col in GetPerInstanceColumnsToSave(perInst.Schema))
                 colsToKeep.Add(col);
 
-            idv = new ColumnsCopyingTransformer(Host, cols.ToArray()).Transform(idv);
+            idv = new ColumnCopyingTransformer(Host, cols.ToArray()).Transform(idv);
             idv = ColumnSelectingTransformer.CreateKeep(Host, idv, colsToKeep.ToArray());
             return GetPerInstanceMetricsCore(idv, perInst.Schema);
         }

src/Microsoft.ML.Data/Evaluators/MulticlassClassifierEvaluator.cs

@@ -1040,7 +1040,7 @@ protected override IDataView GetOverallResultsCore(IDataView overall)
 
         private IDataView ChangeTopKAccColumnName(IDataView input)
         {
-            input = new ColumnsCopyingTransformer(Host, (MultiClassClassifierEvaluator.TopKAccuracy, string.Format(TopKAccuracyFormat, _outputTopKAcc))).Transform(input);
+            input = new ColumnCopyingTransformer(Host, (MultiClassClassifierEvaluator.TopKAccuracy, string.Format(TopKAccuracyFormat, _outputTopKAcc))).Transform(input);
             return ColumnSelectingTransformer.CreateDrop(Host, input, MultiClassClassifierEvaluator.TopKAccuracy);
         }
 

src/Microsoft.ML.Data/MLContext.cs

@@ -35,6 +35,7 @@ public sealed class MLContext : IHostEnvironment
         /// Trainers and tasks specific to clustering problems.
         /// </summary>
         public ClusteringContext Clustering { get; }
+
         /// <summary>
         /// Trainers and tasks specific to ranking problems.
         /// </summary>

src/Microsoft.ML.Data/StaticPipe/Reconciler.cs

@@ -64,7 +64,7 @@ public EstimatorReconciler() : base() { }
         /// subset of estimator transforms do not allow this: they produce columns whose names are unconfigurable. For
         /// these, there is this collection which provides the names used by the analysis tool. If the estimator under
         /// construction must use one of the names here, then they are responsible for "saving" the column they will
-        /// overwrite using applications of the <see cref="ColumnsCopyingEstimator"/>. Note that if the estimator under
+        /// overwrite using applications of the <see cref="ColumnCopyingEstimator"/>. Note that if the estimator under
         /// construction has complete control over what columns it produces, there is no need for it to pay this argument
         /// any attention.</param>
         /// <returns>Returns an estimator.</returns>

src/Microsoft.ML.Data/StaticPipe/StaticPipeUtils.cs

@@ -163,7 +163,7 @@ internal static IDataReaderEstimator<TIn, IDataReader<TIn>>
 
             // If any renamings were necessary, create the CopyColumns estimator.
             if (toCopy.Count > 0)
-                estimator = new ColumnsCopyingEstimator(env, toCopy.ToArray());
+                estimator = new ColumnCopyingEstimator(env, toCopy.ToArray());
 
             // First clear the inputs from zero-dependencies yet to be resolved.
             foreach (var col in baseInputs)
@@ -282,7 +282,7 @@ internal static IDataReaderEstimator<TIn, IDataReader<TIn>>
             // If any final renamings were necessary, insert the appropriate CopyColumns transform.
             if (toCopy.Count > 0)
             {
-                var copyEstimator = new ColumnsCopyingEstimator(env, toCopy.ToArray());
+                var copyEstimator = new ColumnCopyingEstimator(env, toCopy.ToArray());
                 if (estimator == null)
                     estimator = copyEstimator;
                 else

src/Microsoft.ML.Data/StaticPipe/TrainerEstimatorReconciler.cs

@@ -56,7 +56,7 @@ protected TrainerEstimatorReconciler(PipelineColumn[] inputs, string[] outputNam
 
         /// <summary>
         /// Produces the estimator. Note that this is made out of <see cref="ReconcileCore(IHostEnvironment, string[])"/>'s
-        /// return value, plus whatever usages of <see cref="ColumnsCopyingEstimator"/> are necessary to avoid collisions with
+        /// return value, plus whatever usages of <see cref="ColumnCopyingEstimator"/> are necessary to avoid collisions with
         /// the output names fed to the constructor. This class provides the implementation, and subclasses should instead
         /// override <see cref="ReconcileCore(IHostEnvironment, string[])"/>.
         /// </summary>
@@ -103,7 +103,7 @@ public sealed override IEstimator<ITransformer> Reconcile(IHostEnvironment env,
                         newInputNames[p.Key] = old2New.ContainsKey(p.Value) ? old2New[p.Value] : p.Value;
                     inputNames = newInputNames;
                 }
-                result = new ColumnsCopyingEstimator(env, old2New.Select(p => (p.Key, p.Value)).ToArray());
+                result = new ColumnCopyingEstimator(env, old2New.Select(p => (p.Key, p.Value)).ToArray());
             }
 
             // Map the inputs to the names.
@@ -129,7 +129,7 @@ public sealed override IEstimator<ITransformer> Reconcile(IHostEnvironment env,
             foreach (var p in old2New)
                 toRename.Add((p.Value, p.Key));
             if (toRename.Count > 0)
-                result = result.Append(new ColumnsCopyingEstimator(env, toRename.ToArray()));
+                result = result.Append(new ColumnCopyingEstimator(env, toRename.ToArray()));
 
             return result;
         }

src/Microsoft.ML.Data/TrainContext.cs

@@ -21,6 +21,8 @@ namespace Microsoft.ML
     public abstract class TrainContextBase
     {
         protected readonly IHost Host;
+
+        [BestFriend]
         internal IHostEnvironment Environment => Host;
 
         /// <summary>
@@ -162,6 +164,7 @@ private void EnsureStratificationColumn(ref IDataView data, ref string stratific
         /// </summary>
         public abstract class ContextInstantiatorBase
         {
+            [BestFriend]
             internal TrainContextBase Owner { get; }
 
             protected ContextInstantiatorBase(TrainContextBase ctx)
@@ -498,7 +501,7 @@ public RegressionEvaluator.Result Evaluate(IDataView data, string label = Defaul
     }
 
     /// <summary>
-    /// The central context for regression trainers.
+    /// The central context for ranking trainers.
     /// </summary>
     public sealed class RankingContext : TrainContextBase
     {

src/Microsoft.ML.Data/Transforms/ColumnBindingsBase.cs

@@ -6,7 +6,7 @@
 using Microsoft.ML.Runtime.CommandLine;
 using Microsoft.ML.Runtime.Internal.Utilities;
 using Microsoft.ML.Runtime.Model;
-using Microsoft.ML.Transforms.Categorical;
+using Microsoft.ML.Transforms.Conversions;
 using System;
 using System.Collections.Generic;
 using System.Linq;