Hide much infrastructure in data (#2300)

* Internalize normalizer infrastructure, normalizer transform, entry-points.
* Internalization of component level command line infrastructure.
* Parse/TryUnparse methods internalized.
* Lockdown of column abstract class constructors to make them unextensible.
* Removal of legacy API derived interfaces and abstract classes for columns.
* Internalization of column bindings utilities and infrastructure.
* Hide filtering/misc IDataTransform implementations and associated entry-points.
* Internalize or move much of entry-points.
* Internalize common outputs of entry-points and deal with resulting fallout.
* Internalization of common inputs.
* Internalization of more model saving infrastructure.

Author: TomFinley

src/Microsoft.ML.Core/CommandLine/CmdParser.cs

@@ -1359,17 +1359,17 @@ public ArgumentInfo(Type type, Argument argDef, Argument[] args, Dictionary<stri
             private static MethodInfo GetParseMethod(Type type)
             {
                 Contracts.AssertValue(type);
-                var meth = type.GetMethod("Parse", new[] { typeof(string) });
-                if (meth != null && meth.IsStatic && meth.IsPublic && meth.ReturnType == type)
+                var meth = type.GetMethod("Parse", BindingFlags.Static | BindingFlags.NonPublic | BindingFlags.Public, binder: null, new[] { typeof(string) }, null);
+                if (meth != null && meth.IsStatic && !meth.IsPrivate && meth.ReturnType == type)
                     return meth;
                 return null;
             }
 
             private static MethodInfo GetUnparseMethod(Type type)
             {
                 Contracts.AssertValue(type);
-                var meth = type.GetMethod("TryUnparse", new[] { typeof(StringBuilder) });
-                if (meth != null && !meth.IsStatic && meth.IsPublic && meth.ReturnType == typeof(bool))
+                var meth = type.GetMethod("TryUnparse", BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public, binder: null, new[] { typeof(StringBuilder) }, null);
+                if (meth != null && !meth.IsPrivate && meth.ReturnType == typeof(bool))
                     return meth;
                 return null;
             }

src/Microsoft.ML.Core/ComponentModel/ComponentCatalog.cs

@@ -468,7 +468,7 @@ private void ScanForEntryPoints(LoadableClassInfo info)
             var type = info.LoaderType;
 
             // Scan for entry points.
-            foreach (var methodInfo in type.GetMethods(BindingFlags.Static | BindingFlags.Public))
+            foreach (var methodInfo in type.GetMethods(BindingFlags.Static | BindingFlags.Public | BindingFlags.NonPublic))
             {
                 var attr = methodInfo.GetCustomAttributes(typeof(TlcModule.EntryPointAttribute), false).FirstOrDefault() as TlcModule.EntryPointAttribute;
                 if (attr == null)
@@ -727,7 +727,7 @@ internal LoadableClassInfo GetLoadableClassInfo(string loadName, Type signatureT
         [BestFriend]
         internal IEnumerable<EntryPointInfo> AllEntryPoints()
         {
-            return _entryPoints.AsEnumerable();
+            return _entryPoints;
         }
 
         [BestFriend]

src/Microsoft.ML.Data/DataLoadSave/PartitionedFileLoader.cs

@@ -91,7 +91,7 @@ public sealed class Column
             [Argument(ArgumentType.Required, HelpText = "Index of the directory representing this column.")]
             public int Source;
 
-            public static Column Parse(string str)
+            internal static Column Parse(string str)
             {
                 Contracts.AssertNonEmpty(str);
 
@@ -103,7 +103,7 @@ public static Column Parse(string str)
                 return null;
             }
 
-            public static bool TryParse(string str, out Column column)
+            private static bool TryParse(string str, out Column column)
             {
                 column = null;
 
@@ -138,7 +138,7 @@ public static bool TryParse(string str, out Column column)
                 return true;
             }
 
-            public bool TryUnparse(StringBuilder sb)
+            internal bool TryUnparse(StringBuilder sb)
             {
                 Contracts.AssertValue(sb);
 

src/Microsoft.ML.Data/DataLoadSave/Text/TextLoader.cs

@@ -74,7 +74,7 @@ public Column(string name, DataKind? type, Range[] source, KeyCount keyCount = n
             [Argument(ArgumentType.Multiple, HelpText = "For a key column, this defines the range of values", ShortName = "key")]
             public KeyCount KeyCount;
 
-            public static Column Parse(string str)
+            internal static Column Parse(string str)
             {
                 Contracts.AssertNonEmpty(str);
 
@@ -125,7 +125,7 @@ public static bool TryParseSourceEx(string str, out Range[] ranges)
                 return true;
             }
 
-            public bool TryUnparse(StringBuilder sb)
+            internal bool TryUnparse(StringBuilder sb)
             {
                 Contracts.AssertValue(sb);
 
@@ -172,7 +172,7 @@ public bool TryUnparse(StringBuilder sb)
             /// <summary>
             ///  Returns <c>true</c> iff the ranges are disjoint, and each range satisfies 0 &lt;= min &lt;= max.
             /// </summary>
-            public bool IsValid()
+            internal bool IsValid()
             {
                 if (Utils.Size(Source) == 0)
                     return false;
@@ -258,7 +258,7 @@ public Range(int min, int? max)
             [Argument(ArgumentType.AtMostOnce, HelpText = "Force scalar columns to be treated as vectors of length one", ShortName = "vector")]
             public bool ForceVector;
 
-            public static Range Parse(string str)
+            internal static Range Parse(string str)
             {
                 Contracts.AssertNonEmpty(str);
 
@@ -314,7 +314,7 @@ private bool TryParse(string str)
                 return true;
             }
 
-            public bool TryUnparse(StringBuilder sb)
+            internal bool TryUnparse(StringBuilder sb)
             {
                 Contracts.AssertValue(sb);
                 char dash = AllOther ? '~' : '-';

src/Microsoft.ML.Data/Dirty/PredictorUtils.cs

@@ -64,7 +64,7 @@ public static void SaveText(IChannel ch, IPredictor predictor, RoleMappedSchema
         }
 
         /// <summary>
-        /// Save the model in binary format (if it can save itself)
+        /// Save the model in binary format (if it can save itself).
         /// </summary>
         public static void SaveBinary(IChannel ch, IPredictor predictor, BinaryWriter writer)
         {

src/Microsoft.ML.Data/EntryPoints/Cache.cs

@@ -13,7 +13,7 @@
 [assembly: LoadableClass(typeof(void), typeof(Cache), null, typeof(SignatureEntryPointModule), "Cache")]
 namespace Microsoft.ML.EntryPoints
 {
-    public static class Cache
+    internal static class Cache
     {
         public enum CachingType
         {

src/Microsoft.ML.Data/EntryPoints/CommonOutputs.cs

@@ -9,9 +9,10 @@
 namespace Microsoft.ML.EntryPoints
 {
     /// <summary>
-    /// Common output classes for trainers and transforms.
+    /// Common output classes for trainers and transform entry-points.
     /// </summary>
-    public static class CommonOutputs
+    [BestFriend]
+    internal static class CommonOutputs
     {
         /// <summary>
         /// The common output class for all transforms.

src/Microsoft.ML.Data/EntryPoints/InputBase.cs

@@ -6,6 +6,7 @@
 using System.Collections.Generic;
 using Microsoft.Data.DataView;
 using Microsoft.ML.CommandLine;
+using Microsoft.ML.Core.Data;
 using Microsoft.ML.Data;
 using Microsoft.ML.Data.IO;
 using Microsoft.ML.Internal.Calibration;
@@ -18,11 +19,17 @@ namespace Microsoft.ML.EntryPoints
     [TlcModule.EntryPointKind(typeof(CommonInputs.ITransformInput))]
     public abstract class TransformInputBase
     {
+        /// <summary>
+        /// The input dataset. Used only in entry-point methods, since the normal API mechanism for feeding in a dataset to
+        /// create an <see cref="ITransformer"/> is to use the <see cref="IEstimator{TTransformer}.Fit(IDataView)"/> method.
+        /// </summary>
+        [BestFriend]
         [Argument(ArgumentType.Required, HelpText = "Input dataset", Visibility = ArgumentAttribute.VisibilityType.EntryPointsOnly, SortOrder = 1)]
-        public IDataView Data;
+        internal IDataView Data;
     }
 
-    public enum CachingOptions
+    [BestFriend]
+    internal enum CachingOptions
     {
         Auto,
         Memory,
@@ -37,10 +44,13 @@ public enum CachingOptions
     public abstract class LearnerInputBase
     {
         /// <summary>
-        /// The data to be used for training.
+        /// The data to be used for training. Used only in entry-points, since in the API the expected mechanism is
+        /// that the user iwll use the <see cref="IEstimator{TTransformer}.Fit(IDataView)"/> or some other train
+        /// method.
         /// </summary>
+        [BestFriend]
         [Argument(ArgumentType.Required, ShortName = "data", HelpText = "The data to be used for training", SortOrder = 1, Visibility = ArgumentAttribute.VisibilityType.EntryPointsOnly)]
-        public IDataView TrainingData;
+        internal IDataView TrainingData;
 
         /// <summary>
         /// Column to use for features.
@@ -49,16 +59,20 @@ public abstract class LearnerInputBase
         public string FeatureColumn = DefaultColumnNames.Features;
 
         /// <summary>
-        /// Normalize option for the feature column.
+        /// Normalize option for the feature column. Used only in entry-points, since in the API the user is expected to do this themselves.
         /// </summary>
+        [BestFriend]
         [Argument(ArgumentType.AtMostOnce, HelpText = "Normalize option for the feature column", ShortName = "norm", SortOrder = 5, Visibility = ArgumentAttribute.VisibilityType.EntryPointsOnly)]
-        public NormalizeOption NormalizeFeatures = NormalizeOption.Auto;
+        internal NormalizeOption NormalizeFeatures = NormalizeOption.Auto;
 
         /// <summary>
-        /// Whether learner should cache input training data.
+        /// Whether learner should cache input training data. Used only in entry-points, since the intended API mechanism
+        /// is that the user will use the <see cref="DataOperationsCatalog.Cache(IDataView, string[])"/> or other method
+        /// like <see cref="EstimatorChain{TLastTransformer}.AppendCacheCheckpoint(IHostEnvironment)"/>.
         /// </summary>
+        [BestFriend]
         [Argument(ArgumentType.LastOccurenceWins, HelpText = "Whether learner should cache input training data", ShortName = "cache", SortOrder = 6, Visibility = ArgumentAttribute.VisibilityType.EntryPointsOnly)]
-        public CachingOptions Caching = CachingOptions.Auto;
+        internal CachingOptions Caching = CachingOptions.Auto;
     }
 
     /// <summary>
@@ -221,7 +235,8 @@ public static TOut Train<TArg, TOut>(IHost host, TArg input,
     /// <summary>
     /// Common input interfaces for TLC components.
     /// </summary>
-    public static class CommonInputs
+    [BestFriend]
+    internal static class CommonInputs
     {
         /// <summary>
         /// Interface that all API transform input classes will implement.

src/Microsoft.ML.Data/EntryPoints/SchemaManipulation.cs

@@ -11,7 +11,7 @@
 
 namespace Microsoft.ML.EntryPoints
 {
-    public static class SchemaManipulation
+    internal static class SchemaManipulation
     {
         [TlcModule.EntryPoint(Name = "Transforms.ColumnConcatenator", Desc = ColumnConcatenatingTransformer.Summary, UserName = ColumnConcatenatingTransformer.UserName, ShortName = ColumnConcatenatingTransformer.LoadName)]
         public static CommonOutputs.TransformOutput ConcatColumns(IHostEnvironment env, ColumnConcatenatingTransformer.Arguments input)

src/Microsoft.ML.Data/EntryPoints/SelectRows.cs

@@ -9,7 +9,10 @@
 
 namespace Microsoft.ML.EntryPoints
 {
-    public static class SelectRows
+    /// <summary>
+    /// Entry point methods for row filtering and selection.
+    /// </summary>
+    internal static class SelectRows
     {
         [TlcModule.EntryPoint(Name = "Transforms.RowRangeFilter", Desc = RangeFilter.Summary, UserName = RangeFilter.UserName, ShortName = RangeFilter.LoaderSignature)]
         public static CommonOutputs.TransformOutput FilterByRange(IHostEnvironment env, RangeFilter.Arguments input)

src/Microsoft.ML.Data/EntryPoints/SummarizePredictor.cs

@@ -15,7 +15,8 @@
 
 namespace Microsoft.ML.EntryPoints
 {
-    public static class SummarizePredictor
+    [BestFriend]
+    internal static class SummarizePredictor
     {
         public abstract class InputBase
         {

src/Microsoft.ML.Data/Evaluators/AnomalyDetectionEvaluator.cs

@@ -764,7 +764,7 @@ public override IEnumerable<MetricColumn> GetOverallMetricColumns()
         }
     }
 
-    public static partial class Evaluate
+    internal static partial class Evaluate
     {
         [TlcModule.EntryPoint(Name = "Models.AnomalyDetectionEvaluator", Desc = "Evaluates an anomaly detection scored dataset.")]
         public static CommonOutputs.CommonEvaluateOutput AnomalyDetection(IHostEnvironment env, AnomalyDetectionMamlEvaluator.Arguments input)

src/Microsoft.ML.Data/Evaluators/BinaryClassifierEvaluator.cs

@@ -1506,7 +1506,7 @@ private protected override IEnumerable<string> GetPerInstanceColumnsToSave(RoleM
         }
     }
 
-    public static partial class Evaluate
+    internal static partial class Evaluate
     {
         [TlcModule.EntryPoint(Name = "Models.BinaryClassificationEvaluator", Desc = "Evaluates a binary classification scored dataset.")]
         public static CommonOutputs.ClassificationEvaluateOutput Binary(IHostEnvironment env, BinaryClassifierMamlEvaluator.Arguments input)

src/Microsoft.ML.Data/Evaluators/ClusteringEvaluator.cs

@@ -851,7 +851,7 @@ private protected override IDataView GetPerInstanceMetricsCore(IDataView perInst
         }
     }
 
-    public static partial class Evaluate
+    internal static partial class Evaluate
     {
         [TlcModule.EntryPoint(Name = "Models.ClusterEvaluator", Desc = "Evaluates a clustering scored dataset.")]
         public static CommonOutputs.CommonEvaluateOutput Clustering(IHostEnvironment env, ClusteringMamlEvaluator.Arguments input)

src/Microsoft.ML.Data/Evaluators/MultiClassClassifierEvaluator.cs

@@ -1029,7 +1029,7 @@ private protected override IDataView GetPerInstanceMetricsCore(IDataView perInst
         }
     }
 
-    public static partial class Evaluate
+    internal static partial class Evaluate
     {
         [TlcModule.EntryPoint(Name = "Models.ClassificationEvaluator", Desc = "Evaluates a multi class classification scored dataset.")]
         public static CommonOutputs.ClassificationEvaluateOutput MultiClass(IHostEnvironment env, MultiClassMamlEvaluator.Arguments input)

src/Microsoft.ML.Data/Evaluators/MultiOutputRegressionEvaluator.cs

@@ -767,7 +767,7 @@ private protected override void PrintFoldResultsCore(IChannel ch, Dictionary<str
         }
     }
 
-    public static partial class Evaluate
+    internal static partial class Evaluate
     {
         [TlcModule.EntryPoint(Name = "Models.MultiOutputRegressionEvaluator", Desc = "Evaluates a multi output regression scored dataset.")]
         public static CommonOutputs.CommonEvaluateOutput MultiOutputRegression(IHostEnvironment env, MultiOutputRegressionMamlEvaluator.Arguments input)

src/Microsoft.ML.Data/Evaluators/QuantileRegressionEvaluator.cs

@@ -552,7 +552,7 @@ private protected override IEnumerable<string> GetPerInstanceColumnsToSave(RoleM
         }
     }
 
-    public static partial class Evaluate
+    internal static partial class Evaluate
     {
         [TlcModule.EntryPoint(Name = "Models.QuantileRegressionEvaluator", Desc = "Evaluates a quantile regression scored dataset.")]
         public static CommonOutputs.CommonEvaluateOutput QuantileRegression(IHostEnvironment env, QuantileRegressionMamlEvaluator.Arguments input)

src/Microsoft.ML.Data/Evaluators/RankerEvaluator.cs

@@ -1049,7 +1049,7 @@ private static Comparison<int> GetCompareItems(List<short> queryLabels, List<Sin
         }
     }
 
-    public static partial class Evaluate
+    internal static partial class Evaluate
     {
         [TlcModule.EntryPoint(Name = "Models.RankerEvaluator", Desc = "Evaluates a ranking scored dataset.")]
         public static CommonOutputs.CommonEvaluateOutput Ranking(IHostEnvironment env, RankerMamlEvaluator.Arguments input)

src/Microsoft.ML.Data/Evaluators/RegressionEvaluator.cs

@@ -371,7 +371,7 @@ private protected override IEnumerable<string> GetPerInstanceColumnsToSave(RoleM
         }
     }
 
-    public static partial class Evaluate
+    internal static partial class Evaluate
     {
         [TlcModule.EntryPoint(Name = "Models.RegressionEvaluator", Desc = "Evaluates a regression scored dataset.")]
         public static CommonOutputs.CommonEvaluateOutput Regression(IHostEnvironment env, RegressionMamlEvaluator.Arguments input)

src/Microsoft.ML.Data/Model/Onnx/OnnxNode.cs

@@ -13,7 +13,8 @@ namespace Microsoft.ML.Model.Onnx
     /// That method creates a with inputs and outputs, but this object can modify the node further
     /// by adding attributes (in ONNX parlance, attributes are more or less constant parameterizations).
     /// </summary>
-    public abstract class OnnxNode
+    [BestFriend]
+    internal abstract class OnnxNode
     {
         public abstract void AddAttribute(string argName, double value);
         public abstract void AddAttribute(string argName, long value);

src/Microsoft.ML.Data/Model/Repository.cs

@@ -24,9 +24,11 @@ public interface ICanSaveModel
     }
 
     /// <summary>
-    /// For saving to a single stream.
+    /// For saving to a single stream. Note that this interface is mostly deprecated in favor of
+    /// saving more comprehensive and composable "model" objects, via <see cref="ICanSaveModel"/>.
     /// </summary>
-    public interface ICanSaveInBinaryFormat
+    [BestFriend]
+    internal interface ICanSaveInBinaryFormat
     {
         void SaveAsBinary(BinaryWriter writer);
     }

src/Microsoft.ML.Data/Prediction/Calibrator.cs

@@ -1070,7 +1070,7 @@ private static NaiveCalibrator Create(IHostEnvironment env, ModelLoadContext ctx
             return new NaiveCalibrator(env, ctx);
         }
 
-        public void SaveAsBinary(BinaryWriter writer)
+        void ICanSaveInBinaryFormat.SaveAsBinary(BinaryWriter writer)
         {
             ModelSaveContext.Save(writer, SaveCore);
         }
@@ -1717,7 +1717,7 @@ private static PavCalibrator Create(IHostEnvironment env, ModelLoadContext ctx)
             return new PavCalibrator(env, ctx);
         }
 
-        public void SaveAsBinary(BinaryWriter writer)
+        void ICanSaveInBinaryFormat.SaveAsBinary(BinaryWriter writer)
         {
             ModelSaveContext.Save(writer, SaveCore);
         }