merge

Author: ganik

README.md

@@ -97,6 +97,9 @@ Console.WriteLine("prediction: " + prediction.Prediction);
 ```
 A cookbook that shows how to use these APIs for a variety of existing and new scenarios can be found [here](docs/code/MlNetCookBook.md).
 
+## API Documentation
+
+See the [ML.NET API Reference Documentation](https://docs.microsoft.com/en-us/dotnet/api/?view=ml-dotnet).
 
 ## Samples
 

build/vsts-ci.yml

@@ -234,13 +234,12 @@ phases:
       nuGetFeedType: internal 
       feedPublish: MachineLearning
 
-# Temporarily disable myget publishing - https://github.com/dotnet/machinelearning/issues/2244
-#  - task: MSBuild@1
-#    displayName: Publish Packages to MyGet Feed
-#    inputs:
-#      solution: build/publish.proj
-#      msbuildArguments: /t:PublishPackages /p:NuGetFeedUrl=$(_NuGetFeedUrl) /p:NuGetApiKey=$(dotnet-myget-org-api-key)
-#      msbuildVersion: 15.0
+  - task: MSBuild@1
+    displayName: Publish Packages to MyGet Feed
+    inputs:
+      solution: build/publish.proj
+      msbuildArguments: /t:PublishPackages /p:NuGetFeedUrl=$(_NuGetFeedUrl) /p:NuGetApiKey=$(dotnet-myget-org-api-key)
+      msbuildVersion: 15.0
 
   - task: MSBuild@1
     displayName: Publish Symbols to SymWeb Symbol Server

docs/samples/Microsoft.ML.Samples/Dynamic/FeatureSelectionTransform.cs

@@ -48,14 +48,14 @@ public static void FeatureSelectionTransform()
             // In this example we define a CountFeatureSelectingEstimator, that selects slots in a feature vector that have more non-default 
             // values than the specified count. This transformation can be used to remove slots with too many missing values.
             var countSelectEst = ml.Transforms.FeatureSelection.SelectFeaturesBasedOnCount(
-                inputColumn: "Features", outputColumn: "FeaturesCountSelect", count: 695);
+                outputColumnName: "FeaturesCountSelect", inputColumnName: "Features", count: 695);
 
             // We also define a MutualInformationFeatureSelectingEstimator that selects the top k slots in a feature 
             // vector based on highest mutual information between that slot and a specified label. Notice that it is possible to 
             // specify the parameter `numBins', which controls the number of bins used in the approximation of the mutual information
             // between features and label.
             var mutualInfoEst = ml.Transforms.FeatureSelection.SelectFeaturesBasedOnMutualInformation(
-                inputColumn: "FeaturesCountSelect", outputColumn: "FeaturesMISelect", labelColumn: "Label", slotsInOutput: 5);
+                outputColumnName: "FeaturesMISelect", inputColumnName: "FeaturesCountSelect", labelColumn: "Label", slotsInOutput: 5);
 
             // Now, we can put the previous two transformations together in a pipeline.
             var pipeline = countSelectEst.Append(mutualInfoEst);

docs/samples/Microsoft.ML.Samples/Dynamic/KeyToValue_Term.cs

@@ -32,15 +32,15 @@ public static void KeyToValue_Term()
             string defaultColumnName = "DefaultKeys";
             // REVIEW create through the catalog extension
             var default_pipeline = new WordTokenizingEstimator(ml, "Review")
-                .Append(new ValueToKeyMappingEstimator(ml, "Review", defaultColumnName));
+                .Append(new ValueToKeyMappingEstimator(ml, defaultColumnName, "Review"));
 
             // Another pipeline, that customizes the advanced settings of the TermEstimator.
             // We can change the maxNumTerm to limit how many keys will get generated out of the set of words, 
             // and condition the order in which they get evaluated by changing sort from the default Occurence (order in which they get encountered) 
             // to value/alphabetically.
             string customizedColumnName = "CustomizedKeys";
             var customized_pipeline = new WordTokenizingEstimator(ml, "Review")
-                .Append(new ValueToKeyMappingEstimator(ml, "Review", customizedColumnName, maxNumTerms: 10, sort: ValueToKeyMappingTransformer.SortOrder.Value));
+                .Append(new ValueToKeyMappingEstimator(ml,customizedColumnName,  "Review", maxNumTerms: 10, sort: ValueToKeyMappingTransformer.SortOrder.Value));
 
             // The transformed data.
             var transformedData_default = default_pipeline.Fit(trainData).Transform(trainData);

docs/samples/Microsoft.ML.Samples/Dynamic/NgramExtraction.cs

@@ -26,9 +26,9 @@ public static void NgramTransform()
             // A pipeline to tokenize text as characters and then combine them together into ngrams
             // The pipeline uses the default settings to featurize.
 
-            var charsPipeline = ml.Transforms.Text.TokenizeCharacters("SentimentText", "Chars", useMarkerCharacters:false);
-            var ngramOnePipeline = ml.Transforms.Text.ProduceNgrams("Chars", "CharsUnigrams", ngramLength:1);
-            var ngramTwpPipeline = ml.Transforms.Text.ProduceNgrams("Chars", "CharsTwograms");
+            var charsPipeline = ml.Transforms.Text.TokenizeCharacters("Chars", "SentimentText", useMarkerCharacters:false);
+            var ngramOnePipeline = ml.Transforms.Text.ProduceNgrams("CharsUnigrams", "Chars", ngramLength:1);
+            var ngramTwpPipeline = ml.Transforms.Text.ProduceNgrams("CharsTwograms", "Chars");
             var oneCharsPipeline = charsPipeline.Append(ngramOnePipeline);
             var twoCharsPipeline = charsPipeline.Append(ngramTwpPipeline);
 

docs/samples/Microsoft.ML.Samples/Dynamic/Normalizer.cs

@@ -33,7 +33,7 @@ public static void Normalizer()
             var transformer = pipeline.Fit(trainData);
 
             var modelParams = transformer.Columns
-                                         .First(x => x.Output == "Induced")
+                                         .First(x => x.Name == "Induced")
                                          .ModelParameters as NormalizingTransformer.AffineNormalizerModelParameters<float>;
 
             Console.WriteLine($"The normalization parameters are: Scale = {modelParams.Scale} and Offset = {modelParams.Offset}");
@@ -66,7 +66,7 @@ public static void Normalizer()
 
             // Composing a different pipeline if we wanted to normalize more than one column at a time. 
             // Using log scale as the normalization mode. 
-            var multiColPipeline = ml.Transforms.Normalize(NormalizingEstimator.NormalizerMode.LogMeanVariance, new[] { ("Induced", "LogInduced"), ("Spontaneous", "LogSpontaneous") });
+            var multiColPipeline = ml.Transforms.Normalize(NormalizingEstimator.NormalizerMode.LogMeanVariance, new[] { ("LogInduced", "Induced"), ("LogSpontaneous", "Spontaneous") });
             // The transformed data.
             var multiColtransformer = multiColPipeline.Fit(trainData);
             var multiColtransformedData = multiColtransformer.Transform(trainData);
@@ -97,7 +97,7 @@ public static void Normalizer()
 
             // Inspect the weights of normalizing the columns
             var multiColModelParams = multiColtransformer.Columns
-                .First(x=> x.Output == "LogInduced")
+                .First(x=> x.Name == "LogInduced")
                 .ModelParameters as NormalizingTransformer.CdfNormalizerModelParameters<float>;
 
             Console.WriteLine($"The normalization parameters are: Mean = {multiColModelParams.Mean} and Stddev = {multiColModelParams.Stddev}");

docs/samples/Microsoft.ML.Samples/Dynamic/OnnxTransform.cs

@@ -35,7 +35,7 @@ public static void OnnxTransformSample()
             var mlContext = new MLContext();
             var data = GetTensorData();
             var idv = mlContext.Data.ReadFromEnumerable(data);
-            var pipeline = new OnnxScoringEstimator(mlContext, modelPath, new[] { inputInfo.Key }, new[] { outputInfo.Key });
+            var pipeline = new OnnxScoringEstimator(mlContext, new[] { outputInfo.Key }, new[] { inputInfo.Key }, modelPath);
 
             // Run the pipeline and get the transformed values
             var transformedValues = pipeline.Fit(idv).Transform(idv);

docs/samples/Microsoft.ML.Samples/Dynamic/TensorFlowTransform.cs

@@ -22,8 +22,8 @@ public static void TensorFlowScoringSample()
             // Create a ML pipeline.
             var pipeline = mlContext.Transforms.ScoreTensorFlowModel(
                 modelLocation, 
-                new[] { nameof(TensorData.input) }, 
-                new[] { nameof(OutputScores.output) });
+                new[] { nameof(OutputScores.output) },
+                new[] { nameof(TensorData.input) });
 
             // Run the pipeline and get the transformed values.
             var estimator = pipeline.Fit(idv);

docs/samples/Microsoft.ML.Samples/Dynamic/TextTransform.cs

@@ -27,11 +27,11 @@ public static void TextTransform()
             // A pipeline for featurization of the "SentimentText" column, and placing the output in a new column named "DefaultTextFeatures"
             // The pipeline uses the default settings to featurize.
             string defaultColumnName = "DefaultTextFeatures";
-            var default_pipeline = ml.Transforms.Text.FeaturizeText("SentimentText", defaultColumnName);
+            var default_pipeline = ml.Transforms.Text.FeaturizeText(defaultColumnName , "SentimentText");
 
             // Another pipeline, that customizes the advanced settings of the FeaturizeText transformer.
             string customizedColumnName = "CustomizedTextFeatures";
-            var customized_pipeline = ml.Transforms.Text.FeaturizeText("SentimentText", customizedColumnName, s =>
+            var customized_pipeline = ml.Transforms.Text.FeaturizeText(customizedColumnName, "SentimentText", s =>
             {
                 s.KeepPunctuations = false;
                 s.KeepNumbers = false;

pkg/Directory.Build.props

@@ -37,8 +37,8 @@
   </ItemGroup>
 
   <ItemGroup Condition="'$(IncludeMLNetNotices)' != 'false'">
-    <Content Include="$(RepoRoot)\THIRD-PARTY-NOTICES.TXT" Pack="true" PackagePath="\" />
-    <Content Include="$(RepoRoot)\LICENSE" Pack="true" PackagePath="\" />
+    <Content Include="$(RepoRoot)\THIRD-PARTY-NOTICES.TXT" Pack="true" PackagePath="" />
+    <Content Include="$(RepoRoot)\LICENSE" Pack="true" PackagePath=""/>
   </ItemGroup>
 
   <ItemGroup Condition="'$(IsSymbolsPackage)' != 'true'">

pkg/Microsoft.ML.Mkl.Redist/Microsoft.ML.Mkl.Redist.nupkgproj

@@ -3,7 +3,7 @@
   <PropertyGroup>
     <Authors>Intel</Authors>
     <TargetFramework>netstandard2.0</TargetFramework>
-    <PackageLicenseFile>NOTICE.txt</PackageLicenseFile>
+    <PackageLicenseFile>LICENSE.txt</PackageLicenseFile>
     <PackageDescription>$(MSBuildProjectName) contains the MKL library redistributed as a NuGet package.</PackageDescription>
     <PackageTags>$(PackageTags) MLNET MKL</PackageTags>
   </PropertyGroup>
@@ -14,6 +14,6 @@
 
   <ItemGroup>
     <Content Include="..\common\CommonPackage.props" Pack="true" PackagePath="build\netstandard2.0\$(MSBuildProjectName).props" />
-    <Content Include="$(PackagesDir)\mlnetmkldeps\$(MlNetMklDepsPackageVersion)\LICENSE.txt" Pack="true" PackagePath=".\NOTICE.txt" /> 
+    <Content Include="$(PackagesDir)\mlnetmkldeps\$(MlNetMklDepsPackageVersion)\LICENSE.txt" Pack="true" PackagePath="" /> 
   </ItemGroup>
 </Project>

pkg/Microsoft.ML.Recommender/Microsoft.ML.Recommender.nupkgproj

@@ -8,7 +8,7 @@
   <ItemGroup> 
     <ProjectReference Include="../Microsoft.ML/Microsoft.ML.nupkgproj" />
     <Content Include="..\common\CommonPackage.props" Pack="true" PackagePath="build\netstandard2.0\$(MSBuildProjectName).props" />
-    <Content Include="$(SourceDir)Native\MatrixFactorizationNative\libmf\COPYRIGHT" Pack="true" PackagePath=".\" />
+    <Content Include="$(SourceDir)Native\MatrixFactorizationNative\libmf\COPYRIGHT" Pack="true" PackagePath="" />
   </ItemGroup> 
 
 </Project>

pkg/Microsoft.ML.TensorFlow.Redist/Microsoft.ML.TensorFlow.Redist.nupkgproj

@@ -19,7 +19,7 @@
 
   <ItemGroup>
     <Content Include="..\common\CommonPackage.props" Pack="true" PackagePath="build\netstandard2.0\$(MSBuildProjectName).props" />
-    <Content Include="$(PackageAssetsPath)$(PackageIdFolderName)\LICENSE.txt" Pack="true" PackagePath=".\" />
-    <Content Condition="Exists('$(PackageAssetsPath)$(PackageIdFolderName)\THIRD_PARTY_NOTICES.txt')" Include="$(PackageAssetsPath)$(PackageIdFolderName)\THIRD_PARTY_NOTICES.txt" Pack="true" PackagePath=".\" />
+    <Content Include="$(PackageAssetsPath)$(PackageIdFolderName)\LICENSE.txt" Pack="true" PackagePath="" />
+    <Content Condition="Exists('$(PackageAssetsPath)$(PackageIdFolderName)\THIRD_PARTY_NOTICES.txt')" Include="$(PackageAssetsPath)$(PackageIdFolderName)\THIRD_PARTY_NOTICES.txt" Pack="true" PackagePath="" />
   </ItemGroup>
 </Project>

src/Microsoft.ML.Core/CommandLine/CmdParser.cs

@@ -1359,17 +1359,17 @@ public ArgumentInfo(Type type, Argument argDef, Argument[] args, Dictionary<stri
             private static MethodInfo GetParseMethod(Type type)
             {
                 Contracts.AssertValue(type);
-                var meth = type.GetMethod("Parse", new[] { typeof(string) });
-                if (meth != null && meth.IsStatic && meth.IsPublic && meth.ReturnType == type)
+                var meth = type.GetMethod("Parse", BindingFlags.Static | BindingFlags.NonPublic | BindingFlags.Public, binder: null, new[] { typeof(string) }, null);
+                if (meth != null && meth.IsStatic && !meth.IsPrivate && meth.ReturnType == type)
                     return meth;
                 return null;
             }
 
             private static MethodInfo GetUnparseMethod(Type type)
             {
                 Contracts.AssertValue(type);
-                var meth = type.GetMethod("TryUnparse", new[] { typeof(StringBuilder) });
-                if (meth != null && !meth.IsStatic && meth.IsPublic && meth.ReturnType == typeof(bool))
+                var meth = type.GetMethod("TryUnparse", BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public, binder: null, new[] { typeof(StringBuilder) }, null);
+                if (meth != null && !meth.IsPrivate && meth.ReturnType == typeof(bool))
                     return meth;
                 return null;
             }

src/Microsoft.ML.Core/ComponentModel/ComponentCatalog.cs

@@ -468,7 +468,7 @@ private void ScanForEntryPoints(LoadableClassInfo info)
             var type = info.LoaderType;
 
             // Scan for entry points.
-            foreach (var methodInfo in type.GetMethods(BindingFlags.Static | BindingFlags.Public))
+            foreach (var methodInfo in type.GetMethods(BindingFlags.Static | BindingFlags.Public | BindingFlags.NonPublic))
             {
                 var attr = methodInfo.GetCustomAttributes(typeof(TlcModule.EntryPointAttribute), false).FirstOrDefault() as TlcModule.EntryPointAttribute;
                 if (attr == null)
@@ -727,7 +727,7 @@ internal LoadableClassInfo GetLoadableClassInfo(string loadName, Type signatureT
         [BestFriend]
         internal IEnumerable<EntryPointInfo> AllEntryPoints()
         {
-            return _entryPoints.AsEnumerable();
+            return _entryPoints;
         }
 
         [BestFriend]

src/Microsoft.ML.Core/Data/IEstimator.cs

@@ -75,29 +75,29 @@ internal Column(string name, VectorKind vecKind, ColumnType itemType, bool isKey
             }
 
             /// <summary>
-            /// Returns whether <paramref name="inputColumn"/> is a valid input, if this object represents a
+            /// Returns whether <paramref name="source"/> is a valid input, if this object represents a
             /// requirement.
             ///
             /// Namely, it returns true iff:
             ///  - The <see cref="Name"/>, <see cref="Kind"/>, <see cref="ItemType"/>, <see cref="IsKey"/> fields match.
-            ///  - The columns of <see cref="Metadata"/> of <paramref name="inputColumn"/> is a superset of our <see cref="Metadata"/> columns.
+            ///  - The columns of <see cref="Metadata"/> of <paramref name="source"/> is a superset of our <see cref="Metadata"/> columns.
             ///  - Each such metadata column is itself compatible with the input metadata column.
             /// </summary>
             [BestFriend]
-            internal bool IsCompatibleWith(Column inputColumn)
+            internal bool IsCompatibleWith(Column source)
             {
-                Contracts.Check(inputColumn.IsValid, nameof(inputColumn));
-                if (Name != inputColumn.Name)
+                Contracts.Check(source.IsValid, nameof(source));
+                if (Name != source.Name)
                     return false;
-                if (Kind != inputColumn.Kind)
+                if (Kind != source.Kind)
                     return false;
-                if (!ItemType.Equals(inputColumn.ItemType))
+                if (!ItemType.Equals(source.ItemType))
                     return false;
-                if (IsKey != inputColumn.IsKey)
+                if (IsKey != source.IsKey)
                     return false;
                 foreach (var metaCol in Metadata)
                 {
-                    if (!inputColumn.Metadata.TryFindColumn(metaCol.Name, out var inputMetaCol))
+                    if (!source.Metadata.TryFindColumn(metaCol.Name, out var inputMetaCol))
                         return false;
                     if (!metaCol.IsCompatibleWith(inputMetaCol))
                         return false;

src/Microsoft.ML.Core/Data/MetadataUtils.cs

@@ -327,6 +327,9 @@ internal static bool HasSlotNames(this Schema.Column column, int vectorSize)
         public static void GetSlotNames(this Schema.Column column, ref VBuffer<ReadOnlyMemory<char>> slotNames)
             => column.Metadata.GetValue(Kinds.SlotNames, ref slotNames);
 
+        public static void GetKeyValues<TValue>(this Schema.Column column, ref VBuffer<TValue> keyValues)
+            => column.Metadata.GetValue(Kinds.KeyValues, ref keyValues);
+
         [BestFriend]
         internal static void GetSlotNames(RoleMappedSchema schema, RoleMappedSchema.ColumnRole role, int vectorSize, ref VBuffer<ReadOnlyMemory<char>> slotNames)
         {