Merge branch 'master' into Ivanidze/KeyToVectorIORBug

Author: Ivan Matantsev

.vsts-dotnet-ci.yml

@@ -31,6 +31,15 @@ phases:
   parameters:
     name: Centos
     buildScript: ./build.sh
+    customMatrixes:
+      Build_Debug_Intrinsics:
+        _configuration: Debug-Intrinsics
+        _config_short: DI
+        _includeBenchmarkData: false
+      Build_Release:
+        _configuration: Release
+        _config_short: R
+        _includeBenchmarkData: true
     queue:
       name: Hosted Ubuntu 1604
       container: CentosContainer
@@ -39,6 +48,15 @@ phases:
   parameters:
     name: Ubuntu
     buildScript: ./build.sh
+    customMatrixes:
+      Build_Debug:
+        _configuration: Debug
+        _config_short: D
+        _includeBenchmarkData: false
+      Build_Release_Intrinsics:
+        _configuration: Release-Intrinsics
+        _config_short: RI
+        _includeBenchmarkData: true
     queue:
       name: Hosted Ubuntu 1604
       container: UbuntuContainer

BuildToolsVersion.txt

@@ -1 +1 @@
-3.0.0-preview1-03129-01
+3.0.0-preview1-03801-01

Microsoft.ML.sln

@@ -100,7 +100,7 @@ Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Microsoft.ML.FSharp.Tests",
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.ImageAnalytics", "src\Microsoft.ML.ImageAnalytics\Microsoft.ML.ImageAnalytics.csproj", "{00E38F77-1E61-4CDF-8F97-1417D4E85053}"
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.HalLearners", "src\Microsoft.ML.HalLearners\Microsoft.ML.HalLearners.csproj", "{A7222F41-1CF0-47D9-B80C-B4D77B027A61}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.Mkl.Components", "src\Microsoft.ML.Mkl.Components\Microsoft.ML.Mkl.Components.csproj", "{A7222F41-1CF0-47D9-B80C-B4D77B027A61}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.TensorFlow", "src\Microsoft.ML.TensorFlow\Microsoft.ML.TensorFlow.csproj", "{570A0B8A-5463-44D2-8521-54C0CA4CACA9}"
 EndProject
@@ -142,18 +142,18 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.StaticPipe", "
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.TensorFlow.StaticPipe", "src\Microsoft.ML.TensorFlow.StaticPipe\Microsoft.ML.TensorFlow.StaticPipe.csproj", "{F95F7AFB-03AF-4D20-BD75-1740B5FF71D3}"
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.HalLearners.StaticPipe", "src\Microsoft.ML.HalLearners.StaticPipe\Microsoft.ML.HalLearners.StaticPipe.csproj", "{2F25EF6A-C754-45BE-AD9E-7DDF46A1B51A}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.Mkl.Components.StaticPipe", "src\Microsoft.ML.Mkl.Components.StaticPipe\Microsoft.ML.Mkl.Components.StaticPipe.csproj", "{2F25EF6A-C754-45BE-AD9E-7DDF46A1B51A}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.OnnxTransformer.StaticPipe", "src\Microsoft.ML.OnnxTransformer.StaticPipe\Microsoft.ML.OnnxTransformer.StaticPipe.csproj", "{D1324668-9568-40F4-AA55-30A9A516C230}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.LightGBM.StaticPipe", "src\Microsoft.ML.LightGBM.StaticPipe\Microsoft.ML.LightGBM.StaticPipe.csproj", "{22C51B08-ACAE-47B2-A312-462DC239A23B}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.TimeSeries.StaticPipe", "src\Microsoft.ML.TimeSeries.StaticPipe\Microsoft.ML.TimeSeries.StaticPipe.csproj", "{06A147ED-15EA-4106-9105-9B745125B470}"
 EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Microsoft.ML.HalLearners", "Microsoft.ML.HalLearners", "{63006A14-B924-48C5-83C9-CFE9DA22B01F}"
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Microsoft.ML.Mkl.Components", "Microsoft.ML.Mkl.Components", "{63006A14-B924-48C5-83C9-CFE9DA22B01F}"
 	ProjectSection(SolutionItems) = preProject
-		pkg\Microsoft.ML.HalLearners\Microsoft.ML.HalLearners.nupkgproj = pkg\Microsoft.ML.HalLearners\Microsoft.ML.HalLearners.nupkgproj
-		pkg\Microsoft.ML.HalLearners\Microsoft.ML.HalLearners.symbols.nupkgproj = pkg\Microsoft.ML.HalLearners\Microsoft.ML.HalLearners.symbols.nupkgproj
+		pkg\Microsoft.ML.Mkl.Components\Microsoft.ML.Mkl.Components.nupkgproj = pkg\Microsoft.ML.Mkl.Components\Microsoft.ML.Mkl.Components.nupkgproj
+		pkg\Microsoft.ML.Mkl.Components\Microsoft.ML.Mkl.Components.symbols.nupkgproj = pkg\Microsoft.ML.Mkl.Components\Microsoft.ML.Mkl.Components.symbols.nupkgproj
 	EndProjectSection
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Microsoft.ML.ImageAnalytics", "Microsoft.ML.ImageAnalytics", "{1229F799-37F0-4282-B9F0-74BFA97CC362}"

README.md

@@ -18,7 +18,7 @@ Along with these ML capabilities, this first release of ML.NET also brings the f
 
 ML.NET runs on Windows, Linux, and macOS using [.NET Core](https://github.com/dotnet/core), or Windows using .NET Framework. 64 bit is supported on all platforms. 32 bit is supported on Windows, except for TensorFlow, LightGBM, and ONNX related functionality.
 
-The current release is 0.10. Check out the [release notes](docs/release-notes/0.10/release-0.10.md) to see what's new.
+The current release is 0.11. Check out the [release notes](docs/release-notes/0.11/release-0.11.md) to see what's new.
 
 First, ensure you have installed [.NET Core 2.1](https://www.microsoft.com/net/learn/get-started) or later. ML.NET also works on the .NET Framework 4.6.1 or later, but 4.7.2 or later is recommended.
 
@@ -46,7 +46,7 @@ To build ML.NET from source please visit our [developers guide](docs/project-doc
 
 |    | Debug | Release |
 |:---|----------------:|------------------:|
-|**Linux**|[![x64-debug](https://dnceng.visualstudio.com/public/_apis/build/status/dotnet/machinelearning/MachineLearning-CI?branchName=master&jobname=Linux&configuration=Build_Debug)](https://dnceng.visualstudio.com/DotNet-Public/_build/latest?definitionId=104&branch=master)|[![x64-release](https://dnceng.visualstudio.com/public/_apis/build/status/dotnet/machinelearning/MachineLearning-CI?branchName=master&jobname=Linux&configuration=Build_Release)](https://dnceng.visualstudio.com/DotNet-Public/_build/latest?definitionId=104&branch=master)|
+|**CentOS**|[![x64-debug](https://dnceng.visualstudio.com/public/_apis/build/status/dotnet/machinelearning/MachineLearning-CI?branchName=master&jobname=CentOS&configuration=Build_Debug_Intrinsics)](https://dnceng.visualstudio.com/DotNet-Public/_build/latest?definitionId=104&branch=master)|[![x64-release](https://dnceng.visualstudio.com/public/_apis/build/status/dotnet/machinelearning/MachineLearning-CI?branchName=master&jobname=CentOS&configuration=Build_Release)](https://dnceng.visualstudio.com/DotNet-Public/_build/latest?definitionId=104&branch=master)|
 |**macOS**|[![x64-debug](https://dnceng.visualstudio.com/public/_apis/build/status/dotnet/machinelearning/MachineLearning-CI?branchName=master&jobname=macOS&configuration=Build_Debug)](https://dnceng.visualstudio.com/DotNet-Public/_build/latest?definitionId=104&branch=master)|[![x64-release](https://dnceng.visualstudio.com/public/_apis/build/status/dotnet/machinelearning/MachineLearning-CI?branchName=master&jobname=macOS&configuration=Build_Release)](https://dnceng.visualstudio.com/DotNet-Public/_build/latest?definitionId=104&branch=master)|
 |**Windows x64**|[![x64-debug](https://dnceng.visualstudio.com/public/_apis/build/status/dotnet/machinelearning/MachineLearning-CI?branchName=master&jobname=Windows_x64&configuration=Build_Debug)](https://dnceng.visualstudio.com/DotNet-Public/_build/latest?definitionId=104&branch=master)|[![x64-release](https://dnceng.visualstudio.com/public/_apis/build/status/dotnet/machinelearning/MachineLearning-CI?branchName=master&jobname=Windows_x64&configuration=Build_Release)](https://dnceng.visualstudio.com/DotNet-Public/_build/latest?definitionId=104&branch=master)|
 |**Windows x86**|[![Build Status](https://dnceng.visualstudio.com/public/_apis/build/status/dotnet/machinelearning/MachineLearning-CI?branchName=master&jobName=Windows_x86&configuration=Build_Debug)](https://dnceng.visualstudio.com/public/_build/latest?definitionId=104?branchName=master)|[![Build Status](https://dnceng.visualstudio.com/public/_apis/build/status/dotnet/machinelearning/MachineLearning-CI?branchName=master&jobName=Windows_x86&configuration=Build_Release)](https://dnceng.visualstudio.com/public/_build/latest?definitionId=104?branchName=master)|
@@ -69,16 +69,14 @@ Here's an example of code to train a model to predict sentiment from text sample
 
 ```C#
 var mlContext = new MLContext();
-var reader = mlContext.Data.CreateTextLoader(new TextLoader.Arguments
-        {
-        Column = new[] {
-            new TextLoader.Column("SentimentText", DataKind.Text, 1),
-            new TextLoader.Column("Label", DataKind.Bool, 0),
-        },
-        HasHeader = true,
-        Separator = ","
-});
-var data = reader.Read(dataPath);
+var loader = mlContext.Data.CreateTextLoader(new[]
+    {
+        new TextLoader.Column("SentimentText", DataKind.String, 1),
+        new TextLoader.Column("Label", DataKind.Boolean, 0),
+    },
+    hasHeader: true,
+    separatorChar: ',');
+var data = loader.Load(dataPath);
 var learningPipeline = mlContext.Transforms.Text.FeaturizeText("Features", "SentimentText")
         .Append(mlContext.BinaryClassification.Trainers.FastTree());
 var model = learningPipeline.Fit(data);
@@ -116,4 +114,3 @@ ML.NET is a [.NET Foundation](https://www.dotnetfoundation.org/projects) project
 There are many .NET related projects on GitHub.
 
 - [.NET home repo](https://github.com/Microsoft/dotnet) - links to 100s of .NET projects, from Microsoft and the community.
-

docs/release-notes/0.11/release-0.11.md

@@ -0,0 +1,40 @@
+# ML.NET 0.11 Release Notes
+
+[ML.NET](https://aka.ms/mlnet) 0.11 will be the last preview release before we reach `Release Candidate` for v1. We continue to push for creating a coherent and clean API surface for [ML.NET](https://aka.ms/mlnet) users. This release includes several bug fixes as well as extensive work on reducing the public API surface. The work on the API is tracked via [this project](https://github.com/dotnet/machinelearning/projects/13). In the upcoming 0.12 (RC1) releases before we reach 1.0, we will continue on refining the API and improving documentation.
+
+### Installation
+
+ML.NET supports Windows, MacOS, and Linux. See [supported OS versions of .NET Core 2.0](https://github.com/dotnet/core/blob/master/release-notes/2.0/2.0-supported-os.md) for more details.
+
+You can install ML.NET NuGet from the CLI using:
+```
+dotnet add package Microsoft.ML
+```
+
+From package manager:
+```
+Install-Package Microsoft.ML
+```
+
+### Release Notes
+
+Below are a few of the highlights from this release. There are many other improvements in the API.
+
+* Creation of components through MLContext: advanced options and other feedback. ([#1798](https://github.coalsom/dotnet/machinelearning/issues/1798))
+* Several issues closed on internalizing public surface. ([19 issues](https://github.com/dotnet/machinelearning/issues?q=is%3Aissue+lockdown+is%3Aclosed))
+* Stop using MEF as part of the public API. ([#2422](https://github.com/dotnet/machinelearning/issues/2422))
+* Several `NameSpace` changes. ([#2326](https://github.com/dotnet/machinelearning/issues/2326))
+* `ONNX` is now `ONNXConverter`. ([#2625](https://github.com/dotnet/machinelearning/pull/2625))
+* `ONNXTransform` is now `ONNXTransformer`. ([#2544](https://github.com/dotnet/machinelearning/pull/2544))
+* `FastTree` has it's own package now. ([#2752](https://github.com/dotnet/machinelearning/issues/2752))
+* `Ensemble` has been moved out of `Microsoft.ML` package. ([#2717](https://github.com/dotnet/machinelearning/issues/2717))
+* Add support for string types in TensorFlowTransformer. ([#2545](https://github.com/dotnet/machinelearning/issues/2545))
+* Make FastTree/LightGBM learned model suitable for public consumption. ([#1960](https://github.com/dotnet/machinelearning/issues/1960))
+
+
+### Acknowledgements
+
+Shoutout to [PaulTFreedman](https://github.com/PaulTFreedman),
+[kant2002](https://github.com/kant2002),
+[jwood803](https://github.com/jwood803), [mareklinka](https://github.com/mareklinka), [elbruno](https://github.com/elbruno) and the [ML.NET](https://aka.ms/mlnet) team for their
+contributions as part of this release!

docs/samples/Microsoft.ML.Samples/Dynamic/Calibrator.cs

@@ -30,22 +30,12 @@ public static void Example()
             // We will train a FastTreeRegression model with 1 tree on these two columns to predict Age.
             string outputColumnName = "Features";
             var pipeline = ml.Transforms.Concatenate(outputColumnName, new[] { "Parity", "Induced" })
-                .Append(ml.Regression.Trainers.FastTree(labelColumnName: "Age", featureColumnName: outputColumnName, numTrees: 1, numLeaves: 2, minDatapointsInLeaves: 1));
+                .Append(ml.Regression.Trainers.FastTree(labelColumnName: "Age", featureColumnName: outputColumnName, numberOfTrees: 1, numberOfLeaves: 2, minimumExampleCountPerLeaf: 1));
 
             var model = pipeline.Fit(trainData);
 
             // Get the trained model parameters.
             var modelParams = model.LastTransformer.Model;
-
-            // Let's see where an example with Parity = 1 and Induced = 1 would end up in the single trained tree.
-            var testRow = new VBuffer<float>(2, new[] { 1.0f, 1.0f });
-            // Use the path object to pass to GetLeaf, which will populate path with the IDs of th nodes from root to leaf.
-            List<int> path = default;
-            // Get the ID of the leaf this example ends up in tree 0.
-            var leafID = modelParams.GetLeaf(0, in testRow, ref path);
-            // Get the leaf value for this leaf ID in tree 0.
-            var leafValue = modelParams.GetLeafValue(0, leafID);
-            Console.WriteLine("The leaf value in tree 0 is: " + leafValue);
         }
     }
 }

docs/samples/Microsoft.ML.Samples/Dynamic/FastTreeRegression.cs

@@ -44,8 +44,7 @@ public static void Example()
             var outData = featureContributionCalculator.Fit(scoredData).Transform(scoredData);
 
             // Let's extract the weights from the linear model to use as a comparison
-            var weights = new VBuffer<float>();
-            model.Model.GetFeatureWeights(ref weights);
+            var weights = model.Model.Weights;
 
             // Let's now walk through the first ten records and see which feature drove the values the most
             // Get prediction scores and contributions
@@ -63,7 +62,7 @@ public static void Example()
                 var value = row.Features[featureOfInterest];
                 var contribution = row.FeatureContributions[featureOfInterest];
                 var name = data.Schema[featureOfInterest + 1].Name;
-                var weight = weights.GetValues()[featureOfInterest];
+                var weight = weights[featureOfInterest];
 
                 Console.WriteLine("{0:0.00}\t{1:0.00}\t{2}\t{3:0.00}\t{4:0.00}\t{5:0.00}",
                     row.MedianHomeValue,