Upgrade to ML.NET 0.11 (dotnet#247)

* initial changes

* fix lightgbm

* changed normalize method

* added tests

* fix tests

* fix test

Author: srsaggam

src/Microsoft.ML.Auto/TrainerExtensions/TrainerExtensionUtil.cs

@@ -51,7 +51,7 @@ public static T CreateOptions<T>(IEnumerable<SweepableParam> sweepParams, string
         {
             var options = Activator.CreateInstance<T>();
             options.LabelColumn = labelColumn;
-            if(sweepParams != null)
+            if (sweepParams != null)
             {
                 UpdateFields(options, sweepParams);
             }
@@ -66,7 +66,7 @@ public static LightGBM.Options CreateLightGbmOptions(IEnumerable<SweepableParam>
             var options = new LightGBM.Options();
             options.LabelColumn = columnInfo.LabelColumn;
             options.WeightColumn = columnInfo.WeightColumn;
-            if(sweepParams != null)
+            if (sweepParams != null)
             {
                 var treeBoosterParams = sweepParams.Where(p => _lightGbmTreeBoosterParamNames.Contains(p.Name));
                 var parentArgParams = sweepParams.Except(treeBoosterParams);
@@ -76,7 +76,7 @@ public static LightGBM.Options CreateLightGbmOptions(IEnumerable<SweepableParam>
             return options;
         }
 
-        public static PipelineNode BuildOvaPipelineNode(ITrainerExtension multiExtension, ITrainerExtension binaryExtension, 
+        public static PipelineNode BuildOvaPipelineNode(ITrainerExtension multiExtension, ITrainerExtension binaryExtension,
             IEnumerable<SweepableParam> sweepParams, ColumnInformation columnInfo)
         {
             var ovaNode = binaryExtension.CreatePipelineNode(sweepParams, columnInfo);
@@ -89,7 +89,7 @@ public static PipelineNode BuildPipelineNode(TrainerName trainerName, IEnumerabl
         {
             var properties = BuildBasePipelineNodeProps(sweepParams, labelColumn, weightColumn);
 
-            if(additionalProperties != null)
+            if (additionalProperties != null)
             {
                 foreach (var property in additionalProperties)
                 {
@@ -131,7 +131,7 @@ private static IDictionary<string, object> BuildLightGbmPipelineNodeProps(IEnume
             string labelColumn, string weightColumn)
         {
             Dictionary<string, object> props = null;
-            if(sweepParams == null)
+            if (sweepParams == null)
             {
                 props = new Dictionary<string, object>();
             }
@@ -141,7 +141,7 @@ private static IDictionary<string, object> BuildLightGbmPipelineNodeProps(IEnume
                 var parentArgParams = sweepParams.Except(treeBoosterParams);
 
                 var treeBoosterProps = treeBoosterParams.ToDictionary(p => p.Name, p => (object)p.ProcessedValue());
-                var treeBoosterCustomProp = new CustomProperty("Options.TreeBooster.Arguments", treeBoosterProps);
+                var treeBoosterCustomProp = new CustomProperty("Options.TreeBooster.Options", treeBoosterProps);
 
                 props = parentArgParams.ToDictionary(p => p.Name, p => (object)p.ProcessedValue());
                 props[LightGbmTreeBoosterPropName] = treeBoosterCustomProp;
@@ -258,7 +258,7 @@ public static void UpdateFields(object obj, IEnumerable<SweepableParam> sweepPar
 
         public static TrainerName GetTrainerName(BinaryClassificationTrainer binaryTrainer)
         {
-            switch(binaryTrainer)
+            switch (binaryTrainer)
             {
                 case BinaryClassificationTrainer.AveragedPerceptron:
                     return TrainerName.AveragedPerceptronBinary;

src/Test/TrainerExtensionsTests.cs

@@ -23,7 +23,7 @@ public void TrainerExtensionInstanceTests()
                 var extension = TrainerExtensionCatalog.GetTrainerExtension(trainerName);
                 var sweepParams = extension.GetHyperparamSweepRanges();
                 Assert.IsNotNull(sweepParams);
-                foreach(var sweepParam in sweepParams)
+                foreach (var sweepParam in sweepParams)
                 {
                     sweepParam.RawValue = 1;
                 }
@@ -67,7 +67,7 @@ public void BuildLightGbmPipelineNode()
     ""CatSmooth"": 10,
     ""CatL2"": 0.5,
     ""Booster"": {
-      ""Name"": ""Options.TreeBooster.Arguments"",
+      ""Name"": ""Options.TreeBooster.Options"",
       ""Properties"": {
         ""RegLambda"": 0.5,
         ""RegAlpha"": 0.5

src/mlnet.Test/ApprovalTests/ConsoleCodeGeneratorTests.GeneratedProjectCodeTest.approved.txt

@@ -10,8 +10,8 @@
     <Compile Include="ConsoleHelper.cs" />
   </ItemGroup>
   <ItemGroup>
-    <PackageReference Include="Microsoft.ML" Version="0.10.0" />
-    <PackageReference Include="Microsoft.ML.HalLearners" Version="0.10.0" />
-    <PackageReference Include="Microsoft.ML.LightGBM" Version="0.10.0" /> 
+    <PackageReference Include="Microsoft.ML" Version="0.11.0-preview-27428-4" />
+    <PackageReference Include="Microsoft.ML.HalLearners" Version="0.11.0-preview-27428-4" />
+    <PackageReference Include="Microsoft.ML.LightGBM" Version="0.11.0-preview-27428-4" /> 
   </ItemGroup>
 </Project>

src/mlnet.Test/ApprovalTests/ConsoleCodeGeneratorTests.GeneratedTrainCodeTest.approved.txt

@@ -1,10 +1,13 @@
-// This is an auto generated file by ML.NET CLI
+//*****************************************************************************************
+//*                                                                                       *
+//* This is an auto-generated file by Microsoft ML.NET CLI (Command-Line Interface) tool. *
+//*                                                                                       *
+//*****************************************************************************************
 
 using System;
 using System.IO;
 using System.Linq;
 using Microsoft.ML;
-using Microsoft.ML.Core.Data;
 using Microsoft.ML.Data;
 using Microsoft.Data.DataView;
 using Microsoft.ML.LightGBM;
@@ -49,26 +52,24 @@ namespace MyNamespace
         private static ITransformer BuildTrainEvaluateAndSaveModel(MLContext mlContext)
         {
             // Data loading
-            IDataView trainingDataView = mlContext.Data.ReadFromTextFile<SampleObservation>(
+            IDataView trainingDataView = mlContext.Data.LoadFromTextFile<SampleObservation>(
                                             path: TrainDataPath,
                                             hasHeader: true,
                                             separatorChar: ',',
-                                            allowQuotedStrings: true,
-                                            trimWhitespace: false,
-                                            supportSparse: true);
-            IDataView testDataView = mlContext.Data.ReadFromTextFile<SampleObservation>(
+                                            allowQuoting: true,
+                                            allowSparse: true);
+            IDataView testDataView = mlContext.Data.LoadFromTextFile<SampleObservation>(
                                             path: TestDataPath,
                                             hasHeader: true,
                                             separatorChar: ',',
-                                            allowQuotedStrings: true,
-                                            trimWhitespace: false,
-                                            supportSparse: true);
+                                            allowQuoting: true,
+                                            allowSparse: true);
 
             // Common data process configuration with pipeline data transformations
             var dataProcessPipeline = mlContext.Transforms.Concatenate("Out", new[] { "In" });
 
             // Set the training algorithm, then create and config the modelBuilder  
-            var trainer = mlContext.BinaryClassification.Trainers.LightGbm(new Options() { NumLeaves = 2, Booster = new Options.TreeBooster.Arguments() { }, LabelColumn = "Label", FeatureColumn = "Features" });
+            var trainer = mlContext.BinaryClassification.Trainers.LightGbm(new Options() { NumLeaves = 2, Booster = new Options.TreeBooster.Options() { }, LabelColumn = "Label", FeatureColumn = "Features" });
             var trainingPipeline = dataProcessPipeline.Append(trainer);
 
             // Train the model fitting to the DataSet
@@ -95,15 +96,14 @@ namespace MyNamespace
         private static void TestSinglePrediction(MLContext mlContext)
         {
             //Load data to test. Could be any test data. For demonstration purpose train data is used here.
-            IDataView trainingDataView = mlContext.Data.ReadFromTextFile<SampleObservation>(
+            IDataView trainingDataView = mlContext.Data.LoadFromTextFile<SampleObservation>(
                                             path: TrainDataPath,
                                             hasHeader: true,
                                             separatorChar: ',',
-                                            allowQuotedStrings: true,
-                                            trimWhitespace: false,
-                                            supportSparse: true);
+                                            allowQuoting: true,
+                                            allowSparse: true);
 
-            var sample = mlContext.CreateEnumerable<SampleObservation>(trainingDataView, false).First();
+            var sample = mlContext.Data.CreateEnumerable<SampleObservation>(trainingDataView, false).First();
 
             ITransformer trainedModel;
             using (var stream = new FileStream(ModelPath, FileMode.Open, FileAccess.Read, FileShare.Read))
@@ -131,23 +131,23 @@ namespace MyNamespace
 
 
         [ColumnName("col1"), LoadColumn(1)]
-        public float Col1 { get; set; }
+        public float col1 { get; set; }
 
 
         [ColumnName("col2"), LoadColumn(0)]
-        public float Col2 { get; set; }
+        public float col2 { get; set; }
 
 
         [ColumnName("col3"), LoadColumn(0)]
-        public string Col3 { get; set; }
+        public string col3 { get; set; }
 
 
         [ColumnName("col4"), LoadColumn(0)]
-        public int Col4 { get; set; }
+        public int col4 { get; set; }
 
 
         [ColumnName("col5"), LoadColumn(0)]
-        public uint Col5 { get; set; }
+        public uint col5 { get; set; }
 
 
     }

src/mlnet.Test/CodeGenTests.cs

@@ -28,7 +28,7 @@ public void TrainerGeneratorBasicNamedParameterTest()
             Pipeline pipeline = new Pipeline(new PipelineNode[] { node });
             CodeGenerator codeGenerator = new CodeGenerator(pipeline, null, null);
             var actual = codeGenerator.GenerateTrainerAndUsings();
-            string expected = "LightGbm(learningRate:0.1f,numLeaves:1,labelColumn:\"Label\",featureColumn:\"Features\")";
+            string expected = "LightGbm(learningRate:0.1f,numLeaves:1,labelColumnName:\"Label\",featureColumnName:\"Features\")";
             Assert.AreEqual(expected, actual.Item1);
             Assert.IsNull(actual.Item2);
         }
@@ -77,7 +77,7 @@ public void TransformGeneratorUsingTest()
             Pipeline pipeline = new Pipeline(new PipelineNode[] { node });
             CodeGenerator codeGenerator = new CodeGenerator(pipeline, null, null);
             var actual = codeGenerator.GenerateTransformsAndUsings();
-            string expectedTransform = "Categorical.OneHotEncoding(new []{new OneHotEncodingEstimator.ColumnInfo(\"Label\",\"Label\")})";
+            string expectedTransform = "Categorical.OneHotEncoding(new []{new OneHotEncodingEstimator.ColumnOptions(\"Label\",\"Label\")})";
             var expectedUsings = "using Microsoft.ML.Transforms.Categorical;\r\n";
             Assert.AreEqual(expectedTransform, actual[0].Item1);
             Assert.AreEqual(expectedUsings, actual[0].Item2);
@@ -136,7 +136,7 @@ public void ColumnGenerationTest()
                 },
                 ColumnInformation = new ColumnInformation() { NumericColumns = new[] { DefaultColumnNames.Features } }
             };
-            
+
             var context = new MLContext();
             var elementProperties = new Dictionary<string, object>();
             PipelineNode node = new PipelineNode("Normalizing", PipelineNodeType.Transform, new string[] { "Label" }, new string[] { "Label" }, elementProperties);