Transforms components docs

docs/samples/Microsoft.ML.Samples/Dynamic/ConcatTransform.cs

@@ -0,0 +1,66 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+        // the alignment of the usings with the methods is intentional so they can display on the same level in the docs site.
+        using Microsoft.ML.Runtime.Data;
+        using Microsoft.ML.Runtime.Api;
+        using Microsoft.ML.Data;
+        using System;
+        using System.Linq;
+        using System.Collections.Generic;
+
+namespace Microsoft.ML.Samples.Dynamic
+{
+    public partial class TransformSamples
+    {
+        class SampleInfertDataWithFeatures
+        {
+            public VBuffer<int> Features { get; set; }
+        }
+
+        public static void ConcatTransform()
+        {
+            // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, 
+            // as well as the source of randomness.
+            var ml = new MLContext(seed: 1, conc: 1);
+
+            // Get a small dataset as an IEnumerable.
+            IEnumerable<SamplesUtils.DatasetUtils.SampleInfertData> data = SamplesUtils.DatasetUtils.GetInfertData();
+            var trainData = ml.CreateStreamingDataView(data);
+
+            // Preview of the data.
+            // Age    Case  Education  induced     parity  pooled.stratum  row_num  ...
+            // 26.0   1.0   0-5yrs      1.0         6.0       3.0      1.0  ...
+            // 42.0   1.0   0-5yrs      1.0         1.0       1.0      2.0  ...
+            // 39.0   1.0   0-5yrs      2.0         6.0       4.0      3.0  ...
+            // 34.0   1.0   0-5yrs      2.0         4.0       2.0      4.0  ...
+            // 35.0   1.0   6-11yrs     1.0         3.0       32.0     5.0  ...
+
+            // A pipeline for concatenating the age, parity and induced columns together in the Features column.
+            string outputColumnName = "Features";
+            var pipeline = new ConcatEstimator(ml, outputColumnName, new[] { "Age", "Parity", "Induced"});
+
+            // The transformed data.
+            var transformedData = pipeline.Fit(trainData).Transform(trainData);
+
+            // Getting the data of the newly created column as an IEnumerable of SampleInfertDataWithFeatures.
+            var featuresColumn = transformedData.AsEnumerable<SampleInfertDataWithFeatures>(ml, reuseRowObject: false);
+
+            Console.WriteLine($"{outputColumnName} column obtained post-transformation.");
+            foreach (var featureRow in featuresColumn)
+            {
+                foreach (var value in featureRow.Features.Values)
+                    Console.Write($"{value} ");
+                Console.WriteLine("");
+            }
+
+            // Features
+            // 26 6 1
+            // 42 1 1
+            // 39 6 2
+            // 34 4 2
+            // 35 3 1
+        }
+    }
+}

docs/samples/Microsoft.ML.Samples/Dynamic/KeyToValue_Term.cs

@@ -0,0 +1,111 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+        // the alignment of the usings with the methods is intentional so they can display on the same level in the docs site.
+        using Microsoft.ML.Data;
+        using Microsoft.ML.Runtime.Api;
+        using Microsoft.ML.Runtime.Data;
+        using Microsoft.ML.Transforms.Text;
+        using System;
+        using System.Collections.Generic;
+        using System.Linq;
+
+namespace Microsoft.ML.Samples.Dynamic
+{
+    public partial class TransformSamples
+    {
+        public static void KeyToValue_Term()
+        {
+            // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, 
+            // as well as the source of randomness.
+            var ml = new MLContext(seed: 1, conc: 1);
+
+            // Get a small dataset as an IEnumerable.
+            IEnumerable<SamplesUtils.DatasetUtils.SampleTopicsData> data = SamplesUtils.DatasetUtils.GetTopicsData();
+            var trainData = ml.CreateStreamingDataView(data);
+
+            // Preview of the topics data; a dataset that contains one column with two set of keys assigned to a body of text
+            // Review and ReviewReverse. The dataset will be used to classify how accurately the keys are assigned to the text.  
+            // Review,                                    ReviewReverse,                 Label
+            // "animals birds cats dogs fish horse", "radiation galaxy universe duck",     1
+            // "horse birds house fish duck cats",   "space galaxy universe radiation",    0
+            // "car truck driver bus pickup",        "bus pickup",                         1
+            // "car truck driver bus pickup horse",  "car truck",                          0
+
+            // A pipeline to convert the terms of the review_reverse column in 
+            // making use of default settings.
+            string defaultColumnName = "DefaultKeys";
+            // REVIEW create through the catalog extension
+            var default_pipeline = new WordTokenizeEstimator(ml, "ReviewReverse", "ReviewReverse")
+                .Append(new TermEstimator(ml, "ReviewReverse" , defaultColumnName));
+
+            // Another pipeline, that customizes the advanced settings of the TermEstimator.
+            // We can change the maxNumTerm to limit how many keys will get generated out of the set of words, 
+            // and condition the order in which they get evaluated by changing sort from the default Occurence (order in which they get encountered) 
+            // to value/alphabetically.
+            string customizedColumnName = "CustomizedKeys";
+            var customized_pipeline = new WordTokenizeEstimator(ml, "ReviewReverse", "ReviewReverse")
+                .Append(new TermEstimator(ml, "ReviewReverse", customizedColumnName, maxNumTerms: 10, sort:TermTransform.SortOrder.Value));
+
+            // The transformed data.
+            var transformedData_default = default_pipeline.Fit(trainData).Transform(trainData);
+            var transformedData_customized = customized_pipeline.Fit(trainData).Transform(trainData);
+
+            // Small helper to print the text inside the columns, in the console. 
+            Action<string, IEnumerable<VBuffer<uint>>> printHelper = (columnName, column) =>
+            {
+                Console.WriteLine($"{columnName} column obtained post-transformation.");
+                foreach (var row in column)
+                {
+                    foreach (var value in row.Values)
+                        Console.Write($"{value} ");
+                    Console.WriteLine("");
+                }
+
+                Console.WriteLine("===================================================");
+            };
+
+            // Preview of the DefaultKeys column obtained after processing the input.
+            var defaultColumn = transformedData_default.GetColumn<VBuffer<uint>>(ml, defaultColumnName);
+            printHelper(defaultColumnName, defaultColumn);
+
+            // DefaultKeys column obtained post-transformation.
+            // 1 2 3 4
+            // 5 2 3 1
+            // 6 7 3 1
+            // 8 9 3 1
+
+            // Previewing the CustomizedKeys column obtained after processing the input.
+            var customizedColumn = transformedData_customized.GetColumn<VBuffer<uint>>(ml, customizedColumnName);
+            printHelper(customizedColumnName, customizedColumn);
+
+            // CustomizedKeys
+            // 6 4 9 3
+            // 7 4 9 6
+            // 1 5 9 6
+            // 2 8 9 6
+
+            // Retrieve the original values, by appending the KeyToValue etimator to the existing pipelines
+            // to convert the keys back to the strings.
+            var pipeline = default_pipeline.Append(new KeyToValueEstimator(ml, defaultColumnName));
+            transformedData_default = pipeline.Fit(trainData).Transform(trainData);
+
+            // Preview of the DefaultColumnName column obtained.
+            var originalColumnBack = transformedData_default.GetColumn<VBuffer<ReadOnlyMemory<char>>>(ml, defaultColumnName);
+
+            foreach (var row in originalColumnBack)
+            {
+                foreach (var value in row.Values)
+                    Console.Write($"{value} ");
+                Console.WriteLine("");
+            }
+
+            // DefaultKeys 
+            // radiation galaxy universe duck
+            // space galaxy universe radiation
+            // bus pickup universe radiation
+            // car truck universe radiation
+        }
+    }
+}

docs/samples/Microsoft.ML.Samples/Dynamic/MinMaxNormalizer.cs

@@ -0,0 +1,88 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+        // the alignment of the usings with the methods is intentional so they can display on the same level in the docs site.
+        using Microsoft.ML.Runtime.Data;
+        using Microsoft.ML.Runtime.Api;
+        using Microsoft.ML.Data;
+        using System;
+        using System.Collections.Generic;
+
+namespace Microsoft.ML.Samples.Dynamic
+{
+    public partial class TransformSamples
+    {
+        public static void MinMaxNormalizer()
+        {
+            // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, 
+            // as well as the source of randomness.
+            var ml = new MLContext(seed: 1, conc: 1);
+
+            // Get a small dataset as an IEnumerable and convert it to an IDataView.
+            IEnumerable<SamplesUtils.DatasetUtils.SampleInfertData> data = SamplesUtils.DatasetUtils.GetInfertData();
+            var trainData = ml.CreateStreamingDataView(data);
+
+            // Preview of the data.
+            // Age  Case  Education  Induced     Parity  PooledStratum  RowNum  ...
+            // 26   1       0-5yrs      1         6         3             1  ...
+            // 42   1       0-5yrs      1         1         1             2  ...
+            // 39   1       0-5yrs      2         6         4             3  ...
+            // 34   1       0-5yrs      2         4         2             4  ...
+            // 35   1       6-11yrs     1         3         32            5  ...
+
+            // A pipeline for normalizing the Induced column.
+            var pipeline = ml.Transforms.Normalizer("Induced");
+            // The transformed (normalized according to Normalizer.NormalizerMode.MinMax) data.
+            var transformedData = pipeline.Fit(trainData).Transform(trainData);
+            // Getting the data of the newly created column, so we can preview it.
+            var normalizedColumn = transformedData.GetColumn<float>(ml, "Induced");
+
+            // A small printing utility.
+            Action<string, IEnumerable<float>> printHelper = (colName, column) =>
+            {
+                Console.WriteLine($"{colName} column obtained post-transformation.");
+                foreach (var row in column)
+                    Console.WriteLine($"{row} ");
+            };
+
+            printHelper("Induced", normalizedColumn);
+
+            // Preview of the data.
+            // Induced
+            // 0.5
+            // 0.5
+            // 1
+            // 1
+            // 0.5
+
+            // Composing a different pipeline if we wanted to normalize more than one column at a time. 
+            // Using log scale as the normalization mode. 
+            var multiColPipeline = ml.Transforms.Normalizer(Normalizer.NormalizerMode.LogMeanVariance, new[] { ("Induced", "LogInduced"), ("Spontaneous", "LogSpontaneous") });
+            // The transformed data.
+            var multiColtransformedData = multiColPipeline.Fit(trainData).Transform(trainData);
+
+            // Getting the newly created columns. 
+            var normalizedInduced = multiColtransformedData.GetColumn<float>(ml, "LogInduced");
+            var normalizedSpont = multiColtransformedData.GetColumn<float>(ml, "LogSpontaneous");
+
+            printHelper("LogInduced", normalizedInduced);
+
+            // LogInduced
+            // 0.2071445
+            // 0.2071445
+            // 0.889631
+            // 0.889631
+            // 0.2071445
+
+            printHelper("LogSpontaneous", normalizedSpont);
+
+            // LogSpontaneous
+            // 0.8413026
+            // 0
+            // 0
+            // 0
+            // 0.1586974
+        }
+    }
+}

docs/samples/Microsoft.ML.Samples/Dynamic/TextTransform.cs

@@ -0,0 +1,84 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+        // the alignment of the usings with the methods is intentional so they can display on the same level in the docs site.
+        using Microsoft.ML.Runtime.Data;
+        using Microsoft.ML.Runtime.Api;
+        using Microsoft.ML.Data;
+        using System;
+        using System.Collections.Generic;
+
+namespace Microsoft.ML.Samples.Dynamic
+{
+    public partial class TransformSamples
+    {
+        public static void TextTransform()
+        {
+            // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, 
+            // as well as the source of randomness.
+            var ml = new MLContext(seed: 1, conc: 1);
+
+            // Get a small dataset as an IEnumerable and convert to IDataView.
+            IEnumerable<SamplesUtils.DatasetUtils.SampleSentimentData> data = SamplesUtils.DatasetUtils.GetSentimentData();
+            var trainData = ml.CreateStreamingDataView(data);
+
+            // Preview of the data.
+            // Sentiment    SentimentText
+            // true         Best game I've ever played.
+            // false        ==RUDE== Dude, 2.
+            // true          Until the next game, this is the best Xbox game!
+
+            // A pipeline for featurization of the "SentimentText" column, and placing the output in a new column named "DefaultTextFeatures"
+            // The pipeline uses the default settings to featurize.
+            string defaultColumnName = "DefaultTextFeatures";
+            var default_pipeline = ml.Transforms.Text.FeaturizeText("SentimentText", defaultColumnName);
+
+            // Another pipeline, that customizes the advanced settings of the FeaturizeText transformer.
+            string customizedColumnName = "CustomizedTextFeatures";
+            var customized_pipeline = ml.Transforms.Text.FeaturizeText("SentimentText", customizedColumnName, s =>
+            {
+                s.KeepPunctuations = false;
+                s.KeepNumbers = false;
+                s.OutputTokens = true;
+                s.TextLanguage = Runtime.Data.TextTransform.Language.English; // supports  English, French, German, Dutch, Italian, Spanish, Japanese
+            });
+
+            // The transformed data for both pipelines.
+            var transformedData_default = default_pipeline.Fit(trainData).Transform(trainData);
+            var transformedData_customized = customized_pipeline.Fit(trainData).Transform(trainData);
+
+            // Small helper to print the text inside the columns, in the console. 
+            Action<string, IEnumerable<VBuffer<float>>> printHelper = (columnName, column) =>
+            {
+                Console.WriteLine($"{columnName} column obtained post-transformation.");
+                foreach (var featureRow in column)
+                {
+                    foreach (var value in featureRow.Values)
+                        Console.Write($"{value} ");
+                    Console.WriteLine("");
+                }
+
+                Console.WriteLine("===================================================");
+            };
+
+            // Preview of the DefaultTextFeatures column obtained after processing the input.
+            var defaultColumn = transformedData_default.GetColumn<VBuffer<float>>(ml, defaultColumnName);
+            printHelper(defaultColumnName, defaultColumn);
+
+            // DefaultTextFeatures 
+            // 0.1924501 0.1924501 0.1924501 0.1924501 0.1924501 0.1924501 0.1924501 0.1924501 0.1924501 0.1924501 0.1924501 0.1924501 0.1924501 0.1924501 0.1924501 0.1924501 0.1924501 0.1924501 0.1924501 0.1924501 0.1924501 0.1924501 0.1924501 0.1924501 0.1924501 0.1924501 0.1924501 0.4472136 0.4472136 0.4472136 0.4472136 0.4472136
+            // 0.2357023 0.2357023 0.2357023 0.2357023 0.4714046 0.2357023 0.2357023 0.2357023 0.2357023 0.2357023 0.2357023 0.2357023 0.2357023 0.2357023 0.2357023 0.5773503 0.5773503 0.5773503 0.1924501 0.1924501 0.1924501 0.1924501 0.1924501 0.1924501 0.1924501 0.1924501 0.1924501 0.4472136 0.4472136 0.4472136 0.4472136 0.4472136
+            // 0 0.1230915 0.1230915 0.1230915 0.1230915 0.246183 0.246183 0.246183 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0.1230915 0 0 0.1230915 0.1230915 0.1230915 0.1230915 0.1230915 0.1230915 0.3692745 0.246183 0.246183 0.1230915 0.1230915 0.1230915 0.1230915 0.1230915 0.1230915 0.1230915 0.1230915 0.1230915 0.246183 0.1230915 0.1230915 0.1230915 0.1230915 0.1230915 0.1230915 0.1230915 0.1230915 0.1230915 0.1230915 0.1230915 0.1230915 0.1230915 0.2886751 0 0 0 0 0 0 0 0.2886751 0.5773503 0.2886751 0.2886751 0.2886751 0.2886751 0.2886751 0.2886751
+
+            // Preview of the CustomizedTextFeatures column obtained after processing the input.
+            var customizedColumn = transformedData_customized.GetColumn<VBuffer<float>>(ml, customizedColumnName);
+            printHelper(customizedColumnName, customizedColumn);
+
+            // CustomizedTextFeatures
+            // 0.2 0.2 0.2 0.2 0.2 0.2 0.2 0.2 0.2 0.2 0.2 0.2 0.2 0.2 0.2 0.2 0.2 0.2 0.2 0.2 0.2 0.2 0.2 0.2 0.2 0.4472136 0.4472136 0.4472136 0.4472136 0.4472136
+            // 0.25 0.25 0.25 0.25 0.5 0.25 0.25 0.25 0.25 0.25 0.25 0.25 0.25 0.7071068 0.7071068 0.2 0.2 0.2 0.2 0.2 0.2 0.2 0.2 0.2 0.2 0.4472136 0.4472136 0.4472136 0.4472136 0.4472136
+            // 0 0.125 0.125 0.125 0.125 0.25 0.25 0.25 0.125 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0.125 0.125 0.125 0.125 0.125 0.125 0.375 0.25 0.25 0.125 0.125 0.125 0.125 0.125 0.125 0.125 0.125 0.25 0.125 0.125 0.125 0.125 0.125 0.125 0.125 0.125 0.125 0.125 0.125 0.125 0.2672612 0.5345225 0 0 0 0 0 0.2672612 0.5345225 0.2672612 0.2672612 0.2672612 0.2672612        }
+        }
+    }
+}

docs/samples/Microsoft.ML.Samples/Program.cs

@@ -2,14 +2,13 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
-namespace Microsoft.ML.Samples
+namespace Microsoft.ML.Samples.Dynamic
 {
     internal static class Program
     {
         static void Main(string[] args)
         {
-            Trainers.SdcaRegression();
-            Transformers.ConcatEstimator();
+            TransformSamples.MinMaxNormalizer();
         }
     }
 }

docs/samples/Microsoft.ML.Samples/Transformers.cs → docs/samples/Microsoft.ML.Samples/Static/ConcatEstimator.cs

@@ -3,7 +3,7 @@
 // See the LICENSE file in the project root for more information.
 
 // the alignment of the usings with the methods is intentional so they can display on the same level in the docs site.
-using Microsoft.ML.Runtime.Api;
+        using Microsoft.ML.Runtime.Api;
         using Microsoft.ML.Runtime.Data;
         using Microsoft.ML.StaticPipe;
         using System;
@@ -12,15 +12,15 @@
 // NOTE: WHEN ADDING TO THE FILE, ALWAYS APPEND TO THE END OF IT. 
 // If you change the existinc content, check that the files referencing it in the XML documentation are still correct, as they reference
 // line by line. 
-namespace Microsoft.ML.Samples
+namespace Microsoft.ML.Samples.Static
 {
-    public static class Transformers
+    public partial class TransformSamples
     {
 
         /// <summary>
         /// The example for the statically typed concat estimator.
         /// </summary>
-        public static void ConcatEstimator()
+        public static void ConcatWith()
         {
             // Create a new environment for ML.NET operations. It can be used for exception tracking and logging, 
             // as well as the source of randomness.