Created samples for TokenizeIntoWords and RemoveStopWords APIs. (#3156)

Author: zeahmed

docs/samples/Microsoft.ML.Samples/Dynamic/StopWordRemoverTransform.cs

@@ -13,7 +13,7 @@ public static void Example()
             // as well as the source of randomness.
             var mlContext = new MLContext();
 
-            // Create an empty data sample list. The 'ApplyWordEmbedding' does not require training data as
+            // Create an empty list as the dataset. The 'ApplyWordEmbedding' does not require training data as
             // the estimator ('WordEmbeddingEstimator') created by 'ApplyWordEmbedding' API is not a trainable estimator.
             // The empty list is only needed to pass input schema to the pipeline.
             var emptySamples = new List<TextData>();

docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/ApplyCustomWordEmbedding.cs

@@ -12,7 +12,7 @@ public static void Example()
             // as well as the source of randomness.
             var mlContext = new MLContext();
 
-            // Create an empty data sample list. The 'ApplyWordEmbedding' does not require training data as
+            // Create an empty list as the dataset. The 'ApplyWordEmbedding' does not require training data as
             // the estimator ('WordEmbeddingEstimator') created by 'ApplyWordEmbedding' API is not a trainable estimator.
             // The empty list is only needed to pass input schema to the pipeline.
             var emptySamples = new List<TextData>();

docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/ApplyWordEmbedding.cs

@@ -12,7 +12,7 @@ public static void Example()
             // as well as the source of randomness.
             var mlContext = new MLContext();
 
-            // Create an empty data sample list. The 'NormalizeText' API does not require training data as
+            // Create an empty list as the dataset. The 'NormalizeText' API does not require training data as
             // the estimator ('TextNormalizingEstimator') created by 'NormalizeText' API is not a trainable estimator.
             // The empty list is only needed to pass input schema to the pipeline.
             var emptySamples = new List<TextData>();

docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/NormalizeText.cs

@@ -0,0 +1,60 @@
+using System;
+using System.Collections.Generic;
+using Microsoft.ML.Transforms.Text;
+
+namespace Microsoft.ML.Samples.Dynamic
+{
+    public static class RemoveDefaultStopWords
+    {
+        public static void Example()
+        {
+            // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, 
+            // as well as the source of randomness.
+            var mlContext = new MLContext();
+
+            // Create an empty list as the dataset. The 'RemoveDefaultStopWords' does not require training data as
+            // the estimator ('StopWordsRemovingEstimator') created by 'RemoveDefaultStopWords' API is not a trainable estimator.
+            // The empty list is only needed to pass input schema to the pipeline.
+            var emptySamples = new List<TextData>();
+
+            // Convert sample list to an empty IDataView.
+            var emptyDataView = mlContext.Data.LoadFromEnumerable(emptySamples);
+
+            // A pipeline for removing stop words from input text/string.
+            // The pipeline first tokenizes text into words then removes stop words.
+            // The 'RemoveDefaultStopWords' API ignores casing of the text/string e.g. 'tHe' and 'the' are considered the same stop words.
+            var textPipeline = mlContext.Transforms.Text.TokenizeIntoWords("Words", "Text")
+                .Append(mlContext.Transforms.Text.RemoveDefaultStopWords("WordsWithoutStopWords", "Words", language: StopWordsRemovingEstimator.Language.English));
+
+            // Fit to data.
+            var textTransformer = textPipeline.Fit(emptyDataView);
+
+            // Create the prediction engine to remove the stop words from the input text/string.
+            var predictionEngine = mlContext.Model.CreatePredictionEngine<TextData, TransformedTextData>(textTransformer);
+
+            // Call the prediction API to remove stop words.
+            var data = new TextData() { Text = "ML.NET's RemoveDefaultStopWords API removes stop words from tHe text/string. It requires the text/string to be tokenized beforehand." };
+            var prediction = predictionEngine.Predict(data);
+
+            // Print the length of the word vector after the stop words removed.
+            Console.WriteLine($"Number of words: {prediction.WordsWithoutStopWords.Length}");
+
+            // Print the word vector without stop words.
+            Console.WriteLine($"\nWords without stop words: {string.Join(",", prediction.WordsWithoutStopWords)}");
+
+            //  Expected output:
+            //   Number of words: 11
+            //   Words without stop words: ML.NET's,RemoveDefaultStopWords,API,removes,stop,words,text/string.,requires,text/string,tokenized,beforehand.
+        }
+
+        public class TextData
+        {
+            public string Text { get; set; }
+        }
+
+        public class TransformedTextData : TextData
+        {
+            public string[] WordsWithoutStopWords { get; set; }
+        }
+    }
+}

docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/RemoveDefaultStopWords.cs

@@ -0,0 +1,60 @@
+using System;
+using System.Collections.Generic;
+using Microsoft.ML.Transforms.Text;
+
+namespace Microsoft.ML.Samples.Dynamic
+{
+    public static class RemoveStopWords
+    {
+        public static void Example()
+        {
+            // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, 
+            // as well as the source of randomness.
+            var mlContext = new MLContext();
+
+            // Create an empty list as the dataset. The 'RemoveStopWords' does not require training data as
+            // the estimator ('CustomStopWordsRemovingEstimator') created by 'RemoveStopWords' API is not a trainable estimator.
+            // The empty list is only needed to pass input schema to the pipeline.
+            var emptySamples = new List<TextData>();
+
+            // Convert sample list to an empty IDataView.
+            var emptyDataView = mlContext.Data.LoadFromEnumerable(emptySamples);
+
+            // A pipeline for removing stop words from input text/string.
+            // The pipeline first tokenizes text into words then removes stop words.
+            // The 'RemoveStopWords' API ignores casing of the text/string e.g. 'tHe' and 'the' are considered the same stop words.
+            var textPipeline = mlContext.Transforms.Text.TokenizeIntoWords("Words", "Text")
+                .Append(mlContext.Transforms.Text.RemoveStopWords("WordsWithoutStopWords", "Words", stopwords: new[] { "a", "the", "from", "by" }));
+
+            // Fit to data.
+            var textTransformer = textPipeline.Fit(emptyDataView);
+
+            // Create the prediction engine to remove the stop words from the input text/string.
+            var predictionEngine = mlContext.Model.CreatePredictionEngine<TextData, TransformedTextData>(textTransformer);
+
+            // Call the prediction API to remove stop words.
+            var data = new TextData() { Text = "ML.NET's RemoveStopWords API removes stop words from tHe text/string using a list of stop words provided by the user." };
+            var prediction = predictionEngine.Predict(data);
+
+            // Print the length of the word vector after the stop words removed.
+            Console.WriteLine($"Number of words: {prediction.WordsWithoutStopWords.Length}");
+
+            // Print the word vector without stop words.
+            Console.WriteLine($"\nWords without stop words: {string.Join(",", prediction.WordsWithoutStopWords)}");
+
+            //  Expected output:
+            //   Number of words: 14
+            //   Words without stop words: ML.NET's,RemoveStopWords,API,removes,stop,words,text/string,using,list,of,stop,words,provided,user.
+        }
+
+        public class TextData
+        {
+            public string Text { get; set; }
+        }
+
+        public class TransformedTextData : TextData
+        {
+            public string[] WordsWithoutStopWords { get; set; }
+        }
+    }
+}

docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/RemoveStopWords.cs

@@ -12,7 +12,7 @@ public static void Example()
             // as well as the source of randomness.
             var mlContext = new MLContext();
 
-            // Create an empty data sample list. The 'TokenizeIntoCharactersAsKeys' does not require training data as
+            // Create an empty list as the dataset. The 'TokenizeIntoCharactersAsKeys' does not require training data as
             // the estimator ('TokenizingByCharactersEstimator') created by 'TokenizeIntoCharactersAsKeys' API is not a trainable estimator.
             // The empty list is only needed to pass input schema to the pipeline.
             var emptySamples = new List<TextData>();

docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/TokenizeIntoCharactersAsKeys.cs

@@ -0,0 +1,59 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace Microsoft.ML.Samples.Dynamic
+{
+    public static class TokenizeIntoWords
+    {
+        public static void Example()
+        {
+            // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, 
+            // as well as the source of randomness.
+            var mlContext = new MLContext();
+
+            // Create an empty list as the dataset. The 'TokenizeIntoWords' does not require training data as
+            // the estimator ('WordTokenizingEstimator') created by 'TokenizeIntoWords' API is not a trainable estimator.
+            // The empty list is only needed to pass input schema to the pipeline.
+            var emptySamples = new List<TextData>();
+
+            // Convert sample list to an empty IDataView.
+            var emptyDataView = mlContext.Data.LoadFromEnumerable(emptySamples);
+
+            // A pipeline for converting text into vector of words.
+            // The following call to 'TokenizeIntoWords' tokenizes text/string into words using space as a separator.
+            // Space is also a default value for the 'separators' argument if it is not specified.
+            var textPipeline = mlContext.Transforms.Text.TokenizeIntoWords("Words", "Text", separators: new[] { ' ' });
+
+            // Fit to data.
+            var textTransformer = textPipeline.Fit(emptyDataView);
+
+            // Create the prediction engine to get the word vector from the input text/string.
+            var predictionEngine = mlContext.Model.CreatePredictionEngine<TextData, TransformedTextData>(textTransformer);
+
+            // Call the prediction API to convert the text into words.
+            var data = new TextData() { Text = "ML.NET's TokenizeIntoWords API splits text/string into words using the list of characters provided as separators." };
+            var prediction = predictionEngine.Predict(data);
+
+            // Print the length of the word vector.
+            Console.WriteLine($"Number of words: {prediction.Words.Length}");
+
+            // Print the word vector.
+            Console.WriteLine($"\nWords: {string.Join(",", prediction.Words)}");
+
+            //  Expected output:
+            //   Number of words: 15
+            //   Words: ML.NET's,TokenizeIntoWords,API,splits,text/string,into,words,using,the,list,of,characters,provided,as,separators.
+        }
+
+        public class TextData
+        {
+            public string Text { get; set; }
+        }
+
+        public class TransformedTextData : TextData
+        {
+            public string[] Words { get; set; }
+        }
+    }
+}

docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/TokenizeIntoWords.cs

@@ -186,6 +186,13 @@ internal static WordEmbeddingEstimator ApplyWordEmbedding(this TransformsCatalog
         /// <param name="outputColumnName">Name of the column resulting from the transformation of <paramref name="inputColumnName"/>.</param>
         /// <param name="inputColumnName">Name of the column to transform. If set to <see langword="null"/>, the value of the <paramref name="outputColumnName"/> will be used as source.</param>
         /// <param name="separators">The separators to use (uses space character by default).</param>
+        /// <example>
+        /// <format type="text/markdown">
+        /// <![CDATA[
+        /// [!code-csharp[TokenizeIntoWords](~/../docs/samples/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/TokenizeIntoWords.cs)]
+        /// ]]>
+        /// </format>
+        /// </example>
         public static WordTokenizingEstimator TokenizeIntoWords(this TransformsCatalog.TextTransforms catalog,
             string outputColumnName,
             string inputColumnName = null,
@@ -254,8 +261,9 @@ internal static NgramExtractingEstimator ProduceNgrams(this TransformsCatalog.Te
         /// <example>
         /// <format type="text/markdown">
         /// <![CDATA[
-        ///  [!code-csharp[FastTree](~/../docs/samples/docs/samples/Microsoft.ML.Samples/Dynamic/StopWordRemoverTransform.cs)]
-        /// ]]></format>
+        /// [!code-csharp[RemoveDefaultStopWords](~/../docs/samples/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/RemoveDefaultStopWords.cs)]
+        /// ]]>
+        /// </format>
         /// </example>
         public static StopWordsRemovingEstimator RemoveDefaultStopWords(this TransformsCatalog.TextTransforms catalog,
             string outputColumnName,
@@ -274,8 +282,9 @@ public static StopWordsRemovingEstimator RemoveDefaultStopWords(this TransformsC
         /// <example>
         /// <format type="text/markdown">
         /// <![CDATA[
-        ///  [!code-csharp[FastTree](~/../docs/samples/docs/samples/Microsoft.ML.Samples/Dynamic/StopWordRemoverTransform.cs)]
-        /// ]]></format>
+        /// [!code-csharp[RemoveStopWords](~/../docs/samples/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/RemoveStopWords.cs)]
+        /// ]]>
+        /// </format>
         /// </example>
         public static CustomStopWordsRemovingEstimator RemoveStopWords(this TransformsCatalog.TextTransforms catalog,
             string outputColumnName,