Skip to content

OutputTokens option in FeaturizeText API #2985

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Mar 19, 2019
Merged

OutputTokens option in FeaturizeText API #2985

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
1771506
example for TokenizeIntoWords and ApplyWordEmbedding
abgoswam Mar 15, 2019
f2ed7c7
test passes
abgoswam Mar 15, 2019
5d6a2f6
uses OutputTokensColumnName to generate column containing tokens
abgoswam Mar 18, 2019
d72c344
taking care of review comments
abgoswam Mar 18, 2019
40eb3f4
Merge branch 'master' into abgoswam/featurize_text
abgoswam Mar 18, 2019
91bc869
merge with latest master
abgoswam Mar 18, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/samples/Microsoft.ML.Samples/Dynamic/TextTransform.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ public static void Example()
{
KeepPunctuations = false,
KeepNumbers = false,
OutputTokens = true,
OutputTokensColumnName = "OutputTokens",
Copy link
Contributor

@rogancarr rogancarr Mar 18, 2019 •
edited by abgoswam
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

OutputTokens [](start = 42, length = 12)

TokenizedText perhaps? #Pending

StopWordsRemoverOptions = new StopWordsRemovingEstimator.Options() { Language = TextFeaturizingEstimator.Language.English }, // supports English, French, German, Dutch, Italian, Spanish, Japanese
}, "SentimentText");

Expand Down
23 changes: 11 additions & 12 deletions src/Microsoft.ML.Transforms/Text/TextFeaturizingEstimator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -163,8 +163,8 @@ public IStopWordsRemoverOptions StopWordsRemoverOptions
[Argument(ArgumentType.AtMostOnce, HelpText = "Whether to keep numbers or remove them.", ShortName = "num", SortOrder = 8)]
public bool KeepNumbers = TextNormalizingEstimator.Defaults.KeepNumbers;

[Argument(ArgumentType.AtMostOnce, HelpText = "Whether to output the transformed text tokens as an additional column.", ShortName = "tokens,showtext,showTransformedText", SortOrder = 9)]
public bool OutputTokens;
[Argument(ArgumentType.AtMostOnce, HelpText = "Column containing the transformed text tokens.", ShortName = "tokens,showtext,showTransformedText", SortOrder = 9)]
public string OutputTokensColumnName;

[Argument(ArgumentType.Multiple, HelpText = "A dictionary of whitelisted terms.", ShortName = "dict", NullName = "<None>", SortOrder = 10, Hide = true)]
internal TermLoaderArguments Dictionary;
Expand Down Expand Up @@ -278,7 +278,7 @@ private sealed class TransformApplierParams
public readonly bool KeepDiacritics;
public readonly bool KeepPunctuations;
public readonly bool KeepNumbers;
public readonly bool OutputTextTokens;
public readonly string OutputTextTokensColumnName;
public readonly TermLoaderArguments Dictionary;

public StopWordsRemovingEstimator.Language StopwordsLanguage
Expand All @@ -305,7 +305,7 @@ internal LpNormNormalizingEstimatorBase.NormFunction LpNorm

// These properties encode the logic needed to determine which transforms to apply.
#region NeededTransforms
public bool NeedsWordTokenizationTransform { get { return WordExtractorFactory != null || NeedsRemoveStopwordsTransform || OutputTextTokens; } }
public bool NeedsWordTokenizationTransform { get { return WordExtractorFactory != null || NeedsRemoveStopwordsTransform || !string.IsNullOrEmpty(OutputTextTokensColumnName); } }

public bool NeedsRemoveStopwordsTransform { get { return StopWordsRemover != null; } }

Expand Down Expand Up @@ -358,7 +358,7 @@ public TransformApplierParams(TextFeaturizingEstimator parent)
KeepDiacritics = parent.OptionalSettings.KeepDiacritics;
KeepPunctuations = parent.OptionalSettings.KeepPunctuations;
KeepNumbers = parent.OptionalSettings.KeepNumbers;
OutputTextTokens = parent.OptionalSettings.OutputTokens;
OutputTextTokensColumnName = parent.OptionalSettings.OutputTokensColumnName;
Dictionary = parent._dictionary;
}
}
Expand All @@ -371,8 +371,6 @@ public TransformApplierParams(TextFeaturizingEstimator parent)

internal const Language DefaultLanguage = Language.English;

private const string TransformedTextColFormat = "{0}_TransformedText";

internal TextFeaturizingEstimator(IHostEnvironment env, string outputColumnName, string inputColumnName = null)
: this(env, outputColumnName, new[] { inputColumnName ?? outputColumnName })
{
Expand Down Expand Up @@ -492,10 +490,10 @@ public ITransformer Fit(IDataView input)
wordFeatureCol = dstCol;
}

if (tparams.OutputTextTokens)
if (!string.IsNullOrEmpty(tparams.OutputTextTokensColumnName))
{
string[] srcCols = wordTokCols ?? textCols;
view = new ColumnConcatenatingTransformer(h, string.Format(TransformedTextColFormat, OutputColumn), srcCols).Transform(view);
view = new ColumnConcatenatingTransformer(h, tparams.OutputTextTokensColumnName, srcCols).Transform(view);
}

if (tparams.CharExtractorFactory != null)
Expand Down Expand Up @@ -564,7 +562,7 @@ public ITransformer Fit(IDataView input)
// Otherwise, simply use the slot names, omitting the original source column names
// entirely. For the Concat transform setting the Key == Value of the TaggedColumn
// KVP signals this intent.
Contracts.Assert(charFeatureCol != null || wordFeatureCol != null || tparams.OutputTextTokens);
Contracts.Assert(charFeatureCol != null || wordFeatureCol != null || !string.IsNullOrEmpty(tparams.OutputTextTokensColumnName));
if (charFeatureCol != null)
srcTaggedCols.Add(new KeyValuePair<string, string>(charFeatureCol, charFeatureCol));
else if (wordFeatureCol != null)
Expand Down Expand Up @@ -613,9 +611,10 @@ public SchemaShape GetOutputSchema(SchemaShape inputSchema)

result[OutputColumn] = new SchemaShape.Column(OutputColumn, SchemaShape.Column.VectorKind.Vector, NumberDataViewType.Single, false,
new SchemaShape(metadata));
if (OptionalSettings.OutputTokens)

if (!string.IsNullOrEmpty(OptionalSettings.OutputTokensColumnName))
{
string name = string.Format(TransformedTextColFormat, OutputColumn);
string name = OptionalSettings.OutputTokensColumnName;
result[name] = new SchemaShape.Column(name, SchemaShape.Column.VectorKind.VariableVector, TextDataViewType.Instance, false);
}

Expand Down
8 changes: 4 additions & 4 deletions test/BaselineOutput/Common/EntryPoints/core_manifest.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22428,9 +22428,9 @@
"Default": true
},
{
"Name": "OutputTokens",
"Type": "Bool",
"Desc": "Whether to output the transformed text tokens as an additional column.",
"Name": "OutputTokensColumnName",
"Type": "String",
"Desc": "Column containing the transformed text tokens.",
"Aliases": [
"tokens",
"showtext",
Expand All @@ -22439,7 +22439,7 @@
"Required": false,
"SortOrder": 9.0,
"IsNullable": false,
"Default": false
"Default": null
},
{
"Name": "Dictionary",
Expand Down
2 changes: 1 addition & 1 deletion test/BaselineOutput/SingleDebug/Text/featurized.tsv
View file
Open in desktop

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion test/BaselineOutput/SingleRelease/Text/featurized.tsv
View file
Open in desktop

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion test/Microsoft.ML.Benchmarks/StochasticDualCoordinateAscentClassifierBench.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ public void TrainSentiment()
var loader = mlContext.Data.LoadFromTextFile(_sentimentDataPath, arguments);
var text = mlContext.Transforms.Text.FeaturizeText("WordEmbeddings", new TextFeaturizingEstimator.Options
{
OutputTokens = true,
OutputTokensColumnName = "WordEmbeddings_TransformedText",
KeepPunctuations = false,
StopWordsRemoverOptions = new StopWordsRemovingEstimator.Options(),
Norm = TextFeaturizingEstimator.NormFunction.None,
Expand Down
22 changes: 11 additions & 11 deletions test/Microsoft.ML.Functional.Tests/Debugging.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,16 +47,16 @@ void InspectIntermediatePipelineSteps()

// create a training pipeline.
var pipeline = mlContext.Transforms.Text.FeaturizeText(
"Features",
new TextFeaturizingEstimator.Options
{
KeepPunctuations = false,
OutputTokens = true,
CharFeatureExtractor = null, // new WordBagEstimator.Options { NgramLength = 0, SkipLength = -1 },
WordFeatureExtractor = new WordBagEstimator.Options { NgramLength = 1},
Norm = TextFeaturizingEstimator.NormFunction.None
},
"SentimentText");
"Features",
new TextFeaturizingEstimator.Options
{
KeepPunctuations = false,
OutputTokensColumnName = "FeaturizeTextTokens",
CharFeatureExtractor = null, // new WordBagEstimator.Options { NgramLength = 0, SkipLength = -1 },
WordFeatureExtractor = new WordBagEstimator.Options { NgramLength = 1},
Norm = TextFeaturizingEstimator.NormFunction.None
},
"SentimentText");

// Fit the pipeline to the data.
var model = pipeline.Fit(data);
Expand All @@ -68,7 +68,7 @@ void InspectIntermediatePipelineSteps()

// Verify that columns can be inspected.
// Validate the tokens column.
var tokensColumn = transformedData.GetColumn<string[]>(transformedData.Schema["Features_TransformedText"]);
var tokensColumn = transformedData.GetColumn<string[]>(transformedData.Schema["FeaturizeTextTokens"]);
var expectedTokens = new string[3][]
{
new string[] {"i", "love", "mlnet"},
Expand Down
4 changes: 2 additions & 2 deletions test/Microsoft.ML.Tests/Transformers/TextFeaturizerTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ public void TextFeaturizerWorkout()
.AsDynamic;

var feat = data.MakeNewEstimator()
.Append(row => row.text.FeaturizeText(options: new TextFeaturizingEstimator.Options { OutputTokens = true, }));
.Append(row => row.text.FeaturizeText(options: new TextFeaturizingEstimator.Options { OutputTokensColumnName = "OutputTokens", }));

TestEstimatorCore(feat.AsDynamic, data.AsDynamic, invalidInput: invalidData);

Expand All @@ -51,7 +51,7 @@ public void TextFeaturizerWorkout()
{
var saver = new TextSaver(ML, new TextSaver.Arguments { Silent = true });
var savedData = ML.Data.TakeRows(feat.Fit(data).Transform(data).AsDynamic, 4);
savedData = ML.Transforms.SelectColumns("Data", "Data_TransformedText").Fit(savedData).Transform(savedData);
savedData = ML.Transforms.SelectColumns("Data", "OutputTokens").Fit(savedData).Transform(savedData);

using (var fs = File.Create(outputPath))
DataSaverUtils.SaveDataView(ch, saver, savedData, fs, keepHidden: true);
Expand Down