Skip to content

Textloader internalizations, documentation, and Arguments refactoring #2417

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 9 commits into from
Feb 20, 2019
Merged

Textloader internalizations, documentation, and Arguments refactoring #2417

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
bc7cb49
renamed arguments, added doc and internalized
artidoro Feb 4, 2019
a6b6d29
fixed TryParseSchema
artidoro Feb 5, 2019
8316bd6
cosmetic change
artidoro Feb 5, 2019
1b70723
add some documentation and cleaned up extensions
artidoro Feb 5, 2019
c0e80a2
review comments
artidoro Feb 7, 2019
3265f01
add option not null check
artidoro Feb 14, 2019
f37668f
merge fix
artidoro Feb 15, 2019
592e75c
fix merge
artidoro Feb 19, 2019
a5cb965
added comments to make the function more clear
artidoro Feb 20, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 6 additions & 8 deletions src/Microsoft.ML.Data/DataLoadSave/DataReaderExtensions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,12 @@ namespace Microsoft.ML
{
public static class DataReaderExtensions
{
public static IDataView Read(this IDataReader<IMultiStreamSource> reader, string path)
{
return reader.Read(new MultiFileSource(path));
}

/// <summary>
/// Reads data from one or more file <paramref name="path"/> into an <see cref="IDataView"/>.
/// </summary>
/// <param name="reader">The reader to use.</param>
/// <param name="path">One or more paths from which to load data.</param>
public static IDataView Read(this IDataReader<IMultiStreamSource> reader, params string[] path)
{
return reader.Read(new MultiFileSource(path));
}
=> reader.Read(new MultiFileSource(path));
}
}
231 changes: 175 additions & 56 deletions src/Microsoft.ML.Data/DataLoadSave/Text/TextLoader.cs
View file
Open in desktop

Large diffs are not rendered by default.

32 changes: 16 additions & 16 deletions src/Microsoft.ML.Data/DataLoadSave/Text/TextLoaderSaverCatalog.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,8 @@ public static class TextLoaderSaverCatalog
/// <param name="dataSample">The optional location of a data sample. The sample can be used to infer column names and number of slots in each column.</param>
public static TextLoader CreateTextLoader(this DataOperationsCatalog catalog,
TextLoader.Column[] columns,
bool hasHeader = TextLoader.DefaultArguments.HasHeader,
char separatorChar = TextLoader.DefaultArguments.Separator,
bool hasHeader = TextLoader.Defaults.HasHeader,
char separatorChar = TextLoader.Defaults.Separator,
IMultiStreamSource dataSample = null)
=> new TextLoader(CatalogUtils.GetEnvironment(catalog), columns, hasHeader, separatorChar, dataSample);

Expand Down Expand Up @@ -53,11 +53,11 @@ public static TextLoader CreateTextLoader(this DataOperationsCatalog catalog,
/// except for 3rd and 5th columns which have values 6 and 3</param>
/// <param name="trimWhitespace">Remove trailing whitespace from lines</param>
public static TextLoader CreateTextLoader<TInput>(this DataOperationsCatalog catalog,
bool hasHeader = TextLoader.DefaultArguments.HasHeader,
char separatorChar = TextLoader.DefaultArguments.Separator,
bool allowQuotedStrings = TextLoader.DefaultArguments.AllowQuoting,
bool supportSparse = TextLoader.DefaultArguments.AllowSparse,
bool trimWhitespace = TextLoader.DefaultArguments.TrimWhitespace)
bool hasHeader = TextLoader.Defaults.HasHeader,
char separatorChar = TextLoader.Defaults.Separator,
bool allowQuotedStrings = TextLoader.Defaults.AllowQuoting,
bool supportSparse = TextLoader.Defaults.AllowSparse,
bool trimWhitespace = TextLoader.Defaults.TrimWhitespace)
=> TextLoader.CreateTextReader<TInput>(CatalogUtils.GetEnvironment(catalog), hasHeader, separatorChar, allowQuotedStrings, supportSparse, trimWhitespace);

/// <summary>
Expand All @@ -72,8 +72,8 @@ public static TextLoader CreateTextLoader<TInput>(this DataOperationsCatalog cat
public static IDataView ReadFromTextFile(this DataOperationsCatalog catalog,
string path,
TextLoader.Column[] columns,
bool hasHeader = TextLoader.DefaultArguments.HasHeader,
char separatorChar = TextLoader.DefaultArguments.Separator)
bool hasHeader = TextLoader.Defaults.HasHeader,
char separatorChar = TextLoader.Defaults.Separator)
{
Contracts.CheckNonEmpty(path, nameof(path));

Expand Down Expand Up @@ -104,11 +104,11 @@ public static IDataView ReadFromTextFile(this DataOperationsCatalog catalog,
/// <returns>The data view.</returns>
public static IDataView ReadFromTextFile<TInput>(this DataOperationsCatalog catalog,
string path,
bool hasHeader = TextLoader.DefaultArguments.HasHeader,
char separatorChar = TextLoader.DefaultArguments.Separator,
bool allowQuotedStrings = TextLoader.DefaultArguments.AllowQuoting,
bool supportSparse = TextLoader.DefaultArguments.AllowSparse,
bool trimWhitespace = TextLoader.DefaultArguments.TrimWhitespace)
bool hasHeader = TextLoader.Defaults.HasHeader,
char separatorChar = TextLoader.Defaults.Separator,
bool allowQuotedStrings = TextLoader.Defaults.AllowQuoting,
bool supportSparse = TextLoader.Defaults.AllowSparse,
bool trimWhitespace = TextLoader.Defaults.TrimWhitespace)
{
Contracts.CheckNonEmpty(path, nameof(path));

Expand Down Expand Up @@ -147,8 +147,8 @@ public static IDataView ReadFromTextFile(this DataOperationsCatalog catalog, str
public static void SaveAsText(this DataOperationsCatalog catalog,
IDataView data,
Stream stream,
char separatorChar = TextLoader.DefaultArguments.Separator,
bool headerRow = TextLoader.DefaultArguments.HasHeader,
char separatorChar = TextLoader.Defaults.Separator,
bool headerRow = TextLoader.Defaults.HasHeader,
bool schema = true,
bool keepHidden = false)
{
Expand Down
14 changes: 7 additions & 7 deletions src/Microsoft.ML.StaticPipe/LocalPathReader.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,20 +2,20 @@
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

using System.Data;
using Microsoft.ML.Data;

namespace Microsoft.ML.StaticPipe
{
public static class LocalPathReader
{
public static DataView<TShape> Read<TShape>(this DataReader<IMultiStreamSource, TShape> reader, string path)
{
return reader.Read(new MultiFileSource(path));
}

/// <summary>
/// Reads data from one or more file <paramref name="path"/> into an <see cref="DataView"/>.
/// </summary>
/// <param name="reader">The reader to use.</param>
/// <param name="path">One or more paths from which to load data.</param>
public static DataView<TShape> Read<TShape>(this DataReader<IMultiStreamSource, TShape> reader, params string[] path)
{
return reader.Read(new MultiFileSource(path));
}
=> reader.Read(new MultiFileSource(path));
}
}
72 changes: 36 additions & 36 deletions test/BaselineOutput/Common/EntryPoints/core_manifest.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -316,42 +316,6 @@
"IsNullable": false,
"Default": null
},
{
"Name": "UseThreads",
"Type": "Bool",
"Desc": "Use separate parsing threads?",
"Aliases": [
"threads"
],
"Required": false,
"SortOrder": 150.0,
"IsNullable": false,
"Default": true
},
{
"Name": "HeaderFile",
"Type": "String",
"Desc": "File containing a header with feature names. If specified, header defined in the data file (header+) is ignored.",
"Aliases": [
"hf"
],
"Required": false,
"SortOrder": 150.0,
"IsNullable": false,
"Default": null
},
{
"Name": "MaxRows",
"Type": "Int",
"Desc": "Maximum number of rows to produce",
"Aliases": [
"rows"
],
"Required": false,
"SortOrder": 150.0,
"IsNullable": true,
"Default": null
},
{
"Name": "AllowQuoting",
"Type": "Bool",
Expand Down Expand Up @@ -428,6 +392,42 @@
"SortOrder": 150.0,
"IsNullable": false,
"Default": false
},
{
"Name": "UseThreads",
"Type": "Bool",
"Desc": "Use separate parsing threads?",
"Aliases": [
"threads"
],
"Required": false,
"SortOrder": 150.0,
"IsNullable": false,
"Default": true
},
{
"Name": "HeaderFile",
"Type": "String",
"Desc": "File containing a header with feature names. If specified, header defined in the data file (header+) is ignored.",
"Aliases": [
"hf"
],
"Required": false,
"SortOrder": 150.0,
"IsNullable": false,
"Default": null
},
{
"Name": "MaxRows",
"Type": "Int",
"Desc": "Maximum number of rows to produce",
"Aliases": [
"rows"
],
"Required": false,
"SortOrder": 150.0,
"IsNullable": true,
"Default": null
}
]
},
Expand Down
2 changes: 1 addition & 1 deletion test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -463,7 +463,7 @@ public void EntryPointCreateEnsemble()
MetadataUtils.Const.ScoreValueKind.Score)
).Transform(individualScores[i]);

individualScores[i] = ColumnSelectingTransformer.CreateDrop(Env, individualScores[i], MetadataUtils.Const.ScoreValueKind.Score);
individualScores[i] = new ColumnSelectingTransformer(Env, null, new[] { MetadataUtils.Const.ScoreValueKind.Score }).Transform(individualScores[i]);
}

var avgEnsembleInput = new EnsembleCreator.ClassifierInput { Models = predictorModels, ModelCombiner = EnsembleCreator.ClassifierCombiner.Average };
Expand Down
4 changes: 2 additions & 2 deletions test/Microsoft.ML.Predictor.Tests/TestPredictors.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1806,7 +1806,7 @@ public void CompareSvmPredictorResultsToLibSvm()
{
using (var env = new LocalEnvironment(1, conc: 1))
{
IDataView trainView = new TextLoader(env, new TextLoader.Arguments(), new MultiFileSource(GetDataPath(TestDatasets.mnistOneClass.trainFilename)));
IDataView trainView = new TextLoader(env, new TextLoader.Options(), new MultiFileSource(GetDataPath(TestDatasets.mnistOneClass.trainFilename)));
trainView =
NormalizeTransform.Create(env,
new NormalizeTransform.MinMaxArguments()
Expand All @@ -1815,7 +1815,7 @@ public void CompareSvmPredictorResultsToLibSvm()
},
trainView);
var trainData = new RoleMappedData(trainView, "Label", "Features");
IDataView testView = new TextLoader(env, new TextLoader.Arguments(), new MultiFileSource(GetDataPath(TestDatasets.mnistOneClass.testFilename)));
IDataView testView = new TextLoader(env, new TextLoader.Options(), new MultiFileSource(GetDataPath(TestDatasets.mnistOneClass.testFilename)));
ApplyTransformUtils.ApplyAllTransformsToData(env, trainView, testView);
var testData = new RoleMappedData(testView, "Label", "Features");

Expand Down