15
15
using Microsoft . ML . Model ;
16
16
using Float = System . Single ;
17
17
18
- [ assembly: LoadableClass ( TextLoader . Summary , typeof ( IDataLoader ) , typeof ( TextLoader ) , typeof ( TextLoader . Arguments ) , typeof ( SignatureDataLoader ) ,
18
+ [ assembly: LoadableClass ( TextLoader . Summary , typeof ( IDataLoader ) , typeof ( TextLoader ) , typeof ( TextLoader . Options ) , typeof ( SignatureDataLoader ) ,
19
19
"Text Loader" , "TextLoader" , "Text" , DocName = "loader/TextLoader.md" ) ]
20
20
21
21
[ assembly: LoadableClass ( TextLoader . Summary , typeof ( IDataLoader ) , typeof ( TextLoader ) , null , typeof ( SignatureLoadDataLoader ) ,
@@ -379,7 +379,7 @@ public bool IsValid()
379
379
}
380
380
}
381
381
382
- public sealed class Arguments : ArgumentsCore
382
+ public sealed class Options : ArgumentsCore
383
383
{
384
384
[ Argument ( ArgumentType . AtMostOnce , HelpText = "Use separate parsing threads?" , ShortName = "threads" , Hide = true ) ]
385
385
public bool UseThreads = true ;
@@ -936,7 +936,7 @@ private static VersionInfo GetVersionInfo()
936
936
/// bumping the version number.
937
937
/// </summary>
938
938
[ Flags ]
939
- private enum Options : uint
939
+ private enum OptionFlags : uint
940
940
{
941
941
TrimWhitespace = 0x01 ,
942
942
HasHeader = 0x02 ,
@@ -950,7 +950,7 @@ private enum Options : uint
950
950
private const int SrcLim = int . MaxValue ;
951
951
952
952
private readonly bool _useThreads ;
953
- private readonly Options _flags ;
953
+ private readonly OptionFlags _flags ;
954
954
private readonly long _maxRows ;
955
955
// Input size is zero for unknown - determined by the data (including sparse rows).
956
956
private readonly int _inputSize ;
@@ -961,7 +961,7 @@ private enum Options : uint
961
961
962
962
private bool HasHeader
963
963
{
964
- get { return ( _flags & Options . HasHeader ) != 0 ; }
964
+ get { return ( _flags & OptionFlags . HasHeader ) != 0 ; }
965
965
}
966
966
967
967
private readonly IHost _host ;
@@ -980,38 +980,38 @@ public TextLoader(IHostEnvironment env, Column[] columns, bool hasHeader = false
980
980
{
981
981
}
982
982
983
- private static Arguments MakeArgs ( Column [ ] columns , bool hasHeader , char [ ] separatorChars )
983
+ private static Options MakeArgs ( Column [ ] columns , bool hasHeader , char [ ] separatorChars )
984
984
{
985
985
Contracts . AssertValue ( separatorChars ) ;
986
- var result = new Arguments { Columns = columns , HasHeader = hasHeader , Separators = separatorChars } ;
986
+ var result = new Options { Columns = columns , HasHeader = hasHeader , Separators = separatorChars } ;
987
987
return result ;
988
988
}
989
989
990
990
/// <summary>
991
991
/// Loads a text file into an <see cref="IDataView"/>. Supports basic mapping from input columns to IDataView columns.
992
992
/// </summary>
993
993
/// <param name="env">The environment to use.</param>
994
- /// <param name="args ">Defines the settings of the load operation.</param>
994
+ /// <param name="options ">Defines the settings of the load operation.</param>
995
995
/// <param name="dataSample">Allows to expose items that can be used for reading.</param>
996
- public TextLoader ( IHostEnvironment env , Arguments args = null , IMultiStreamSource dataSample = null )
996
+ public TextLoader ( IHostEnvironment env , Options options = null , IMultiStreamSource dataSample = null )
997
997
{
998
- args = args ?? new Arguments ( ) ;
998
+ options = options ?? new Options ( ) ;
999
999
1000
1000
Contracts . CheckValue ( env , nameof ( env ) ) ;
1001
1001
_host = env . Register ( RegistrationName ) ;
1002
- _host . CheckValue ( args , nameof ( args ) ) ;
1002
+ _host . CheckValue ( options , nameof ( options ) ) ;
1003
1003
_host . CheckValueOrNull ( dataSample ) ;
1004
1004
1005
1005
if ( dataSample == null )
1006
1006
dataSample = new MultiFileSource ( null ) ;
1007
1007
1008
1008
IMultiStreamSource headerFile = null ;
1009
- if ( ! string . IsNullOrWhiteSpace ( args . HeaderFile ) )
1010
- headerFile = new MultiFileSource ( args . HeaderFile ) ;
1009
+ if ( ! string . IsNullOrWhiteSpace ( options . HeaderFile ) )
1010
+ headerFile = new MultiFileSource ( options . HeaderFile ) ;
1011
1011
1012
- var cols = args . Columns ;
1012
+ var cols = options . Columns ;
1013
1013
bool error ;
1014
- if ( Utils . Size ( cols ) == 0 && ! TryParseSchema ( _host , headerFile ?? dataSample , ref args , out cols , out error ) )
1014
+ if ( Utils . Size ( cols ) == 0 && ! TryParseSchema ( _host , headerFile ?? dataSample , ref options , out cols , out error ) )
1015
1015
{
1016
1016
if ( error )
1017
1017
throw _host . Except ( "TextLoader options embedded in the file are invalid" ) ;
@@ -1026,43 +1026,43 @@ public TextLoader(IHostEnvironment env, Arguments args = null, IMultiStreamSourc
1026
1026
}
1027
1027
_host . Assert ( Utils . Size ( cols ) > 0 ) ;
1028
1028
1029
- _useThreads = args . UseThreads ;
1029
+ _useThreads = options . UseThreads ;
1030
1030
1031
- if ( args . TrimWhitespace )
1032
- _flags |= Options . TrimWhitespace ;
1033
- if ( headerFile == null && args . HasHeader )
1034
- _flags |= Options . HasHeader ;
1035
- if ( args . AllowQuoting )
1036
- _flags |= Options . AllowQuoting ;
1037
- if ( args . AllowSparse )
1038
- _flags |= Options . AllowSparse ;
1031
+ if ( options . TrimWhitespace )
1032
+ _flags |= OptionFlags . TrimWhitespace ;
1033
+ if ( headerFile == null && options . HasHeader )
1034
+ _flags |= OptionFlags . HasHeader ;
1035
+ if ( options . AllowQuoting )
1036
+ _flags |= OptionFlags . AllowQuoting ;
1037
+ if ( options . AllowSparse )
1038
+ _flags |= OptionFlags . AllowSparse ;
1039
1039
1040
1040
// REVIEW: This should be persisted (if it should be maintained).
1041
- _maxRows = args . MaxRows ?? long . MaxValue ;
1042
- _host . CheckUserArg ( _maxRows >= 0 , nameof ( args . MaxRows ) ) ;
1041
+ _maxRows = options . MaxRows ?? long . MaxValue ;
1042
+ _host . CheckUserArg ( _maxRows >= 0 , nameof ( options . MaxRows ) ) ;
1043
1043
1044
1044
// Note that _maxDim == 0 means sparsity is illegal.
1045
- _inputSize = args . InputSize ?? 0 ;
1045
+ _inputSize = options . InputSize ?? 0 ;
1046
1046
_host . Check ( _inputSize >= 0 , "inputSize" ) ;
1047
1047
if ( _inputSize >= SrcLim )
1048
1048
_inputSize = SrcLim - 1 ;
1049
1049
1050
- _host . CheckNonEmpty ( args . Separator , nameof ( args . Separator ) , "Must specify a separator" ) ;
1050
+ _host . CheckNonEmpty ( options . Separator , nameof ( options . Separator ) , "Must specify a separator" ) ;
1051
1051
1052
1052
//Default arg.Separator is tab and default args.Separators is also a '\t'.
1053
1053
//At a time only one default can be different and whichever is different that will
1054
1054
//be used.
1055
- if ( args . Separators . Length > 1 || args . Separators [ 0 ] != '\t ' )
1055
+ if ( options . Separators . Length > 1 || options . Separators [ 0 ] != '\t ' )
1056
1056
{
1057
1057
var separators = new HashSet < char > ( ) ;
1058
- foreach ( char c in args . Separators )
1058
+ foreach ( char c in options . Separators )
1059
1059
separators . Add ( NormalizeSeparator ( c . ToString ( ) ) ) ;
1060
1060
1061
1061
_separators = separators . ToArray ( ) ;
1062
1062
}
1063
1063
else
1064
1064
{
1065
- string sep = args . Separator . ToLowerInvariant ( ) ;
1065
+ string sep = options . Separator . ToLowerInvariant ( ) ;
1066
1066
if ( sep == "," )
1067
1067
_separators = new char [ ] { ',' } ;
1068
1068
else
@@ -1103,7 +1103,7 @@ private char NormalizeSeparator(string sep)
1103
1103
return ',' ;
1104
1104
case "colon" :
1105
1105
case ":" :
1106
- _host . CheckUserArg ( ( _flags & Options . AllowSparse ) == 0 , nameof ( Arguments . Separator ) ,
1106
+ _host . CheckUserArg ( ( _flags & OptionFlags . AllowSparse ) == 0 , nameof ( Options . Separator ) ,
1107
1107
"When the separator is colon, turn off allowSparse" ) ;
1108
1108
return ':' ;
1109
1109
case "semicolon" :
@@ -1115,7 +1115,7 @@ private char NormalizeSeparator(string sep)
1115
1115
default :
1116
1116
char ch = sep [ 0 ] ;
1117
1117
if ( sep . Length != 1 || ch < ' ' || '0' <= ch && ch <= '9' || ch == '"' )
1118
- throw _host . ExceptUserArg ( nameof ( Arguments . Separator ) , "Illegal separator: '{0}'" , sep ) ;
1118
+ throw _host . ExceptUserArg ( nameof ( Options . Separator ) , "Illegal separator: '{0}'" , sep ) ;
1119
1119
return sep [ 0 ] ;
1120
1120
}
1121
1121
}
@@ -1134,7 +1134,7 @@ private sealed class LoaderHolder
1134
1134
// If so, update args and set cols to the combined set of columns.
1135
1135
// If not, set error to true if there was an error condition.
1136
1136
private static bool TryParseSchema ( IHost host , IMultiStreamSource files ,
1137
- ref Arguments args , out Column [ ] cols , out bool error )
1137
+ ref Options options , out Column [ ] cols , out bool error )
1138
1138
{
1139
1139
host . AssertValue ( host ) ;
1140
1140
host . AssertValue ( files ) ;
@@ -1144,7 +1144,7 @@ private static bool TryParseSchema(IHost host, IMultiStreamSource files,
1144
1144
1145
1145
// Verify that the current schema-defining arguments are default.
1146
1146
// Get settings just for core arguments, not everything.
1147
- string tmp = CmdParser . GetSettings ( host , args , new ArgumentsCore ( ) ) ;
1147
+ string tmp = CmdParser . GetSettings ( host , options , new ArgumentsCore ( ) ) ;
1148
1148
1149
1149
// Try to get the schema information from the file.
1150
1150
string str = Cursor . GetEmbeddedArgs ( files ) ;
@@ -1176,12 +1176,12 @@ private static bool TryParseSchema(IHost host, IMultiStreamSource files,
1176
1176
1177
1177
// Make sure the loader binds to us.
1178
1178
var info = host . ComponentCatalog . GetLoadableClassInfo < SignatureDataLoader > ( loader . Name ) ;
1179
- if ( info . Type != typeof ( IDataLoader ) || info . ArgType != typeof ( Arguments ) )
1179
+ if ( info . Type != typeof ( IDataLoader ) || info . ArgType != typeof ( Options ) )
1180
1180
goto LDone ;
1181
1181
1182
- var argsNew = new Arguments ( ) ;
1182
+ var argsNew = new Options ( ) ;
1183
1183
// Copy the non-core arguments to the new args (we already know that all the core arguments are default).
1184
- var parsed = CmdParser . ParseArguments ( host , CmdParser . GetSettings ( host , args , new Arguments ( ) ) , argsNew ) ;
1184
+ var parsed = CmdParser . ParseArguments ( host , CmdParser . GetSettings ( host , options , new Options ( ) ) , argsNew ) ;
1185
1185
ch . Assert ( parsed ) ;
1186
1186
// Copy the core arguments to the new args.
1187
1187
if ( ! CmdParser . ParseArguments ( host , loader . GetSettingsString ( ) , argsNew , typeof ( ArgumentsCore ) , msg => ch . Error ( msg ) ) )
@@ -1192,7 +1192,7 @@ private static bool TryParseSchema(IHost host, IMultiStreamSource files,
1192
1192
goto LDone ;
1193
1193
1194
1194
error = false ;
1195
- args = argsNew ;
1195
+ options = argsNew ;
1196
1196
1197
1197
LDone :
1198
1198
return ! error ;
@@ -1202,16 +1202,16 @@ private static bool TryParseSchema(IHost host, IMultiStreamSource files,
1202
1202
/// <summary>
1203
1203
/// Checks whether the source contains the valid TextLoader.Arguments depiction.
1204
1204
/// </summary>
1205
- public static bool FileContainsValidSchema ( IHostEnvironment env , IMultiStreamSource files , out Arguments args )
1205
+ public static bool FileContainsValidSchema ( IHostEnvironment env , IMultiStreamSource files , out Options options )
1206
1206
{
1207
1207
Contracts . CheckValue ( env , nameof ( env ) ) ;
1208
1208
var h = env . Register ( RegistrationName ) ;
1209
1209
h . CheckValue ( files , nameof ( files ) ) ;
1210
- args = new Arguments ( ) ;
1210
+ options = new Options ( ) ;
1211
1211
Column [ ] cols ;
1212
1212
bool error ;
1213
- bool found = TryParseSchema ( h , files , ref args , out cols , out error ) ;
1214
- return found && ! error && args . IsValid ( ) ;
1213
+ bool found = TryParseSchema ( h , files , ref options , out cols , out error ) ;
1214
+ return found && ! error && options . IsValid ( ) ;
1215
1215
}
1216
1216
1217
1217
private TextLoader ( IHost host , ModelLoadContext ctx )
@@ -1236,8 +1236,8 @@ private TextLoader(IHost host, ModelLoadContext ctx)
1236
1236
host . CheckDecode ( cbFloat == sizeof ( Float ) ) ;
1237
1237
_maxRows = ctx . Reader . ReadInt64 ( ) ;
1238
1238
host . CheckDecode ( _maxRows > 0 ) ;
1239
- _flags = ( Options ) ctx . Reader . ReadUInt32 ( ) ;
1240
- host . CheckDecode ( ( _flags & ~ Options . All ) == 0 ) ;
1239
+ _flags = ( OptionFlags ) ctx . Reader . ReadUInt32 ( ) ;
1240
+ host . CheckDecode ( ( _flags & ~ OptionFlags . All ) == 0 ) ;
1241
1241
_inputSize = ctx . Reader . ReadInt32 ( ) ;
1242
1242
host . CheckDecode ( 0 <= _inputSize && _inputSize < SrcLim ) ;
1243
1243
@@ -1253,7 +1253,7 @@ private TextLoader(IHost host, ModelLoadContext ctx)
1253
1253
}
1254
1254
1255
1255
if ( _separators . Contains ( ':' ) )
1256
- host . CheckDecode ( ( _flags & Options . AllowSparse ) == 0 ) ;
1256
+ host . CheckDecode ( ( _flags & OptionFlags . AllowSparse ) == 0 ) ;
1257
1257
1258
1258
_bindings = new Bindings ( ctx , this ) ;
1259
1259
_parser = new Parser ( this ) ;
@@ -1273,14 +1273,14 @@ internal static TextLoader Create(IHostEnvironment env, ModelLoadContext ctx)
1273
1273
// These are legacy constructors needed for ComponentCatalog.
1274
1274
internal static IDataLoader Create ( IHostEnvironment env , ModelLoadContext ctx , IMultiStreamSource files )
1275
1275
=> ( IDataLoader ) Create ( env , ctx ) . Read ( files ) ;
1276
- internal static IDataLoader Create ( IHostEnvironment env , Arguments args , IMultiStreamSource files )
1277
- => ( IDataLoader ) new TextLoader ( env , args , files ) . Read ( files ) ;
1276
+ internal static IDataLoader Create ( IHostEnvironment env , Options options , IMultiStreamSource files )
1277
+ => ( IDataLoader ) new TextLoader ( env , options , files ) . Read ( files ) ;
1278
1278
1279
1279
/// <summary>
1280
1280
/// Convenience method to create a <see cref="TextLoader"/> and use it to read a specified file.
1281
1281
/// </summary>
1282
- internal static IDataView ReadFile ( IHostEnvironment env , Arguments args , IMultiStreamSource fileSource )
1283
- => new TextLoader ( env , args , fileSource ) . Read ( fileSource ) ;
1282
+ internal static IDataView ReadFile ( IHostEnvironment env , Options options , IMultiStreamSource fileSource )
1283
+ => new TextLoader ( env , options , fileSource ) . Read ( fileSource ) ;
1284
1284
1285
1285
void ICanSaveModel . Save ( ModelSaveContext ctx )
1286
1286
{
@@ -1298,7 +1298,7 @@ void ICanSaveModel.Save(ModelSaveContext ctx)
1298
1298
// bindings
1299
1299
ctx . Writer . Write ( sizeof ( Float ) ) ;
1300
1300
ctx . Writer . Write ( _maxRows ) ;
1301
- _host . Assert ( ( _flags & ~ Options . All ) == 0 ) ;
1301
+ _host . Assert ( ( _flags & ~ OptionFlags . All ) == 0 ) ;
1302
1302
ctx . Writer . Write ( ( uint ) _flags ) ;
1303
1303
_host . Assert ( 0 <= _inputSize && _inputSize < SrcLim ) ;
1304
1304
ctx . Writer . Write ( _inputSize ) ;
@@ -1367,7 +1367,7 @@ internal static TextLoader CreateTextReader<TInput>(IHostEnvironment host,
1367
1367
columns . Add ( column ) ;
1368
1368
}
1369
1369
1370
- Arguments args = new Arguments
1370
+ Options options = new Options
1371
1371
{
1372
1372
HasHeader = hasHeader ,
1373
1373
Separators = new [ ] { separator } ,
@@ -1377,7 +1377,7 @@ internal static TextLoader CreateTextReader<TInput>(IHostEnvironment host,
1377
1377
Columns = columns . ToArray ( )
1378
1378
} ;
1379
1379
1380
- return new TextLoader ( host , args ) ;
1380
+ return new TextLoader ( host , options ) ;
1381
1381
}
1382
1382
1383
1383
private sealed class BoundLoader : IDataLoader
0 commit comments