Skip to content

Commit 4365d98

Browse files
srsaggamDmitry-A
srsaggam
authored and
Dmitry-A
committed
changed CreateTextLoader to ReadFromTextFile method. (dotnet#140)
* changed textloader to readfromtextfile method * formatting
1 parent c802783 commit 4365d98

File tree

2 files changed

+78
-63
lines changed

2 files changed

+78
-63
lines changed

‎src/mlnet/Templates/MLCodeGen.cs

+47-34
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,6 @@ public virtual string TransformText()
3535
using Microsoft.ML;
3636
using Microsoft.ML.Core.Data;
3737
using Microsoft.ML.Data;
38-
using static Microsoft.ML.Data.TextLoader;
3938
using Microsoft.Data.DataView;
4039
");
4140
this.Write(this.ToStringHelper.ToStringWithCulture(GeneratedUsings));
@@ -70,28 +69,49 @@ static void Main(string[] args)
7069
7170
private static ITransformer BuildTrainEvaluateAndSaveModel(MLContext mlContext)
7271
{
73-
// Common data loading configuration
74-
TextLoader textLoader = GetTextLoader(mlContext);
75-
76-
IDataView trainingDataView = textLoader.Read(TrainDataPath);
77-
");
72+
// Data loading
73+
IDataView trainingDataView = mlContext.Data.ReadFromTextFile<SampleObservation>(
74+
path: TrainDataPath,
75+
hasHeader : ");
76+
this.Write(this.ToStringHelper.ToStringWithCulture(HasHeader.ToString().ToLowerInvariant()));
77+
this.Write(",\r\n separatorChar : \'");
78+
this.Write(this.ToStringHelper.ToStringWithCulture(Separator));
79+
this.Write("\',\r\n allowQuotedStrings : ");
80+
this.Write(this.ToStringHelper.ToStringWithCulture(AllowQuoting.ToString().ToLowerInvariant()));
81+
this.Write(",\r\n trimWhitespace : ");
82+
this.Write(this.ToStringHelper.ToStringWithCulture(TrimWhiteSpace.ToString().ToLowerInvariant()));
83+
this.Write(" ,\r\n supportSparse : ");
84+
this.Write(this.ToStringHelper.ToStringWithCulture(AllowSparse.ToString().ToLowerInvariant()));
85+
this.Write(");\r\n");
7886
if(!string.IsNullOrEmpty(TestPath)){
79-
this.Write(" IDataView testDataView = textLoader.Read(TestDataPath);\r\n");
87+
this.Write(" IDataView testDataView = mlContext.Data.ReadFromTextFile<SampleObserv" +
88+
"ation>(\r\n path: TestDataPath,\r\n " +
89+
" hasHeader : ");
90+
this.Write(this.ToStringHelper.ToStringWithCulture(HasHeader.ToString().ToLowerInvariant()));
91+
this.Write(",\r\n separatorChar : \'");
92+
this.Write(this.ToStringHelper.ToStringWithCulture(Separator));
93+
this.Write("\',\r\n allowQuotedStrings : ");
94+
this.Write(this.ToStringHelper.ToStringWithCulture(AllowQuoting.ToString().ToLowerInvariant()));
95+
this.Write(",\r\n trimWhitespace : ");
96+
this.Write(this.ToStringHelper.ToStringWithCulture(TrimWhiteSpace.ToString().ToLowerInvariant()));
97+
this.Write(" ,\r\n supportSparse : ");
98+
this.Write(this.ToStringHelper.ToStringWithCulture(AllowSparse.ToString().ToLowerInvariant()));
99+
this.Write(");\r\n");
80100
}
81101
this.Write("\r\n");
82102
if(Transforms.Count >0 ) {
83103
this.Write(" // Common data process configuration with pipeline data transformatio" +
84104
"ns \r\n\r\n var dataProcessPipeline = ");
85105
for(int i=0;i<Transforms.Count;i++)
86-
{
87-
if(i>0)
88-
{ Write("\n .Append(");
89-
}
90-
Write("mlContext.Transforms."+Transforms[i]);
91-
if(i>0)
92-
{ Write(")");
93-
}
94-
}
106+
{
107+
if(i>0)
108+
{ Write("\n .Append(");
109+
}
110+
Write("mlContext.Transforms."+Transforms[i]);
111+
if(i>0)
112+
{ Write(")");
113+
}
114+
}
95115
this.Write(";\r\n");
96116
}
97117
this.Write("\r\n // Set the training algorithm, then create and config the modelBuil" +
@@ -157,30 +177,23 @@ private static ITransformer BuildTrainEvaluateAndSaveModel(MLContext mlContext)
157177
return trainedModel;
158178
}
159179
160-
private static TextLoader GetTextLoader(MLContext mlContext)
180+
// (OPTIONAL) Try/test a single prediction by loding the model from the file, first.
181+
private static void TestSinglePrediction(MLContext mlContext)
161182
{
162-
return mlContext.Data.CreateTextLoader<SampleObservation>(
163-
hasHeader : ");
183+
//Load data to test. Could be any test data. For demonstration purpose train data is used here.
184+
IDataView trainingDataView = mlContext.Data.ReadFromTextFile<SampleObservation>(
185+
path: TrainDataPath,
186+
hasHeader : ");
164187
this.Write(this.ToStringHelper.ToStringWithCulture(HasHeader.ToString().ToLowerInvariant()));
165-
this.Write(",\r\n separatorChar : \'");
188+
this.Write(",\r\n separatorChar : \'");
166189
this.Write(this.ToStringHelper.ToStringWithCulture(Separator));
167-
this.Write("\',\r\n allowQuotedStrings : ");
190+
this.Write("\',\r\n allowQuotedStrings : ");
168191
this.Write(this.ToStringHelper.ToStringWithCulture(AllowQuoting.ToString().ToLowerInvariant()));
169-
this.Write(",\r\n trimWhitespace : ");
192+
this.Write(",\r\n trimWhitespace : ");
170193
this.Write(this.ToStringHelper.ToStringWithCulture(TrimWhiteSpace.ToString().ToLowerInvariant()));
171-
this.Write(" ,\r\n supportSparse : ");
194+
this.Write(" ,\r\n supportSparse : ");
172195
this.Write(this.ToStringHelper.ToStringWithCulture(AllowSparse.ToString().ToLowerInvariant()));
173-
this.Write(@"
174-
);
175-
}
176-
177-
// (OPTIONAL) Try/test a single prediction by loding the model from the file, first.
178-
private static void TestSinglePrediction(MLContext mlContext)
179-
{
180-
TextLoader textLoader = GetTextLoader(mlContext);
181-
182-
//Load data to test. Could be any test data. For demonstration purpose train data is used here.
183-
IDataView trainingDataView = textLoader.Read(TrainDataPath);
196+
this.Write(@");
184197
185198
var sample = mlContext.CreateEnumerable<SampleObservation>(trainingDataView, false).First();
186199

‎src/mlnet/Templates/MLCodeGen.tt

+31-29
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@ using System.Linq;
1313
using Microsoft.ML;
1414
using Microsoft.ML.Core.Data;
1515
using Microsoft.ML.Data;
16-
using static Microsoft.ML.Data.TextLoader;
1716
using Microsoft.Data.DataView;
1817
<#= GeneratedUsings #>
1918

@@ -48,27 +47,37 @@ namespace MlnetSample
4847

4948
private static ITransformer BuildTrainEvaluateAndSaveModel(MLContext mlContext)
5049
{
51-
// Common data loading configuration
52-
TextLoader textLoader = GetTextLoader(mlContext);
53-
54-
IDataView trainingDataView = textLoader.Read(TrainDataPath);
50+
// Data loading
51+
IDataView trainingDataView = mlContext.Data.ReadFromTextFile<SampleObservation>(
52+
path: TrainDataPath,
53+
hasHeader : <#= HasHeader.ToString().ToLowerInvariant() #>,
54+
separatorChar : '<#= Separator #>',
55+
allowQuotedStrings : <#= AllowQuoting.ToString().ToLowerInvariant() #>,
56+
trimWhitespace : <#= TrimWhiteSpace.ToString().ToLowerInvariant() #> ,
57+
supportSparse : <#= AllowSparse.ToString().ToLowerInvariant() #>);
5558
<# if(!string.IsNullOrEmpty(TestPath)){ #>
56-
IDataView testDataView = textLoader.Read(TestDataPath);
59+
IDataView testDataView = mlContext.Data.ReadFromTextFile<SampleObservation>(
60+
path: TestDataPath,
61+
hasHeader : <#= HasHeader.ToString().ToLowerInvariant() #>,
62+
separatorChar : '<#= Separator #>',
63+
allowQuotedStrings : <#= AllowQuoting.ToString().ToLowerInvariant() #>,
64+
trimWhitespace : <#= TrimWhiteSpace.ToString().ToLowerInvariant() #> ,
65+
supportSparse : <#= AllowSparse.ToString().ToLowerInvariant() #>);
5766
<# } #>
5867

5968
<# if(Transforms.Count >0 ) {#>
6069
// Common data process configuration with pipeline data transformations
6170

6271
var dataProcessPipeline = <# for(int i=0;i<Transforms.Count;i++)
63-
{
64-
if(i>0)
65-
{ Write("\n .Append(");
66-
}
67-
Write("mlContext.Transforms."+Transforms[i]);
68-
if(i>0)
69-
{ Write(")");
70-
}
71-
}#>;
72+
{
73+
if(i>0)
74+
{ Write("\n .Append(");
75+
}
76+
Write("mlContext.Transforms."+Transforms[i]);
77+
if(i>0)
78+
{ Write(")");
79+
}
80+
}#>;
7281
<#}#>
7382

7483
// Set the training algorithm, then create and config the modelBuilder
@@ -117,24 +126,17 @@ else{#>
117126
return trainedModel;
118127
}
119128

120-
private static TextLoader GetTextLoader(MLContext mlContext)
121-
{
122-
return mlContext.Data.CreateTextLoader<SampleObservation>(
123-
hasHeader : <#= HasHeader.ToString().ToLowerInvariant() #>,
124-
separatorChar : '<#= Separator #>',
125-
allowQuotedStrings : <#= AllowQuoting.ToString().ToLowerInvariant() #>,
126-
trimWhitespace : <#= TrimWhiteSpace.ToString().ToLowerInvariant() #> ,
127-
supportSparse : <#= AllowSparse.ToString().ToLowerInvariant() #>
128-
);
129-
}
130-
131129
// (OPTIONAL) Try/test a single prediction by loding the model from the file, first.
132130
private static void TestSinglePrediction(MLContext mlContext)
133131
{
134-
TextLoader textLoader = GetTextLoader(mlContext);
135-
136132
//Load data to test. Could be any test data. For demonstration purpose train data is used here.
137-
IDataView trainingDataView = textLoader.Read(TrainDataPath);
133+
IDataView trainingDataView = mlContext.Data.ReadFromTextFile<SampleObservation>(
134+
path: TrainDataPath,
135+
hasHeader : <#= HasHeader.ToString().ToLowerInvariant() #>,
136+
separatorChar : '<#= Separator #>',
137+
allowQuotedStrings : <#= AllowQuoting.ToString().ToLowerInvariant() #>,
138+
trimWhitespace : <#= TrimWhiteSpace.ToString().ToLowerInvariant() #> ,
139+
supportSparse : <#= AllowSparse.ToString().ToLowerInvariant() #>);
138140

139141
var sample = mlContext.CreateEnumerable<SampleObservation>(trainingDataView, false).First();
140142

0 commit comments

Comments
 (0)