Skip to content

Commit 2f7052c

Browse files
TomFinleyTomFinley
committed
Remove older non-MLContext based skip/take convenience API.
1 parent cf249f0 commit 2f7052c

19 files changed

+149
-225
lines changed

‎src/Microsoft.ML.Data/Transforms/SkipTakeFilter.cs

Lines changed: 0 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -287,17 +287,4 @@ protected override bool MoveNextCore()
287287
}
288288
}
289289
}
290-
291-
public static class TakeFilter
292-
{
293-
/// <summary>
294-
/// A helper method to create <see cref="SkipTakeFilter"/> transform by taking the top rows defined by the <paramref name="count"/> parameter.
295-
/// <see cref="SkipTakeFilter"/> when created with <see cref="SkipTakeFilter.TakeArguments"/> behaves as 'TakeFilter'.
296-
/// </summary>
297-
/// <param name="env">Host Environment.</param>
298-
/// <param name="input">>Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
299-
/// <param name="count">Number of rows to take</param>
300-
public static IDataView Create(IHostEnvironment env, IDataView input, long count = SkipTakeFilter.Arguments.DefaultTake)
301-
=> SkipTakeFilter.Create(env, new SkipTakeFilter.TakeArguments() { Count = count }, input);
302-
}
303290
}

‎test/BaselineOutput/SingleDebug/Rff/featurized.tsv

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
#@ col=RffVectorFloat:R4:9-14
77
#@ }
88
15 8:Label
9-
5 1 1 1 2 1 3 1 0 0.3418174 -0.465289354 0.247575819 -0.5215741 0.576975167 -0.02080728
10-
5 4 4 5 7 10 3 2 0 0.566368 -0.112074189 0.169326738 -0.5519618 0.517083168 -0.2568235
11-
3 1 1 1 2 2 3 1 0 0.497458071 -0.293033749 0.419352561 -0.39683342 0.572824 -0.07215268
12-
6 8 8 1 3 4 3 7 0 -0.382303447 -0.432640046 0.289225727 -0.4996817 0.571184337 -0.08415316
9+
5 1 1 1 2 1 3 1 0 -0.157029659 -0.555585265 0.490177631 -0.305056125 0.35670203 -0.453979075
10+
5 4 4 5 7 10 3 2 0 -0.375955045 0.43816793 -0.5670244 0.108704455 -0.271485656 -0.5095379
11+
3 1 1 1 2 2 3 1 0 -0.08380841 -0.571235061 0.4856296 -0.312245429 0.389987826 -0.4257262
12+
6 8 8 1 3 4 3 7 0 -0.2813567 0.504154444 -0.266616732 -0.512102365 -0.5723418 -0.07588247

‎test/BaselineOutput/SingleRelease/Rff/featurized.tsv

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
#@ col=RffVectorFloat:R4:9-14
77
#@ }
88
15 8:Label
9-
5 1 1 1 2 1 3 1 0 0.3418174 -0.465289354 0.247575819 -0.5215741 0.576975167 -0.02080728
10-
5 4 4 5 7 10 3 2 0 0.566368 -0.112074189 0.169326738 -0.5519618 0.517083168 -0.2568235
11-
3 1 1 1 2 2 3 1 0 0.497458071 -0.293033749 0.419352561 -0.39683342 0.572824 -0.07215268
12-
6 8 8 1 3 4 3 7 0 -0.382303447 -0.432640046 0.289225727 -0.4996817 0.571184337 -0.08415316
9+
5 1 1 1 2 1 3 1 0 -0.157029659 -0.555585265 0.490177631 -0.305056125 0.35670203 -0.453979075
10+
5 4 4 5 7 10 3 2 0 -0.375955045 0.43816793 -0.5670244 0.108704455 -0.271485656 -0.5095379
11+
3 1 1 1 2 2 3 1 0 -0.08380841 -0.571235061 0.4856296 -0.312245429 0.389987826 -0.4257262
12+
6 8 8 1 3 4 3 7 0 -0.2813567 0.504154444 -0.266616732 -0.512102365 -0.5723418 -0.07588247

‎test/Microsoft.ML.StaticPipelineTesting/StaticPipeTests.cs

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -387,11 +387,11 @@ public void Normalizer()
387387
[Fact]
388388
public void NormalizerWithOnFit()
389389
{
390-
var env = new MLContext(0);
390+
var ml = new MLContext(0);
391391
var dataPath = GetDataPath("generated_regression_dataset.csv");
392392
var dataSource = new MultiFileSource(dataPath);
393393

394-
var reader = TextLoaderStatic.CreateReader(env,
394+
var reader = TextLoaderStatic.CreateReader(ml,
395395
c => c.LoadFloat(0, 2),
396396
separator: ';', hasHeader: true);
397397
var data = reader.Read(dataSource);
@@ -415,9 +415,9 @@ public void NormalizerWithOnFit()
415415
// Just for fun, let's also write out some of the lines of the data to the console.
416416
using (var stream = new MemoryStream())
417417
{
418-
IDataView v = ColumnSelectingTransformer.CreateKeep(env, tdata.AsDynamic, new[] { "r", "ncdf", "n", "b" });
419-
v = TakeFilter.Create(env, v, 10);
420-
var saver = new TextSaver(env, new TextSaver.Arguments()
418+
IDataView v = ColumnSelectingTransformer.CreateKeep(ml, tdata.AsDynamic, new[] { "r", "ncdf", "n", "b" });
419+
v = ml.Data.TakeRows(v, 10);
420+
var saver = new TextSaver(ml, new TextSaver.Arguments()
421421
{
422422
Dense = true,
423423
Separator = ",",
@@ -770,10 +770,10 @@ public void PrincipalComponentAnalysis()
770770
[Fact]
771771
public void NAIndicatorStatic()
772772
{
773-
var env = new MLContext(0);
773+
var ml = new MLContext(0);
774774

775775
string dataPath = GetDataPath("breast-cancer.txt");
776-
var reader = TextLoaderStatic.CreateReader(env, ctx => (
776+
var reader = TextLoaderStatic.CreateReader(ml, ctx => (
777777
ScalarFloat: ctx.LoadFloat(1),
778778
ScalarDouble: ctx.LoadDouble(1),
779779
VectorFloat: ctx.LoadFloat(1, 4),
@@ -790,12 +790,12 @@ public void NAIndicatorStatic()
790790
D: row.VectorDoulbe.IsMissingValue()
791791
));
792792

793-
IDataView newData = TakeFilter.Create(env, est.Fit(data).Transform(data).AsDynamic, 4);
793+
IDataView newData = ml.Data.TakeRows(est.Fit(data).Transform(data).AsDynamic, 4);
794794
Assert.NotNull(newData);
795-
bool[] ScalarFloat = newData.GetColumn<bool>(env, "A").ToArray();
796-
bool[] ScalarDouble = newData.GetColumn<bool>(env, "B").ToArray();
797-
bool[][] VectorFloat = newData.GetColumn<bool[]>(env, "C").ToArray();
798-
bool[][] VectorDoulbe = newData.GetColumn<bool[]>(env, "D").ToArray();
795+
bool[] ScalarFloat = newData.GetColumn<bool>(ml, "A").ToArray();
796+
bool[] ScalarDouble = newData.GetColumn<bool>(ml, "B").ToArray();
797+
bool[][] VectorFloat = newData.GetColumn<bool[]>(ml, "C").ToArray();
798+
bool[][] VectorDoulbe = newData.GetColumn<bool[]>(ml, "D").ToArray();
799799

800800
Assert.NotNull(ScalarFloat);
801801
Assert.NotNull(ScalarDouble);

‎test/Microsoft.ML.Tests/FeatureContributionTests.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -199,7 +199,7 @@ private void TestFeatureContribution(
199199
using (var ch = Env.Start("save"))
200200
{
201201
var saver = new TextSaver(ML, new TextSaver.Arguments { Silent = true, OutputHeader = false });
202-
IDataView savedData = TakeFilter.Create(ML, est.Fit(data).Transform(data), 4);
202+
var savedData = ML.Data.TakeRows(est.Fit(data).Transform(data), 4);
203203
using (var fs = File.Create(outputPath))
204204
DataSaverUtils.SaveDataView(ch, saver, savedData, fs, keepHidden: true);
205205
}

‎test/Microsoft.ML.Tests/TermEstimatorTests.cs

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ void TestDifferentTypes()
5555
{
5656
string dataPath = GetDataPath("adult.tiny.with-schema.txt");
5757

58-
var loader = new TextLoader(Env, new TextLoader.Arguments
58+
var loader = new TextLoader(ML, new TextLoader.Arguments
5959
{
6060
Columns = new[]{
6161
new TextLoader.Column("float1", DataKind.R4, 9),
@@ -70,7 +70,7 @@ void TestDifferentTypes()
7070
HasHeader = true
7171
}, new MultiFileSource(dataPath));
7272

73-
var pipe = new ValueToKeyMappingEstimator(Env, new[]{
73+
var pipe = new ValueToKeyMappingEstimator(ML, new[]{
7474
new ValueToKeyMappingEstimator.ColumnInfo("TermFloat1", "float1"),
7575
new ValueToKeyMappingEstimator.ColumnInfo("TermFloat4", "float4"),
7676
new ValueToKeyMappingEstimator.ColumnInfo("TermDouble1", "double1"),
@@ -80,11 +80,11 @@ void TestDifferentTypes()
8080
new ValueToKeyMappingEstimator.ColumnInfo("TermText2", "text2")
8181
});
8282
var data = loader.Read(dataPath);
83-
data = TakeFilter.Create(Env, data, 10);
83+
data = ML.Data.TakeRows(data, 10);
8484
var outputPath = GetOutputPath("Term", "Term.tsv");
8585
using (var ch = Env.Start("save"))
8686
{
87-
var saver = new TextSaver(Env, new TextSaver.Arguments { Silent = true });
87+
var saver = new TextSaver(ML, new TextSaver.Arguments { Silent = true });
8888
using (var fs = File.Create(outputPath))
8989
DataSaverUtils.SaveDataView(ch, saver, pipe.Fit(data).Transform(data), fs, keepHidden: true);
9090
}

‎test/Microsoft.ML.Tests/Transformers/CategoricalHashTests.cs

Lines changed: 7 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ public void CategoricalHashWorkout()
6666
public void CategoricalHashStatic()
6767
{
6868
string dataPath = GetDataPath("breast-cancer.txt");
69-
var reader = TextLoaderStatic.CreateReader(Env, ctx => (
69+
var reader = TextLoaderStatic.CreateReader(ML, ctx => (
7070
ScalarString: ctx.LoadText(1),
7171
VectorString: ctx.LoadText(1, 4)));
7272
var data = reader.Read(dataPath);
@@ -91,14 +91,10 @@ public void CategoricalHashStatic()
9191
TestEstimatorCore(est.AsDynamic, data.AsDynamic, invalidInput: invalidData);
9292

9393
var outputPath = GetOutputPath("CategoricalHash", "featurized.tsv");
94-
using (var ch = Env.Start("save"))
95-
{
96-
var saver = new TextSaver(Env, new TextSaver.Arguments { Silent = true });
97-
var savedData = TakeFilter.Create(Env, est.Fit(data).Transform(data).AsDynamic, 4);
98-
var view = ColumnSelectingTransformer.CreateKeep(Env, savedData, new[] { "A", "B", "C", "D", "E", "F" });
99-
using (var fs = File.Create(outputPath))
100-
DataSaverUtils.SaveDataView(ch, saver, view, fs, keepHidden: true);
101-
}
94+
var savedData = ML.Data.TakeRows(est.Fit(data).Transform(data).AsDynamic, 4);
95+
var view = ColumnSelectingTransformer.CreateKeep(ML, savedData, new[] { "A", "B", "C", "D", "E", "F" });
96+
using (var fs = File.Create(outputPath))
97+
ML.Data.SaveAsText(view, fs, headerRow: true, keepHidden: true);
10298

10399
CheckEquality("CategoricalHash", "featurized.tsv");
104100
Done();
@@ -130,7 +126,6 @@ public void TestMetadataPropagation()
130126
Done();
131127
}
132128

133-
134129
private void ValidateMetadata(IDataView result)
135130
{
136131
VBuffer<ReadOnlyMemory<char>> slots = default;
@@ -224,9 +219,10 @@ public void TestOldSavingAndLoading()
224219
});
225220
var result = pipe.Fit(dataView).Transform(dataView);
226221
var resultRoles = new RoleMappedData(result);
222+
using (var ch = Env.Start("saving"))
227223
using (var ms = new MemoryStream())
228224
{
229-
TrainUtils.SaveModel(Env, Env.Start("saving"), ms, null, resultRoles);
225+
TrainUtils.SaveModel(Env, ch, ms, null, resultRoles);
230226
ms.Position = 0;
231227
var loadedView = ModelFileUtils.LoadTransforms(Env, dataView, ms);
232228
}

‎test/Microsoft.ML.Tests/Transformers/CategoricalTests.cs

Lines changed: 5 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -141,7 +141,7 @@ public void CategoricalOneHotEncodingFromSideData()
141141
public void CategoricalStatic()
142142
{
143143
string dataPath = GetDataPath("breast-cancer.txt");
144-
var reader = TextLoaderStatic.CreateReader(Env, ctx => (
144+
var reader = TextLoaderStatic.CreateReader(ML, ctx => (
145145
ScalarString: ctx.LoadText(1),
146146
VectorString: ctx.LoadText(1, 4)));
147147
var data = reader.Read(dataPath);
@@ -160,14 +160,10 @@ public void CategoricalStatic()
160160
TestEstimatorCore(est.AsDynamic, data.AsDynamic, invalidInput: invalidData);
161161

162162
var outputPath = GetOutputPath("Categorical", "featurized.tsv");
163-
using (var ch = Env.Start("save"))
164-
{
165-
var saver = new TextSaver(Env, new TextSaver.Arguments { Silent = true });
166-
var savedData = TakeFilter.Create(Env, est.Fit(data).Transform(data).AsDynamic, 4);
167-
var view = new ColumnSelectingTransformer(Env, new string[] { "A", "B", "C", "D", "E" }, null, false).Transform(savedData);
168-
using (var fs = File.Create(outputPath))
169-
DataSaverUtils.SaveDataView(ch, saver, view, fs, keepHidden: true);
170-
}
163+
var savedData = ML.Data.TakeRows(est.Fit(data).Transform(data).AsDynamic, 4);
164+
var view = new ColumnSelectingTransformer(ML, new string[] { "A", "B", "C", "D", "E" }, null, false).Transform(savedData);
165+
using (var fs = File.Create(outputPath))
166+
ML.Data.SaveAsText(view, fs, headerRow: true, keepHidden: true);
171167

172168
CheckEquality("Categorical", "featurized.tsv");
173169
Done();

‎test/Microsoft.ML.Tests/Transformers/ConcatTests.cs

Lines changed: 14 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ void TestConcat()
2626
string dataPath = GetDataPath("adult.tiny.with-schema.txt");
2727

2828
var source = new MultiFileSource(dataPath);
29-
var loader = new TextLoader(Env, new TextLoader.Arguments
29+
var loader = new TextLoader(ML, new TextLoader.Arguments
3030
{
3131
Columns = new[]{
3232
new TextLoader.Column("float1", DataKind.R4, 9),
@@ -44,12 +44,13 @@ ColumnType GetType(Schema schema, string name)
4444
Assert.True(schema.TryGetColumnIndex(name, out int cIdx), $"Could not find '{name}'");
4545
return schema[cIdx].Type;
4646
}
47-
var pipe = new ColumnConcatenatingEstimator(Env, "f1", "float1")
48-
.Append(new ColumnConcatenatingEstimator(Env, "f2", "float1", "float1"))
49-
.Append(new ColumnConcatenatingEstimator(Env, "f3", "float4", "float1"))
50-
.Append(new ColumnConcatenatingEstimator(Env, "f4", "float6", "vfloat", "float1"));
5147

52-
data = TakeFilter.Create(Env, data, 10);
48+
var pipe = ML.Transforms.Concatenate("f1", "float1")
49+
.Append(ML.Transforms.Concatenate("f2", "float1", "float1"))
50+
.Append(ML.Transforms.Concatenate("f3", "float4", "float1"))
51+
.Append(ML.Transforms.Concatenate("f4", "float6", "vfloat", "float1"));
52+
53+
data = ML.Data.TakeRows(data, 10);
5354
data = pipe.Fit(data).Transform(data);
5455

5556
ColumnType t;
@@ -62,13 +63,13 @@ ColumnType GetType(Schema schema, string name)
6263
t = GetType(data.Schema, "f4");
6364
Assert.True(t is VectorType vt4 && vt4.ItemType == NumberType.R4 && vt4.Size == 0);
6465

65-
data = ColumnSelectingTransformer.CreateKeep(Env, data, new[] { "f1", "f2", "f3", "f4" });
66+
data = ML.Transforms.SelectColumns("f1", "f2", "f3", "f4").Fit(data).Transform(data);
6667

6768
var subdir = Path.Combine("Transform", "Concat");
6869
var outputPath = GetOutputPath(subdir, "Concat1.tsv");
6970
using (var ch = Env.Start("save"))
7071
{
71-
var saver = new TextSaver(Env, new TextSaver.Arguments { Silent = true, Dense = true });
72+
var saver = new TextSaver(ML, new TextSaver.Arguments { Silent = true, Dense = true });
7273
using (var fs = File.Create(outputPath))
7374
DataSaverUtils.SaveDataView(ch, saver, data, fs, keepHidden: false);
7475
}
@@ -83,7 +84,7 @@ public void ConcatWithAliases()
8384
string dataPath = GetDataPath("adult.tiny.with-schema.txt");
8485

8586
var source = new MultiFileSource(dataPath);
86-
var loader = new TextLoader(Env, new TextLoader.Arguments
87+
var loader = new TextLoader(ML, new TextLoader.Arguments
8788
{
8889
Columns = new[]{
8990
new TextLoader.Column("float1", DataKind.R4, 9),
@@ -101,9 +102,9 @@ ColumnType GetType(Schema schema, string name)
101102
return schema[cIdx].Type;
102103
}
103104

104-
data = TakeFilter.Create(Env, data, 10);
105+
data = ML.Data.TakeRows(data, 10);
105106

106-
var concater = new ColumnConcatenatingTransformer(Env,
107+
var concater = new ColumnConcatenatingTransformer(ML,
107108
new ColumnConcatenatingTransformer.ColumnInfo("f2", new[] { ("float1", "FLOAT1"), ("float1", "FLOAT2") }),
108109
new ColumnConcatenatingTransformer.ColumnInfo("f3", new[] { ("float4", "FLOAT4"), ("float1", "FLOAT1") }));
109110
data = concater.Transform(data);
@@ -126,13 +127,13 @@ ColumnType GetType(Schema schema, string name)
126127
t = GetType(data.Schema, "f3");
127128
Assert.True(t is VectorType vt3 && vt3.ItemType == NumberType.R4 && vt3.Size == 5);
128129

129-
data = ColumnSelectingTransformer.CreateKeep(Env, data, new[] { "f2", "f3" });
130+
data = ColumnSelectingTransformer.CreateKeep(ML, data, new[] { "f2", "f3" });
130131

131132
var subdir = Path.Combine("Transform", "Concat");
132133
var outputPath = GetOutputPath(subdir, "Concat2.tsv");
133134
using (var ch = Env.Start("save"))
134135
{
135-
var saver = new TextSaver(Env, new TextSaver.Arguments { Silent = true, Dense = true });
136+
var saver = new TextSaver(ML, new TextSaver.Arguments { Silent = true, Dense = true });
136137
using (var fs = File.Create(outputPath))
137138
DataSaverUtils.SaveDataView(ch, saver, data, fs, keepHidden: false);
138139
}

‎test/Microsoft.ML.Tests/Transformers/ConvertTests.cs

Lines changed: 7 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -8,11 +8,9 @@
88
using Microsoft.Data.DataView;
99
using Microsoft.ML.Core.Data;
1010
using Microsoft.ML.Data;
11-
using Microsoft.ML.Data.IO;
1211
using Microsoft.ML.Model;
1312
using Microsoft.ML.RunTests;
1413
using Microsoft.ML.Tools;
15-
using Microsoft.ML.Transforms;
1614
using Microsoft.ML.Transforms.Categorical;
1715
using Microsoft.ML.Transforms.Conversions;
1816
using Xunit;
@@ -78,7 +76,7 @@ public void TestConvertWorkout()
7876
var data = new[] { new TestClass() { A = 1, B = new int[2] { 1,4 } },
7977
new TestClass() { A = 2, B = new int[2] { 3,4 } }};
8078
var dataView = ML.Data.ReadFromEnumerable(data);
81-
var pipe = new TypeConvertingEstimator(Env, columns: new[] {new TypeConvertingTransformer.ColumnInfo("ConvA", DataKind.R4, "A"),
79+
var pipe = new TypeConvertingEstimator(ML, columns: new[] {new TypeConvertingTransformer.ColumnInfo("ConvA", DataKind.R4, "A"),
8280
new TypeConvertingTransformer.ColumnInfo("ConvB", DataKind.R4, "B")});
8381

8482
TestEstimatorCore(pipe, dataView);
@@ -117,7 +115,7 @@ public void TestConvertWorkout()
117115
};
118116

119117
var allTypesDataView = ML.Data.ReadFromEnumerable(allTypesData);
120-
var allTypesPipe = new TypeConvertingEstimator(Env, columns: new[] {
118+
var allTypesPipe = new TypeConvertingEstimator(ML, columns: new[] {
121119
new TypeConvertingTransformer.ColumnInfo("ConvA", DataKind.R4, "AA"),
122120
new TypeConvertingTransformer.ColumnInfo("ConvB", DataKind.R4, "AB"),
123121
new TypeConvertingTransformer.ColumnInfo("ConvC", DataKind.R4, "AC"),
@@ -134,13 +132,9 @@ public void TestConvertWorkout()
134132
TestEstimatorCore(allTypesPipe, allTypesDataView);
135133

136134
var outputPath = GetOutputPath("Convert", "Types.tsv");
137-
using (var ch = Env.Start("save"))
138-
{
139-
var saver = new TextSaver(Env, new TextSaver.Arguments { Silent = true });
140-
var savedData = TakeFilter.Create(Env, allTypesPipe.Fit(allTypesDataView).Transform(allTypesDataView), 2);
141-
using (var fs = File.Create(outputPath))
142-
DataSaverUtils.SaveDataView(ch, saver, savedData, fs, keepHidden: true);
143-
}
135+
var savedData = ML.Data.TakeRows(allTypesPipe.Fit(allTypesDataView).Transform(allTypesDataView), 2);
136+
using (var fs = File.Create(outputPath))
137+
ML.Data.SaveAsText(savedData, fs, headerRow: true, keepHidden: true);
144138

145139
CheckEquality("Convert", "Types.tsv");
146140
Done();
@@ -268,9 +262,9 @@ public void TypeConvertKeyBackCompatTest()
268262
using (var ch = Env.Start("load"))
269263
{
270264
using (var fs = File.OpenRead(modelPath))
271-
modelOld = ML.Model.Load(fs);
265+
modelOld = ML.Model.Load(fs);
272266
}
273-
var outDataOld = modelOld.Transform(dataView);
267+
var outDataOld = modelOld.Transform(dataView);
274268

275269
var modelNew = ML.Transforms.Conversion.ConvertType(new[] { new TypeConvertingTransformer.ColumnInfo("convertedKey",
276270
DataKind.U8, "key", new KeyCount(4)) }).Fit(dataView);

0 commit comments

Comments
 (0)