Skip to content

Commit 055abae

Browse files
author
Ivan Matantsev
committed
Merge branch 'master' into Ivanidze/KeyToVectorIORBug
2 parents 058eaac + 3b9d407 commit 055abae

File tree

334 files changed

+4749
-3243
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

334 files changed

+4749
-3243
lines changed

‎build/Dependencies.props

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@
88
<SystemMemoryVersion>4.5.1</SystemMemoryVersion>
99
<SystemReflectionEmitLightweightPackageVersion>4.3.0</SystemReflectionEmitLightweightPackageVersion>
1010
<SystemThreadingTasksDataflowPackageVersion>4.8.0</SystemThreadingTasksDataflowPackageVersion>
11-
<SystemComponentModelCompositionVersion>4.5.0</SystemComponentModelCompositionVersion>
1211
</PropertyGroup>
1312

1413
<!-- Other/Non-Core Product Dependencies -->

‎build/ci/phase-template.yml

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,10 @@ phases:
1414
_arch: ${{ parameters.architecture }}
1515
_codeCoverage: ${{ parameters.codeCoverage }}
1616
queue:
17-
timeoutInMinutes: 45
17+
${{ if eq(variables._codeCoverage, 'false') }}:
18+
timeoutInMinutes: 30
19+
${{ if eq(variables._codeCoverage, 'true') }}:
20+
timeoutInMinutes: 60
1821
parallel: 99
1922
matrix:
2023
${{ if eq(parameters.customMatrixes, '') }}:

‎docs/code/MlNetCookBook.md

Lines changed: 22 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -219,10 +219,10 @@ private class AdultData
219219

220220
// Read the data into a data view.
221221
var trainData = mlContext.Data.ReadFromTextFile<AdultData>(trainDataPath,
222-
// First line of the file is a header, not a data row.
223-
hasHeader: true,
224222
// Default separator is tab, but we need a semicolon.
225-
separatorChar: ';'
223+
separatorChar: ';',
224+
// First line of the file is a header, not a data row.
225+
hasHeader: true
226226
);
227227

228228
```
@@ -328,7 +328,7 @@ In the file above, the last column (12th) is label that we predict, and all the
328328
// First, we define the reader: specify the data columns and where to find them in the text file.
329329
// Read the data into a data view. Remember though, readers are lazy, so the actual reading will happen when the data is accessed.
330330
var trainData = mlContext.Data.ReadFromTextFile<AdultData>(dataPath,
331-
// First line of the file is a header, not a data row.
331+
// Default separator is tab, but the dataset has comma.
332332
separatorChar: ','
333333
);
334334

@@ -372,7 +372,7 @@ Assuming the example above was used to train the model, here's how you calculate
372372
```csharp
373373
// Read the test dataset.
374374
var testData = mlContext.Data.ReadFromTextFile<AdultData>(testDataPath,
375-
// First line of the file is a header, not a data row.
375+
// Default separator is tab, but the dataset has comma.
376376
separatorChar: ','
377377
);
378378
// Calculate metrics of the model on the test data.
@@ -959,7 +959,7 @@ public static ITransformer TrainModel(MLContext mlContext, IDataView trainData)
959959
// Construct the learning pipeline.
960960
var estimator = mlContext.Transforms.CustomMapping(mapping, null)
961961
.AppendCacheCheckpoint(mlContext)
962-
.Append(mlContext.BinaryClassification.Trainers.FastTree(label: "Label"));
962+
.Append(mlContext.BinaryClassification.Trainers.FastTree(labelColumnName: "Label"));
963963

964964
return estimator.Fit(trainData);
965965
}
@@ -970,35 +970,35 @@ Please note that you need to make your `mapping` operation into a 'pure function
970970
- It should not have side effects (we may call it arbitrarily at any time, or omit the call)
971971

972972
One important caveat is: if you want your custom transformation to be part of your saved model, you will need to provide a `contractName` for it.
973-
At loading time, you will need to reconstruct the custom transformer and inject it into MLContext.
973+
At loading time, you will need to register the custom transformer with the MLContext.
974974

975975
Here is a complete example that saves and loads a model with a custom mapping.
976976
```csharp
977977
/// <summary>
978-
/// One class that contains all custom mappings that we need for our model.
978+
/// One class that contains the custom mapping functionality that we need for our model.
979+
///
980+
/// It has a <see cref="CustomMappingFactoryAttributeAttribute"/> on it and
981+
/// derives from <see cref="CustomMappingFactory{TSrc, TDst}"/>.
979982
/// </summary>
980-
public class CustomMappings
983+
[CustomMappingFactoryAttribute(nameof(CustomMappings.IncomeMapping))]
984+
public class CustomMappings : CustomMappingFactory<InputRow, OutputRow>
981985
{
982986
// This is the custom mapping. We now separate it into a method, so that we can use it both in training and in loading.
983987
public static void IncomeMapping(InputRow input, OutputRow output) => output.Label = input.Income > 50000;
984988

985-
// MLContext is needed to create a new transformer. We are using 'Import' to have ML.NET populate
986-
// this property.
987-
[Import]
988-
public MLContext MLContext { get; set; }
989-
990-
// We are exporting the custom transformer by the name 'IncomeMapping'.
991-
[Export(nameof(IncomeMapping))]
992-
public ITransformer MyCustomTransformer
993-
=> MLContext.Transforms.CustomMappingTransformer<InputRow, OutputRow>(IncomeMapping, nameof(IncomeMapping));
989+
// This factory method will be called when loading the model to get the mapping operation.
990+
public override Action<InputRow, OutputRow> GetMapping()
991+
{
992+
return IncomeMapping;
993+
}
994994
}
995995
```
996996

997997
```csharp
998998
// Construct the learning pipeline. Note that we are now providing a contract name for the custom mapping:
999999
// otherwise we will not be able to save the model.
10001000
var estimator = mlContext.Transforms.CustomMapping<InputRow, OutputRow>(CustomMappings.IncomeMapping, nameof(CustomMappings.IncomeMapping))
1001-
.Append(mlContext.BinaryClassification.Trainers.FastTree(label: "Label"));
1001+
.Append(mlContext.BinaryClassification.Trainers.FastTree(labelColumnName: "Label"));
10021002

10031003
// If memory is enough, we can cache the data in-memory to avoid reading them from file
10041004
// when it will be accessed multiple times.
@@ -1013,8 +1013,9 @@ using (var fs = File.Create(modelPath))
10131013

10141014
// Now pretend we are in a different process.
10151015
1016-
// Create a custom composition container for all our custom mapping actions.
1017-
newContext.CompositionContainer = new CompositionContainer(new TypeCatalog(typeof(CustomMappings)));
1016+
// Register the assembly that contains 'CustomMappings' with the ComponentCatalog
1017+
// so it can be found when loading the model.
1018+
newContext.ComponentCatalog.RegisterAssembly(typeof(CustomMappings).Assembly);
10181019

10191020
// Now we can load the model.
10201021
ITransformer loadedModel;

‎docs/samples/Microsoft.ML.Samples/Dynamic/Calibrator.cs

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,8 @@ public static void Example()
3434
HasHeader = true,
3535
Columns = new[]
3636
{
37-
new TextLoader.Column("Sentiment", DataKind.BL, 0),
38-
new TextLoader.Column("SentimentText", DataKind.Text, 1)
37+
new TextLoader.Column("Sentiment", DataKind.Boolean, 0),
38+
new TextLoader.Column("SentimentText", DataKind.String, 1)
3939
}
4040
});
4141

@@ -50,8 +50,8 @@ public static void Example()
5050
// the "Features" column produced by FeaturizeText as the features column.
5151
var pipeline = mlContext.Transforms.Text.FeaturizeText("SentimentText", "Features")
5252
.Append(mlContext.BinaryClassification.Trainers.StochasticDualCoordinateAscentNonCalibrated(
53-
labelColumn: "Sentiment",
54-
featureColumn: "Features",
53+
labelColumnName: "Sentiment",
54+
featureColumnName: "Features",
5555
l2Const: 0.001f,
5656
loss: new HingeLoss())); // By specifying loss: new HingeLoss(), StochasticDualCoordinateAscent will train a support vector machine (SVM).
5757

‎docs/samples/Microsoft.ML.Samples/Dynamic/FastTreeRegression.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ public static void Example()
2929
// We will train a FastTreeRegression model with 1 tree on these two columns to predict Age.
3030
string outputColumnName = "Features";
3131
var pipeline = ml.Transforms.Concatenate(outputColumnName, new[] { "Parity", "Induced" })
32-
.Append(ml.Regression.Trainers.FastTree(labelColumn: "Age", featureColumn: outputColumnName, numTrees: 1, numLeaves: 2, minDatapointsInLeaves: 1));
32+
.Append(ml.Regression.Trainers.FastTree(labelColumnName: "Age", featureColumnName: outputColumnName, numTrees: 1, numLeaves: 2, minDatapointsInLeaves: 1));
3333

3434
var model = pipeline.Fit(trainData);
3535

‎docs/samples/Microsoft.ML.Samples/Dynamic/FeatureSelectionTransform.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,8 @@ public static void Example()
3333
var reader = ml.Data.CreateTextLoader(
3434
columns: new[]
3535
{
36-
new TextLoader.Column("Label", DataKind.BL, 0),
37-
new TextLoader.Column("Features", DataKind.Num, new [] { new TextLoader.Range(1, 9) })
36+
new TextLoader.Column("Label", DataKind.Boolean, 0),
37+
new TextLoader.Column("Features", DataKind.Single, new [] { new TextLoader.Range(1, 9) })
3838
},
3939
hasHeader: true
4040
);

‎docs/samples/Microsoft.ML.Samples/Dynamic/FieldAwareFactorizationMachine.cs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,8 @@ public static void Example()
2525
var reader = mlContext.Data.CreateTextLoader(
2626
columns: new[]
2727
{
28-
new TextLoader.Column("Sentiment", DataKind.BL, 0),
29-
new TextLoader.Column("SentimentText", DataKind.Text, 1)
28+
new TextLoader.Column("Sentiment", DataKind.Boolean, 0),
29+
new TextLoader.Column("SentimentText", DataKind.String, 1)
3030
},
3131
hasHeader: true
3232
);
@@ -46,7 +46,7 @@ public static void Example()
4646
// the "Features" column produced by FeaturizeText as the features column.
4747
var pipeline = mlContext.Transforms.Text.FeaturizeText("SentimentText", "Features")
4848
.AppendCacheCheckpoint(mlContext) // Add a data-cache step within a pipeline.
49-
.Append(mlContext.BinaryClassification.Trainers.FieldAwareFactorizationMachine(labelColumn: "Sentiment", featureColumns: new[] { "Features" }));
49+
.Append(mlContext.BinaryClassification.Trainers.FieldAwareFactorizationMachine(labelColumnName: "Sentiment", featureColumnNames: new[] { "Features" }));
5050

5151
// Fit the model.
5252
var model = pipeline.Fit(data);

‎docs/samples/Microsoft.ML.Samples/Dynamic/GeneralizedAdditiveModels.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ public static void Example()
2828
.ToArray();
2929
var pipeline = mlContext.Transforms.Concatenate("Features", featureNames)
3030
.Append(mlContext.Regression.Trainers.GeneralizedAdditiveModels(
31-
labelColumn: labelName, featureColumn: "Features", maxBins: 16));
31+
labelColumnName: labelName, featureColumnName: "Features", maxBins: 16));
3232
var fitPipeline = pipeline.Fit(data);
3333

3434
// Extract the model from the pipeline

‎docs/samples/Microsoft.ML.Samples/Dynamic/ImageAnalytics/ConvertToGrayScale.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,8 @@ public static void Example()
2727
{
2828
Columns = new[]
2929
{
30-
new TextLoader.Column("ImagePath", DataKind.TX, 0),
31-
new TextLoader.Column("Name", DataKind.TX, 1),
30+
new TextLoader.Column("ImagePath", DataKind.String, 0),
31+
new TextLoader.Column("Name", DataKind.String, 1),
3232
}
3333
}).Read(imagesDataFile);
3434

‎docs/samples/Microsoft.ML.Samples/Dynamic/ImageAnalytics/ExtractPixels.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,8 @@ public static void Example()
2828
{
2929
Columns = new[]
3030
{
31-
new TextLoader.Column("ImagePath", DataKind.TX, 0),
32-
new TextLoader.Column("Name", DataKind.TX, 1),
31+
new TextLoader.Column("ImagePath", DataKind.String, 0),
32+
new TextLoader.Column("Name", DataKind.String, 1),
3333
}
3434
}).Read(imagesDataFile);
3535

‎docs/samples/Microsoft.ML.Samples/Dynamic/ImageAnalytics/LoadImages.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,8 @@ public static void Example()
2727
{
2828
Columns = new[]
2929
{
30-
new TextLoader.Column("ImagePath", DataKind.TX, 0),
31-
new TextLoader.Column("Name", DataKind.TX, 1),
30+
new TextLoader.Column("ImagePath", DataKind.String, 0),
31+
new TextLoader.Column("Name", DataKind.String, 1),
3232
}
3333
}).Read(imagesDataFile);
3434

‎docs/samples/Microsoft.ML.Samples/Dynamic/ImageAnalytics/ResizeImages.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,8 @@ public static void Example()
2727
{
2828
Columns = new[]
2929
{
30-
new TextLoader.Column("ImagePath", DataKind.TX, 0),
31-
new TextLoader.Column("Name", DataKind.TX, 1),
30+
new TextLoader.Column("ImagePath", DataKind.String, 0),
31+
new TextLoader.Column("Name", DataKind.String, 1),
3232
}
3333
}).Read(imagesDataFile);
3434

‎docs/samples/Microsoft.ML.Samples/Dynamic/LogisticRegression.cs

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -37,21 +37,21 @@ public static void Example()
3737
HasHeader = true,
3838
Columns = new[]
3939
{
40-
new TextLoader.Column("age", DataKind.R4, 0),
41-
new TextLoader.Column("workclass", DataKind.Text, 1),
42-
new TextLoader.Column("fnlwgt", DataKind.R4, 2),
43-
new TextLoader.Column("education", DataKind.Text, 3),
44-
new TextLoader.Column("education-num", DataKind.R4, 4),
45-
new TextLoader.Column("marital-status", DataKind.Text, 5),
46-
new TextLoader.Column("occupation", DataKind.Text, 6),
47-
new TextLoader.Column("relationship", DataKind.Text, 7),
48-
new TextLoader.Column("ethnicity", DataKind.Text, 8),
49-
new TextLoader.Column("sex", DataKind.Text, 9),
50-
new TextLoader.Column("capital-gain", DataKind.R4, 10),
51-
new TextLoader.Column("capital-loss", DataKind.R4, 11),
52-
new TextLoader.Column("hours-per-week", DataKind.R4, 12),
53-
new TextLoader.Column("native-country", DataKind.Text, 13),
54-
new TextLoader.Column("Label", DataKind.Bool, 14)
40+
new TextLoader.Column("age", DataKind.Single, 0),
41+
new TextLoader.Column("workclass", DataKind.String, 1),
42+
new TextLoader.Column("fnlwgt", DataKind.Single, 2),
43+
new TextLoader.Column("education", DataKind.String, 3),
44+
new TextLoader.Column("education-num", DataKind.Single, 4),
45+
new TextLoader.Column("marital-status", DataKind.String, 5),
46+
new TextLoader.Column("occupation", DataKind.String, 6),
47+
new TextLoader.Column("relationship", DataKind.String, 7),
48+
new TextLoader.Column("ethnicity", DataKind.String, 8),
49+
new TextLoader.Column("sex", DataKind.String, 9),
50+
new TextLoader.Column("capital-gain", DataKind.Single, 10),
51+
new TextLoader.Column("capital-loss", DataKind.Single, 11),
52+
new TextLoader.Column("hours-per-week", DataKind.Single, 12),
53+
new TextLoader.Column("native-country", DataKind.String, 13),
54+
new TextLoader.Column("Label", DataKind.Boolean, 14)
5555
}
5656
});
5757

‎docs/samples/Microsoft.ML.Samples/Dynamic/PermutationFeatureImportance/PfiBinaryClassificationExample.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ public static void Example()
2323
var pipeline = mlContext.Transforms.Concatenate("Features", featureNames)
2424
.Append(mlContext.Transforms.Normalize("Features"))
2525
.Append(mlContext.BinaryClassification.Trainers.LogisticRegression(
26-
labelColumn: labelName, featureColumn: "Features"));
26+
labelColumnName: labelName, featureColumnName: "Features"));
2727
var model = pipeline.Fit(data);
2828

2929
// Extract the model from the pipeline

‎docs/samples/Microsoft.ML.Samples/Dynamic/TensorFlow/TextClassification.cs

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -34,12 +34,12 @@ public static void Example()
3434

3535
// This is the dictionary to convert words into the integer indexes.
3636
var lookupMap = mlContext.Data.ReadFromTextFile(Path.Combine(modelLocation, "imdb_word_index.csv"),
37-
columns: new[]
37+
columns: new[]
3838
{
39-
new TextLoader.Column("Words", DataKind.TX, 0),
40-
new TextLoader.Column("Ids", DataKind.I4, 1),
39+
new TextLoader.Column("Words", DataKind.String, 0),
40+
new TextLoader.Column("Ids", DataKind.Int32, 1),
4141
},
42-
separatorChar: ','
42+
separatorChar: ','
4343
);
4444

4545
// Load the TensorFlow model once.

‎docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/AveragedPerceptron.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ namespace Microsoft.ML.Samples.Dynamic.Trainers.BinaryClassification
55
public static class AveragedPerceptron
66
{
77
// In this examples we will use the adult income dataset. The goal is to predict
8-
// if a person's income is above $50K or not, based on different pieces of information about that person.
8+
// if a person's income is above $50K or not, based on demographic information about that person.
99
// For more details about this dataset, please see https://archive.ics.uci.edu/ml/datasets/adult.
1010
public static void Example()
1111
{

‎docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/AveragedPerceptronWithOptions.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ namespace Microsoft.ML.Samples.Dynamic.Trainers.BinaryClassification
66
public static class AveragedPerceptronWithOptions
77
{
88
// In this examples we will use the adult income dataset. The goal is to predict
9-
// if a person's income is above $50K or not, based on different pieces of information about that person.
9+
// if a person's income is above $50K or not, based on demographic information about that person.
1010
// For more details about this dataset, please see https://archive.ics.uci.edu/ml/datasets/adult.
1111
public static void Example()
1212
{

‎docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/SDCALogisticRegression.cs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,8 @@ public static void Example()
2828
var reader = mlContext.Data.CreateTextLoader(
2929
columns: new[]
3030
{
31-
new TextLoader.Column("Sentiment", DataKind.BL, 0),
32-
new TextLoader.Column("SentimentText", DataKind.Text, 1)
31+
new TextLoader.Column("Sentiment", DataKind.Boolean, 0),
32+
new TextLoader.Column("SentimentText", DataKind.String, 1)
3333
},
3434
hasHeader: true
3535
);
@@ -49,7 +49,7 @@ public static void Example()
4949
// the "Features" column produced by FeaturizeText as the features column.
5050
var pipeline = mlContext.Transforms.Text.FeaturizeText("SentimentText", "Features")
5151
.AppendCacheCheckpoint(mlContext) // Add a data-cache step within a pipeline.
52-
.Append(mlContext.BinaryClassification.Trainers.StochasticDualCoordinateAscent(labelColumn: "Sentiment", featureColumn: "Features", l2Const: 0.001f));
52+
.Append(mlContext.BinaryClassification.Trainers.StochasticDualCoordinateAscent(labelColumnName: "Sentiment", featureColumnName: "Features", l2Const: 0.001f));
5353

5454
// Step 3: Run Cross-Validation on this pipeline.
5555
var cvResults = mlContext.BinaryClassification.CrossValidate(data, pipeline, labelColumn: "Sentiment");

‎docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/SDCASupportVectorMachine.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ public static void Example()
4141
// Step 2: Create a binary classifier. This trainer may produce a logistic regression model.
4242
// We set the "Label" column as the label of the dataset, and the "Features" column as the features column.
4343
var pipeline = mlContext.BinaryClassification.Trainers.StochasticDualCoordinateAscentNonCalibrated(
44-
labelColumn: "Label", featureColumn: "Features", loss: new HingeLoss(), l2Const: 0.001f);
44+
labelColumnName: "Label", featureColumnName: "Features", loss: new HingeLoss(), l2Const: 0.001f);
4545

4646
// Step 3: Train the pipeline created.
4747
var model = pipeline.Fit(data);

0 commit comments

Comments
 (0)