-
Notifications
You must be signed in to change notification settings - Fork 1.9k
[WIP] TT Template for managing samples... #3017
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,97 @@ | ||
using System; | ||
using System.Collections.Generic; | ||
using System.Linq; | ||
using Microsoft.ML.Data; | ||
|
||
namespace Microsoft.ML.Samples.Dynamic.Trainers.Regression | ||
{ | ||
public static class FastTree2 | ||
{ | ||
// This example requires installation of additional NuGet package | ||
// <a href='https://www.nuget.org/packages/Microsoft.ML.FastTree/'>Microsoft.ML.FastTree</a>. | ||
public static void Example() | ||
{ | ||
// Create a new context for ML.NET operations. It can be used for exception tracking and logging, | ||
// as a catalog of available operations and as the source of randomness. | ||
// Setting the seed to a fixed number in this example to make outputs deterministic. | ||
var mlContext = new MLContext(seed: 0); | ||
|
||
// Create a list of training examples. | ||
var examples = GenerateRandomDataPoints(1000); | ||
|
||
// Convert the examples list to an IDataView object, which is consumable by ML.NET API. | ||
var trainingData = mlContext.Data.LoadFromEnumerable(examples); | ||
|
||
// Define the trainer. | ||
var pipeline = mlContext.Regression.Trainers.FastTree(); | ||
|
||
// Train the model. | ||
var model = pipeline.Fit(trainingData); | ||
|
||
// Create testing examples. Use different random seed to make it different from training data. | ||
var testData = mlContext.Data.LoadFromEnumerable(GenerateRandomDataPoints(500, seed:123)); | ||
|
||
// Run the model on test data set. | ||
var transformedTestData = model.Transform(testData); | ||
|
||
// Convert IDataView object to a list. | ||
var predictions = mlContext.Data.CreateEnumerable<Prediction>(transformedTestData, reuseRowObject: false).ToList(); | ||
|
||
// Look at 5 predictions | ||
foreach (var p in predictions.Take(5)) | ||
Console.WriteLine($"Label: {p.Label:F3}, Prediction: {p.Score:F3}"); | ||
|
||
// Expected output: | ||
// Label: 0.985, Prediction: 0.938 | ||
// Label: 0.155, Prediction: 0.131 | ||
// Label: 0.515, Prediction: 0.517 | ||
// Label: 0.566, Prediction: 0.519 | ||
// Label: 0.096, Prediction: 0.089 | ||
|
||
|
||
// Evaluate the overall metrics | ||
var metrics = mlContext.Regression.Evaluate(transformedTestData); | ||
SamplesUtils.ConsoleUtils.PrintMetrics(metrics); | ||
|
||
// Expected output: | ||
// Mean Absolute Error: 0.05 | ||
// Mean Squared Error: 0.00 | ||
// Root Mean Squared Error: 0.06 | ||
// RSquared: 0.95 | ||
} | ||
|
||
private static IEnumerable<DataPoint> GenerateRandomDataPoints(int count, int seed=0) | ||
{ | ||
var random = new Random(seed); | ||
float randomFloat() => (float)random.NextDouble(); | ||
for (int i = 0; i < count; i++) | ||
{ | ||
var label = randomFloat(); | ||
yield return new DataPoint | ||
{ | ||
Label = label, | ||
// Create random features that are correlated with label. | ||
Features = Enumerable.Repeat(label, 50).Select(x => x + randomFloat()).ToArray() | ||
}; | ||
} | ||
} | ||
|
||
// Example with label and 50 feature values. A data set is a collection of such examples. | ||
private class DataPoint | ||
{ | ||
public float Label { get; set; } | ||
[VectorType(50)] | ||
public float[] Features { get; set; } | ||
} | ||
|
||
// Class used to capture predictions. | ||
private class Prediction | ||
{ | ||
// Original label. | ||
public float Label { get; set; } | ||
// Predicted score from the trainer. | ||
public float Score { get; set; } | ||
} | ||
} | ||
} | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,23 @@ | ||
<#@ include file="RegressionTemplate.txt"#> | ||
|
||
<#+ | ||
string ClassName="FastTree2"; | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I have identified these placeholder in regression samples. |
||
string Comments = @"// This example requires installation of additional NuGet package | ||
// <a href='https://www.nuget.org/packages/Microsoft.ML.FastTree/'>Microsoft.ML.FastTree</a>."; | ||
|
||
string TrainingCode = @"var pipeline = mlContext.Regression.Trainers.FastTree();"; | ||
|
||
string ExpectedOutputPerInstance= @"// Expected output: | ||
// Label: 0.985, Prediction: 0.938 | ||
// Label: 0.155, Prediction: 0.131 | ||
// Label: 0.515, Prediction: 0.517 | ||
// Label: 0.566, Prediction: 0.519 | ||
// Label: 0.096, Prediction: 0.089 | ||
"; | ||
|
||
string ExpectedOutput = @"// Expected output: | ||
// Mean Absolute Error: 0.05 | ||
// Mean Squared Error: 0.00 | ||
// Root Mean Squared Error: 0.06 | ||
// RSquared: 0.95"; | ||
#> |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,85 @@ | ||
using System; | ||
using System.Collections.Generic; | ||
using System.Linq; | ||
using Microsoft.ML.Data; | ||
|
||
namespace Microsoft.ML.Samples.Dynamic.Trainers.Regression | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Since this is a txt, do we get Intellisence in VS? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Its is a stub code (an agreed upon template) e.g. I took it from Shahab's samples for regression and made it as template. You wont want get Intellisence for it but you get Intellisence for the generated file like In reply to: 267094827 [](ancestors = 267094827) There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I'm wondering if managing lots of txt files will be more work than standardizing files directly. I'm on the fence. For example, it's really hard to get indentation correct in txt files. In reply to: 267099333 [](ancestors = 267099333,267094827) There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think there will be one file for each ML task we have e.g. Regression, Classification etc. and that also in case when we have multiple samples using same pattern and one .tt for each sample. The .cs file will be auto generated. So, this is how it will work. Create a sample in a normal way using .cs file. Once the code is agreed upon and there are a couple of places the same pattern is used then turn that into .tt file and make placeholders in it. Also, if there is a problem in template, the generated csharp file will have the same problem as the template file. Any indentation errors etc. will definitely be caught in the review (or even during development). I think t4 template is just a way of automatically doing copy-paste. In reply to: 267100884 [](ancestors = 267100884,267099333,267094827) There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. i think in the long term having templates like this will improve maintainability and consistency; also to me it's easier to to than copy paste. In reply to: 267112240 [](ancestors = 267112240,267100884,267099333,267094827) There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. also I think we can get away with having to hard code the outputs and have the template calculate and print it directly. that would simplify the .tt files substantially. I'm working on that in my new PR. In reply to: 267114254 [](ancestors = 267114254,267112240,267100884,267099333,267094827) There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. In corefx/coreclr we using the file extension |
||
{ | ||
public static class <#=ClassName#> | ||
{ | ||
<#=Comments#> | ||
public static void Example() | ||
{ | ||
// Create a new context for ML.NET operations. It can be used for exception tracking and logging, | ||
// as a catalog of available operations and as the source of randomness. | ||
// Setting the seed to a fixed number in this example to make outputs deterministic. | ||
var mlContext = new MLContext(seed: 0); | ||
|
||
// Create a list of training examples. | ||
var examples = GenerateRandomDataPoints(1000); | ||
|
||
// Convert the examples list to an IDataView object, which is consumable by ML.NET API. | ||
var trainingData = mlContext.Data.LoadFromEnumerable(examples); | ||
|
||
// Define the trainer. | ||
<#=TrainingCode#> | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Everything in <# ... #> is a placeholder. |
||
|
||
// Train the model. | ||
var model = pipeline.Fit(trainingData); | ||
|
||
// Create testing examples. Use different random seed to make it different from training data. | ||
var testData = mlContext.Data.LoadFromEnumerable(GenerateRandomDataPoints(500, seed:123)); | ||
|
||
// Run the model on test data set. | ||
var transformedTestData = model.Transform(testData); | ||
|
||
// Convert IDataView object to a list. | ||
var predictions = mlContext.Data.CreateEnumerable<Prediction>(transformedTestData, reuseRowObject: false).ToList(); | ||
|
||
// Look at 5 predictions | ||
foreach (var p in predictions.Take(5)) | ||
Console.WriteLine($"Label: {p.Label:F3}, Prediction: {p.Score:F3}"); | ||
|
||
<#=ExpectedOutputPerInstance#> | ||
|
||
// Evaluate the overall metrics | ||
var metrics = mlContext.Regression.Evaluate(transformedTestData); | ||
SamplesUtils.ConsoleUtils.PrintMetrics(metrics); | ||
|
||
<#=ExpectedOutput#> | ||
} | ||
|
||
private static IEnumerable<DataPoint> GenerateRandomDataPoints(int count, int seed=0) | ||
{ | ||
var random = new Random(seed); | ||
float randomFloat() => (float)random.NextDouble(); | ||
for (int i = 0; i < count; i++) | ||
{ | ||
var label = randomFloat(); | ||
yield return new DataPoint | ||
{ | ||
Label = label, | ||
// Create random features that are correlated with label. | ||
Features = Enumerable.Repeat(label, 50).Select(x => x + randomFloat()).ToArray() | ||
}; | ||
} | ||
} | ||
|
||
// Example with label and 50 feature values. A data set is a collection of such examples. | ||
private class DataPoint | ||
{ | ||
public float Label { get; set; } | ||
[VectorType(50)] | ||
public float[] Features { get; set; } | ||
} | ||
|
||
// Class used to capture predictions. | ||
private class Prediction | ||
{ | ||
// Original label. | ||
public float Label { get; set; } | ||
// Predicted score from the trainer. | ||
public float Score { get; set; } | ||
} | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -35,4 +35,23 @@ | |
|
||
</ItemGroup> | ||
|
||
<ItemGroup> | ||
<None Update="Dynamic\Trainers\Regression\FastTreeTemplate.tt"> | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
is this Runtime Text Template or Text Template? At what point does it run? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. To be more specific, it is design (or compile) time template. In VS studio, In reply to: 267099394 [](ancestors = 267099394) |
||
<Generator>TextTemplatingFileGenerator</Generator> | ||
<LastGenOutput>FastTreeTemplate.cs</LastGenOutput> | ||
</None> | ||
</ItemGroup> | ||
|
||
<ItemGroup> | ||
<Service Include="{508349b6-6b84-4df5-91f0-309beebad82d}" /> | ||
</ItemGroup> | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. what is this for? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Not sure! I added .tt files and VS studio did it automatically. Maybe @eerhardt can tell us. |
||
|
||
<ItemGroup> | ||
<Compile Update="Dynamic\Trainers\Regression\FastTreeTemplate.cs"> | ||
<DesignTime>True</DesignTime> | ||
<AutoGen>True</AutoGen> | ||
<DependentUpon>FastTreeTemplate.tt</DependentUpon> | ||
</Compile> | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. do you add these manually or through VS UI? |
||
</ItemGroup> | ||
|
||
</Project> |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This file is auto generated from
FastTreeTemplate.tt
andRegressionTemplate.txt
.