Skip to content

Commit 06977ed

Browse files
zeahmedzeahmed
committed
Merge remote-tracking branch 'upstream/master' into learners_check_2
2 parents 5c139a1 + 49a96c5 commit 06977ed

File tree

47 files changed

+672
-439
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

47 files changed

+672
-439
lines changed

‎docs/samples/Microsoft.ML.Samples/Dynamic/FeatureContributionCalculationTransform.cs

+2-3
Original file line numberDiff line numberDiff line change
@@ -44,8 +44,7 @@ public static void Example()
4444
var outData = featureContributionCalculator.Fit(scoredData).Transform(scoredData);
4545

4646
// Let's extract the weights from the linear model to use as a comparison
47-
var weights = new VBuffer<float>();
48-
model.Model.GetFeatureWeights(ref weights);
47+
var weights = model.Model.Weights;
4948

5049
// Let's now walk through the first ten records and see which feature drove the values the most
5150
// Get prediction scores and contributions
@@ -63,7 +62,7 @@ public static void Example()
6362
var value = row.Features[featureOfInterest];
6463
var contribution = row.FeatureContributions[featureOfInterest];
6564
var name = data.Schema[featureOfInterest + 1].Name;
66-
var weight = weights.GetValues()[featureOfInterest];
65+
var weight = weights[featureOfInterest];
6766

6867
Console.WriteLine("{0:0.00}\t{1:0.00}\t{2}\t{3:0.00}\t{4:0.00}\t{5:0.00}",
6968
row.MedianHomeValue,

‎docs/samples/Microsoft.ML.Samples/Dynamic/TensorFlow/ImageClassification.cs

+1-2
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,7 @@ public static void Example()
2020
var idv = mlContext.Data.LoadFromEnumerable(data);
2121

2222
// Create a ML pipeline.
23-
var pipeline = mlContext.Transforms.ScoreTensorFlowModel(
24-
modelLocation,
23+
var pipeline = mlContext.Model.LoadTensorFlowModel(modelLocation).ScoreTensorFlowModel(
2524
new[] { nameof(OutputScores.output) },
2625
new[] { nameof(TensorData.input) });
2726

‎docs/samples/Microsoft.ML.Samples/Dynamic/TensorFlow/TextClassification.cs

+3-3
Original file line numberDiff line numberDiff line change
@@ -45,8 +45,8 @@ public static void Example()
4545
// Load the TensorFlow model once.
4646
// - Use it for quering the schema for input and output in the model
4747
// - Use it for prediction in the pipeline.
48-
var modelInfo = TensorFlowUtils.LoadTensorFlowModel(mlContext, modelLocation);
49-
var schema = modelInfo.GetModelSchema();
48+
var tensorFlowModel = mlContext.Model.LoadTensorFlowModel(modelLocation);
49+
var schema = tensorFlowModel.GetModelSchema();
5050
var featuresType = (VectorType)schema["Features"].Type;
5151
Console.WriteLine("Name: {0}, Type: {1}, Shape: (-1, {2})", "Features", featuresType.ItemType.RawType, featuresType.Dimensions[0]);
5252
var predictionType = (VectorType)schema["Prediction/Softmax"].Type;
@@ -72,7 +72,7 @@ public static void Example()
7272
var engine = mlContext.Transforms.Text.TokenizeWords("TokenizedWords", "Sentiment_Text")
7373
.Append(mlContext.Transforms.Conversion.ValueMap(lookupMap, "Words", "Ids", new ColumnOptions[] { ("VariableLenghtFeatures", "TokenizedWords") }))
7474
.Append(mlContext.Transforms.CustomMapping(ResizeFeaturesAction, "Resize"))
75-
.Append(mlContext.Transforms.ScoreTensorFlowModel(modelInfo, new[] { "Prediction/Softmax" }, new[] { "Features" }))
75+
.Append(tensorFlowModel.ScoreTensorFlowModel(new[] { "Prediction/Softmax" }, new[] { "Features" }))
7676
.Append(mlContext.Transforms.CopyColumns(("Prediction", "Prediction/Softmax")))
7777
.Fit(dataView)
7878
.CreatePredictionEngine<IMDBSentiment, OutputScores>(mlContext);

‎docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/AnomalyDetection/RandomizedPcaSample.cs

+86
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Linq;
4+
using Microsoft.ML.Data;
5+
6+
namespace Microsoft.ML.Samples.Dynamic.Trainers.AnomalyDetection
7+
{
8+
public static class RandomizedPcaSample
9+
{
10+
public static void Example()
11+
{
12+
// Create a new context for ML.NET operations. It can be used for exception tracking and logging,
13+
// as a catalog of available operations and as the source of randomness.
14+
// Setting the seed to a fixed number in this example to make outputs deterministic.
15+
var mlContext = new MLContext(seed: 0);
16+
17+
// Training data.
18+
var samples = new List<DataPoint>()
19+
{
20+
new DataPoint(){ Features = new float[3] {1, 0, 0} },
21+
new DataPoint(){ Features = new float[3] {0, 2, 1} },
22+
new DataPoint(){ Features = new float[3] {1, 2, 3} },
23+
new DataPoint(){ Features = new float[3] {0, 1, 0} },
24+
new DataPoint(){ Features = new float[3] {0, 2, 1} },
25+
new DataPoint(){ Features = new float[3] {-100, 50, -100} }
26+
};
27+
28+
// Convert the List<DataPoint> to IDataView, a consumble format to ML.NET functions.
29+
var data = mlContext.Data.LoadFromEnumerable(samples);
30+
31+
// Create an anomaly detector. Its underlying algorithm is randomized PCA.
32+
var pipeline = mlContext.AnomalyDetection.Trainers.RandomizedPca(featureColumnName: nameof(DataPoint.Features), rank: 1, center: false);
33+
34+
// Train the anomaly detector.
35+
var model = pipeline.Fit(data);
36+
37+
// Apply the trained model on the training data.
38+
var transformed = model.Transform(data);
39+
40+
// Read ML.NET predictions into IEnumerable<Result>.
41+
var results = mlContext.Data.CreateEnumerable<Result>(transformed, reuseRowObject: false).ToList();
42+
43+
// Let's go through all predictions.
44+
for (int i = 0; i < samples.Count; ++i)
45+
{
46+
// The i-th example's prediction result.
47+
var result = results[i];
48+
49+
// The i-th example's feature vector in text format.
50+
var featuresInText = string.Join(',', samples[i].Features);
51+
52+
if (result.PredictedLabel)
53+
// The i-th sample is predicted as an inlier.
54+
Console.WriteLine("The {0}-th example with features [{1}] is an inlier with a score of being inlier {2}",
55+
i, featuresInText, result.Score);
56+
else
57+
// The i-th sample is predicted as an outlier.
58+
Console.WriteLine("The {0}-th example with features [{1}] is an outlier with a score of being inlier {2}",
59+
i, featuresInText, result.Score);
60+
}
61+
// Lines printed out should be
62+
// The 0 - th example with features[1, 0, 0] is an inlier with a score of being inlier 0.7453707
63+
// The 1 - th example with features[0, 2, 1] is an inlier with a score of being inlier 0.9999999
64+
// The 2 - th example with features[1, 2, 3] is an inlier with a score of being inlier 0.8450122
65+
// The 3 - th example with features[0, 1, 0] is an inlier with a score of being inlier 0.9428905
66+
// The 4 - th example with features[0, 2, 1] is an inlier with a score of being inlier 0.9999999
67+
// The 5 - th example with features[-100, 50, -100] is an outlier with a score of being inlier 0
68+
}
69+
70+
// Example with 3 feature values. A training data set is a collection of such examples.
71+
private class DataPoint
72+
{
73+
[VectorType(3)]
74+
public float[] Features { get; set; }
75+
}
76+
77+
// Class used to capture prediction of DataPoint.
78+
private class Result
79+
{
80+
// Outlier gets false while inlier has true.
81+
public bool PredictedLabel { get; set; }
82+
// Outlier gets smaller score.
83+
public float Score { get; set; }
84+
}
85+
}
86+
}

‎docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/AnomalyDetection/RandomizedPcaSampleWithOptions.cs

+93
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Linq;
4+
using Microsoft.ML.Data;
5+
6+
namespace Microsoft.ML.Samples.Dynamic.Trainers.AnomalyDetection
7+
{
8+
public static class RandomizedPcaSampleWithOptions
9+
{
10+
public static void Example()
11+
{
12+
// Create a new context for ML.NET operations. It can be used for exception tracking and logging,
13+
// as a catalog of available operations and as the source of randomness.
14+
// Setting the seed to a fixed number in this example to make outputs deterministic.
15+
var mlContext = new MLContext(seed: 0);
16+
17+
// Training data.
18+
var samples = new List<DataPoint>()
19+
{
20+
new DataPoint(){ Features = new float[3] {1, 0, 0} },
21+
new DataPoint(){ Features = new float[3] {0, 2, 1} },
22+
new DataPoint(){ Features = new float[3] {1, 2, 3} },
23+
new DataPoint(){ Features = new float[3] {0, 1, 0} },
24+
new DataPoint(){ Features = new float[3] {0, 2, 1} },
25+
new DataPoint(){ Features = new float[3] {-100, 50, -100} }
26+
};
27+
28+
// Convert the List<DataPoint> to IDataView, a consumble format to ML.NET functions.
29+
var data = mlContext.Data.LoadFromEnumerable(samples);
30+
31+
var options = new ML.Trainers.RandomizedPcaTrainer.Options()
32+
{
33+
FeatureColumnName = nameof(DataPoint.Features),
34+
Rank = 1,
35+
Seed = 10,
36+
};
37+
38+
// Create an anomaly detector. Its underlying algorithm is randomized PCA.
39+
var pipeline = mlContext.AnomalyDetection.Trainers.RandomizedPca(options);
40+
41+
// Train the anomaly detector.
42+
var model = pipeline.Fit(data);
43+
44+
// Apply the trained model on the training data.
45+
var transformed = model.Transform(data);
46+
47+
// Read ML.NET predictions into IEnumerable<Result>.
48+
var results = mlContext.Data.CreateEnumerable<Result>(transformed, reuseRowObject: false).ToList();
49+
50+
// Let's go through all predictions.
51+
for (int i = 0; i < samples.Count; ++i)
52+
{
53+
// The i-th example's prediction result.
54+
var result = results[i];
55+
56+
// The i-th example's feature vector in text format.
57+
var featuresInText = string.Join(',', samples[i].Features);
58+
59+
if (result.PredictedLabel)
60+
// The i-th sample is predicted as an inlier.
61+
Console.WriteLine("The {0}-th example with features [{1}] is an inlier with a score of being inlier {2}",
62+
i, featuresInText, result.Score);
63+
else
64+
// The i-th sample is predicted as an outlier.
65+
Console.WriteLine("The {0}-th example with features [{1}] is an outlier with a score of being inlier {2}",
66+
i, featuresInText, result.Score);
67+
}
68+
// Lines printed out should be
69+
// The 0 - th example with features[1, 0, 0] is an inlier with a score of being inlier 0.7453707
70+
// The 1 - th example with features[0, 2, 1] is an inlier with a score of being inlier 0.9999999
71+
// The 2 - th example with features[1, 2, 3] is an inlier with a score of being inlier 0.8450122
72+
// The 3 - th example with features[0, 1, 0] is an inlier with a score of being inlier 0.9428905
73+
// The 4 - th example with features[0, 2, 1] is an inlier with a score of being inlier 0.9999999
74+
// The 5 - th example with features[-100, 50, -100] is an outlier with a score of being inlier 0
75+
}
76+
77+
// Example with 3 feature values. A training data set is a collection of such examples.
78+
private class DataPoint
79+
{
80+
[VectorType(3)]
81+
public float[] Features { get; set; }
82+
}
83+
84+
// Class used to capture prediction of DataPoint.
85+
private class Result
86+
{
87+
// Outlier gets false while inlier has true.
88+
public bool PredictedLabel { get; set; }
89+
// Outlier gets smaller score.
90+
public float Score { get; set; }
91+
}
92+
}
93+
}

‎docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Recommendation/MatrixFactorization.cs

-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
using System;
22
using System.Collections.Generic;
3-
using Microsoft.ML.Data;
43
using static Microsoft.ML.SamplesUtils.DatasetUtils;
54

65
namespace Microsoft.ML.Samples.Dynamic.Trainers.Recommendation

‎docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Recommendation/MatrixFactorizationWithOptions.cs

-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
using System;
22
using System.Collections.Generic;
3-
using Microsoft.ML.Data;
43
using Microsoft.ML.Trainers;
54
using static Microsoft.ML.SamplesUtils.DatasetUtils;
65

‎docs/samples/Microsoft.ML.Samples/Static/SDCARegression.cs

+3-5
Original file line numberDiff line numberDiff line change
@@ -46,12 +46,10 @@ public static void SdcaRegression()
4646
var model = learningPipeline.Fit(trainData);
4747

4848
// Check the weights that the model learned
49-
VBuffer<float> weights = default;
50-
pred.GetFeatureWeights(ref weights);
49+
var weights = pred.Weights;
5150

52-
var weightsValues = weights.GetValues();
53-
Console.WriteLine($"weight 0 - {weightsValues[0]}");
54-
Console.WriteLine($"weight 1 - {weightsValues[1]}");
51+
Console.WriteLine($"weight 0 - {weights[0]}");
52+
Console.WriteLine($"weight 1 - {weights[1]}");
5553

5654
// Evaluate how the model is doing on the test data
5755
var dataWithPredictions = model.Transform(testData);

‎src/Microsoft.ML.Data/Dirty/PredictorInterfaces.cs

+2-1
Original file line numberDiff line numberDiff line change
@@ -146,7 +146,8 @@ internal interface ICanSaveInSourceCode
146146
/// <summary>
147147
/// Interface implemented by components that can assign weights to features.
148148
/// </summary>
149-
public interface IHaveFeatureWeights
149+
[BestFriend]
150+
internal interface IHaveFeatureWeights
150151
{
151152
/// <summary>
152153
/// Returns the weights for the features.

‎src/Microsoft.ML.DnnAnalyzer/Microsoft.ML.DnnAnalyzer/DnnAnalyzer.cs

+34-2
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,10 @@
33
// See the LICENSE file in the project root for more information.
44

55
using System;
6-
using Microsoft.ML.Transforms.TensorFlow;
6+
using System.Collections.Generic;
7+
using System.Linq;
8+
using Microsoft.Data.DataView;
9+
using Microsoft.ML.Data;
710

811
namespace Microsoft.ML.DnnAnalyzer
912
{
@@ -17,11 +20,40 @@ public static void Main(string[] args)
1720
return;
1821
}
1922

20-
foreach (var (name, opType, type, inputs) in TensorFlowUtils.GetModelNodes(new MLContext(), args[0]))
23+
foreach (var (name, opType, type, inputs) in GetModelNodes(args[0]))
2124
{
2225
var inputsString = inputs.Length == 0 ? "" : $", input nodes: {string.Join(", ", inputs)}";
2326
Console.WriteLine($"Graph node: '{name}', operation type: '{opType}', output type: '{type}'{inputsString}");
2427
}
2528
}
29+
30+
private static IEnumerable<(string, string, DataViewType, string[])> GetModelNodes(string modelPath)
31+
{
32+
var mlContext = new MLContext();
33+
var tensorFlowModel = mlContext.Model.LoadTensorFlowModel(modelPath);
34+
var schema = tensorFlowModel.GetModelSchema();
35+
36+
for (int i = 0; i < schema.Count; i++)
37+
{
38+
var name = schema[i].Name;
39+
var type = schema[i].Type;
40+
41+
var metadataType = schema[i].Annotations.Schema.GetColumnOrNull("TensorflowOperatorType")?.Type;
42+
ReadOnlyMemory<char> opType = default;
43+
schema[i].Annotations.GetValue("TensorflowOperatorType", ref opType);
44+
metadataType = schema[i].Annotations.Schema.GetColumnOrNull("TensorflowUpstreamOperators")?.Type;
45+
VBuffer <ReadOnlyMemory<char>> inputOps = default;
46+
if (metadataType != null)
47+
{
48+
schema[i].Annotations.GetValue("TensorflowUpstreamOperators", ref inputOps);
49+
}
50+
51+
string[] inputOpsResult = inputOps.DenseValues()
52+
.Select(input => input.ToString())
53+
.ToArray();
54+
55+
yield return (name, opType.ToString(), type, inputOpsResult);
56+
}
57+
}
2658
}
2759
}

‎src/Microsoft.ML.DnnImageFeaturizer.AlexNet/AlexNetExtension.cs

+2-1
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
// The .NET Foundation licenses this file to you under the MIT license.
33
// See the LICENSE file in the project root for more information.
44

5+
using System;
56
using System.IO;
67
using Microsoft.ML.Data;
78

@@ -21,7 +22,7 @@ public static class AlexNetExtension
2122
/// </summary>
2223
public static EstimatorChain<ColumnCopyingTransformer> AlexNet(this DnnImageModelSelector dnnModelContext, IHostEnvironment env, string outputColumnName, string inputColumnName)
2324
{
24-
return AlexNet(dnnModelContext, env, outputColumnName, inputColumnName, Path.Combine(AssemblyPathHelpers.GetExecutingAssemblyLocation(), "DnnImageModels"));
25+
return AlexNet(dnnModelContext, env, outputColumnName, inputColumnName, Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "DnnImageModels"));
2526
}
2627

2728
/// <summary>

‎src/Microsoft.ML.DnnImageFeaturizer.AlexNet/AssemblyPathHelpers.cs

-19
This file was deleted.

‎src/Microsoft.ML.DnnImageFeaturizer.ResNet101/Microsoft.ML.DnnImageFeaturizer.ResNet101.csproj

-4
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,6 @@
55
<IncludeInPackage>Microsoft.ML.DnnImageFeaturizer.ResNet101</IncludeInPackage>
66
</PropertyGroup>
77

8-
<ItemGroup>
9-
<Compile Include="..\Microsoft.ML.DnnImageFeaturizer.AlexNet\AssemblyPathHelpers.cs" />
10-
</ItemGroup>
11-
128
<ItemGroup>
139
<ProjectReference Include="..\Microsoft.ML.OnnxTransformer\Microsoft.ML.OnnxTransformer.csproj" />
1410
</ItemGroup>

‎src/Microsoft.ML.DnnImageFeaturizer.ResNet101/ResNet101Extension.cs

+2-1
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
// The .NET Foundation licenses this file to you under the MIT license.
33
// See the LICENSE file in the project root for more information.
44

5+
using System;
56
using System.IO;
67
using Microsoft.ML.Data;
78

@@ -21,7 +22,7 @@ public static class ResNet101Extension
2122
/// </summary>
2223
public static EstimatorChain<ColumnCopyingTransformer> ResNet101(this DnnImageModelSelector dnnModelContext, IHostEnvironment env, string outputColumnName, string inputColumnName)
2324
{
24-
return ResNet101(dnnModelContext, env, outputColumnName, inputColumnName, Path.Combine(AssemblyPathHelpers.GetExecutingAssemblyLocation(), "DnnImageModels"));
25+
return ResNet101(dnnModelContext, env, outputColumnName, inputColumnName, Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "DnnImageModels"));
2526
}
2627

2728
/// <summary>

0 commit comments

Comments
 (0)