From fc139f727d19d77dddd251ccb1e4523b31628714 Mon Sep 17 00:00:00 2001 From: Antonio Velazquez Date: Fri, 10 Jan 2020 17:22:17 -0800 Subject: [PATCH 01/11] Modified Program.cs --- docs/samples/Microsoft.ML.Samples/Program.cs | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/docs/samples/Microsoft.ML.Samples/Program.cs b/docs/samples/Microsoft.ML.Samples/Program.cs index 4c46399421..61c0172aae 100644 --- a/docs/samples/Microsoft.ML.Samples/Program.cs +++ b/docs/samples/Microsoft.ML.Samples/Program.cs @@ -10,20 +10,7 @@ public static class Program internal static void RunAll() { - int samples = 0; - foreach (var type in Assembly.GetExecutingAssembly().GetTypes()) - { - var sample = type.GetMethod("Example", BindingFlags.Public | BindingFlags.Static | BindingFlags.FlattenHierarchy); - - if (sample != null) - { - Console.WriteLine(type.Name); - sample.Invoke(null, null); - samples++; - } - } - - Console.WriteLine("Number of samples that ran without any exception: " + samples); + ImageClassificationDefault2.Example(); } } } From 7147772301cbccd1c53a39083bbfb5a5ed5059f6 Mon Sep 17 00:00:00 2001 From: Antonio Velazquez Date: Fri, 10 Jan 2020 17:22:47 -0800 Subject: [PATCH 02/11] Added other sample for Image Classification --- .../ImageClassificationDefault2.cs | 344 ++++++++++++++++++ 1 file changed, 344 insertions(+) create mode 100644 docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/ImageClassification/ImageClassificationDefault2.cs diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/ImageClassification/ImageClassificationDefault2.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/ImageClassification/ImageClassificationDefault2.cs new file mode 100644 index 0000000000..03a61b5836 --- /dev/null +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/ImageClassification/ImageClassificationDefault2.cs @@ -0,0 +1,344 @@ + +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Net; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.ML; +using Microsoft.ML.Data; +using static Microsoft.ML.DataOperationsCatalog; + +namespace Samples.Dynamic +{ + public class ImageClassificationDefault2 + { + public static void Example() + { + // Set the path for input images. + string assetsRelativePath = @"../../../assets"; + string assetsPath = GetAbsolutePath(assetsRelativePath); + + string imagesDownloadFolderPath = Path.Combine(assetsPath, "inputs", + "images"); + + //Download the image set and unzip, set the path to image folder. + string finalImagesFolderName = DownloadImageSet( + imagesDownloadFolderPath); + + string fullImagesetFolderPath = Path.Combine( + imagesDownloadFolderPath, finalImagesFolderName); + + MLContext mlContext = new MLContext(seed: 1); + mlContext.Log += MlContext_Log; + + // Load all the original images info + IEnumerable images = LoadImagesFromDirectory( + folder: fullImagesetFolderPath, useFolderNameAsLabel: true); + + // Shuffle images. + IDataView shuffledFullImagesDataset = mlContext.Data.ShuffleRows( + mlContext.Data.LoadFromEnumerable(images)); + + // Apply transforms to the input dataset: + // MapValueToKey : map 'string' type labels to keys + // LoadImages : load raw images to "Image" column + shuffledFullImagesDataset = mlContext.Transforms.Conversion + .MapValueToKey("Label", keyOrdinality: Microsoft.ML.Transforms + .ValueToKeyMappingEstimator.KeyOrdinality.ByValue) + .Append(mlContext.Transforms.LoadRawImageBytes("Image", + fullImagesetFolderPath, "ImagePath")) + .Fit(shuffledFullImagesDataset) + .Transform(shuffledFullImagesDataset); + + // Split the data 90:10 into train and test sets. + TrainTestData trainTestData = mlContext.Data.TrainTestSplit( + shuffledFullImagesDataset, testFraction: 0.1, seed: 1); + + IDataView trainDataset = trainTestData.TrainSet; + IDataView testDataset = trainTestData.TestSet; + + // Create the ImageClassification pipeline by just passing the + // input feature and label column name. + var pipeline = mlContext.MulticlassClassification.Trainers + .ImageClassification(featureColumnName:"Image") + .Append(mlContext.Transforms.Conversion.MapKeyToValue( + outputColumnName: "PredictedLabel", + inputColumnName: "PredictedLabel")); + + Console.WriteLine("*** Training the image classification model " + + "with DNN Transfer Learning on top of the selected " + + "pre-trained model/architecture ***"); + + // Train the model. + // This involves calculating the bottleneck values, and then + // training the final layerSample output is: + // [Source=ImageClassificationTrainer; ImageClassificationTrainer, Kind=Trace] Phase: Bottleneck Computation, Dataset used: Train, Image Index: 1 + // [Source=ImageClassificationTrainer; ImageClassificationTrainer, Kind=Trace] Phase: Bottleneck Computation, Dataset used: Train, Image Index: 2 + // ... + // [Source=ImageClassificationTrainer; ImageClassificationTrainer, Kind=Trace] Phase: Training, Dataset used: Train, Batch Processed Count: 18, Learning Rate: 0.01 Epoch: 0, Accuracy: 0.9, Cross-Entropy: 0.481340 + // ... + // [Source=ImageClassificationTrainer; ImageClassificationTrainer, Kind=Trace] Phase: Training, Dataset used: Train, Batch Processed Count: 18, Learning Rate: 0.004759203 Epoch: 25, Accuracy: 1, Cross-Entropy: 0.04848097 + // [Source=ImageClassificationTrainer; ImageClassificationTrainer, Kind=Trace] Phase: Training, Dataset used: Train, Batch Processed Count: 18, Learning Rate: 0.004473651 Epoch: 26, Accuracy: 1, Cross-Entropy: 0.04930306 + var trainedModel = pipeline.Fit(trainDataset); + + Console.WriteLine("Training with transfer learning finished."); + + // Save the trained model. + mlContext.Model.Save(trainedModel, shuffledFullImagesDataset.Schema, + "model.zip"); + + // Load the trained and saved model for prediction. + ITransformer loadedModel; + DataViewSchema schema; + using (var file = File.OpenRead("model.zip")) + loadedModel = mlContext.Model.Load(file, out schema); + + // Evaluate the model on the test dataset. + // Sample output: + // Making bulk predictions and evaluating model's quality... + // Micro-accuracy: 0.925925925925926,macro-accuracy = 0.933333333333333 + EvaluateModel(mlContext, testDataset, loadedModel); + + // Predict on a single image class using an in-memory image. + // Sample output: + // Scores : [0.8657553,0.006911285,1.46484E-05,0.1266835,0.0006352618], Predicted Label : daisy + TrySinglePrediction(fullImagesetFolderPath, mlContext, loadedModel); + + Console.WriteLine("Prediction on a single image finished."); + + Console.WriteLine("Press any key to finish"); + Console.ReadKey(); + } + + private static void MlContext_Log(object sender, LoggingEventArgs e) + { + if (e.Message.StartsWith("[Source=ImageClassificationTrainer;")) + { + Console.WriteLine(e.Message); + } + } + + // Predict on a single image. + private static void TrySinglePrediction(string imagesForPredictions, + MLContext mlContext, ITransformer trainedModel) + { + // Create prediction function to try one prediction. + var predictionEngine = mlContext.Model + .CreatePredictionEngine(trainedModel); + + // Load test images. + IEnumerable testImages = + LoadInMemoryImagesFromDirectory(imagesForPredictions, false); + + // Create an in-memory image object from the first image in the test data. + InMemoryImageData imageToPredict = new InMemoryImageData + { + Image = testImages.First().Image + }; + + // Predict on the single image. + var prediction = predictionEngine.Predict(imageToPredict); + + Console.WriteLine($"Scores : [{string.Join(",", prediction.Score)}], " + + $"Predicted Label : {prediction.PredictedLabel}"); + } + + // Evaluate the trained model on the passed test dataset. + private static void EvaluateModel(MLContext mlContext, + IDataView testDataset, ITransformer trainedModel) + { + Console.WriteLine("Making bulk predictions and evaluating model's " + + "quality..."); + + // Evaluate the model on the test data and get the evaluation metrics. + IDataView predictions = trainedModel.Transform(testDataset); + var metrics = mlContext.MulticlassClassification.Evaluate(predictions); + + Console.WriteLine($"Micro-accuracy: {metrics.MicroAccuracy}," + + $"macro-accuracy = {metrics.MacroAccuracy}"); + + Console.WriteLine("Predicting and Evaluation complete."); + } + + //Load the Image Data from input directory. + public static IEnumerable LoadImagesFromDirectory(string folder, + bool useFolderNameAsLabel = true) + { + var files = Directory.GetFiles(folder, "*", + searchOption: SearchOption.AllDirectories); + foreach (var file in files) + { + if (Path.GetExtension(file) != ".jpg") + continue; + + var label = Path.GetFileName(file); + if (useFolderNameAsLabel) + label = Directory.GetParent(file).Name; + else + { + for (int index = 0; index < label.Length; index++) + { + if (!char.IsLetter(label[index])) + { + label = label.Substring(0, index); + break; + } + } + } + + yield return new ImageData() + { + ImagePath = file, + Label = label + }; + + } + } + + // Load In memory raw images from directory. + public static IEnumerable + LoadInMemoryImagesFromDirectory(string folder, + bool useFolderNameAsLabel = true) + { + var files = Directory.GetFiles(folder, "*", + searchOption: SearchOption.AllDirectories); + foreach (var file in files) + { + if (Path.GetExtension(file) != ".jpg") + continue; + + var label = Path.GetFileName(file); + if (useFolderNameAsLabel) + label = Directory.GetParent(file).Name; + else + { + for (int index = 0; index < label.Length; index++) + { + if (!char.IsLetter(label[index])) + { + label = label.Substring(0, index); + break; + } + } + } + + yield return new InMemoryImageData() + { + Image = File.ReadAllBytes(file), + Label = label + }; + + } + } + + // Download and unzip the image dataset. + public static string DownloadImageSet(string imagesDownloadFolder) + { + // get a set of images to teach the network about the new classes + + //SINGLE SMALL FLOWERS IMAGESET (200 files) + string fileName = "flower_photos_small_set.zip"; + string url = $"https://aka.ms/mlnet-resources/datasets/flower_photos_small_set.zip"; + + Download(url, imagesDownloadFolder, fileName); + UnZip(Path.Combine(imagesDownloadFolder, fileName), imagesDownloadFolder); + + return Path.GetFileNameWithoutExtension(fileName); + } + + // Download file to destination directory from input URL. + public static bool Download(string url, string destDir, string destFileName) + { + if (destFileName == null) + destFileName = url.Split(Path.DirectorySeparatorChar).Last(); + + Directory.CreateDirectory(destDir); + + string relativeFilePath = Path.Combine(destDir, destFileName); + + if (File.Exists(relativeFilePath)) + { + Console.WriteLine($"{relativeFilePath} already exists."); + return false; + } + + var wc = new WebClient(); + Console.WriteLine($"Downloading {relativeFilePath}"); + var download = Task.Run(() => wc.DownloadFile(url, relativeFilePath)); + while (!download.IsCompleted) + { + Thread.Sleep(1000); + Console.Write("."); + } + Console.WriteLine(""); + Console.WriteLine($"Downloaded {relativeFilePath}"); + + return true; + } + + // Unzip the file to destination folder. + public static void UnZip(String gzArchiveName, String destFolder) + { + var flag = gzArchiveName.Split(Path.DirectorySeparatorChar) + .Last() + .Split('.') + .First() + ".bin"; + + if (File.Exists(Path.Combine(destFolder, flag))) return; + + Console.WriteLine($"Extracting."); + ZipFile.ExtractToDirectory(gzArchiveName, destFolder); + + File.Create(Path.Combine(destFolder, flag)); + Console.WriteLine(""); + Console.WriteLine("Extracting is completed."); + } + + // Get absolute path from relative path. + public static string GetAbsolutePath(string relativePath) + { + FileInfo _dataRoot = new FileInfo(typeof( + ImageClassificationDefault).Assembly.Location); + + string assemblyFolderPath = _dataRoot.Directory.FullName; + + string fullPath = Path.Combine(assemblyFolderPath, relativePath); + + return fullPath; + } + + // InMemoryImageData class holding the raw image byte array and label. + public class InMemoryImageData + { + [LoadColumn(0)] + public byte[] Image; + + [LoadColumn(1)] + public string Label; + } + + // ImageData class holding the imagepath and label. + public class ImageData + { + [LoadColumn(0)] + public string ImagePath; + + [LoadColumn(1)] + public string Label; + } + + // ImagePrediction class holding the score and predicted label metrics. + public class ImagePrediction + { + [ColumnName("Score")] + public float[] Score; + + [ColumnName("PredictedLabel")] + public string PredictedLabel; + } + } +} From 3a7a0bb8f50321e6e7853685cede25217d957f98 Mon Sep 17 00:00:00 2001 From: Antonio Velazquez Date: Mon, 13 Jan 2020 11:22:28 -0800 Subject: [PATCH 03/11] Added sample to test cancelling method --- .../ImageClassificationDefault2.cs | 105 +++++++++++++----- .../Microsoft.ML.Samples.csproj | 1 + docs/samples/Microsoft.ML.Samples/Program.cs | 2 +- 3 files changed, 82 insertions(+), 26 deletions(-) rename docs/samples/Microsoft.ML.Samples/{Dynamic/Trainers/MulticlassClassification/ImageClassification => }/ImageClassificationDefault2.cs (80%) diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/ImageClassification/ImageClassificationDefault2.cs b/docs/samples/Microsoft.ML.Samples/ImageClassificationDefault2.cs similarity index 80% rename from docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/ImageClassification/ImageClassificationDefault2.cs rename to docs/samples/Microsoft.ML.Samples/ImageClassificationDefault2.cs index 03a61b5836..157aa1be73 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/ImageClassification/ImageClassificationDefault2.cs +++ b/docs/samples/Microsoft.ML.Samples/ImageClassificationDefault2.cs @@ -9,13 +9,59 @@ using System.Threading.Tasks; using Microsoft.ML; using Microsoft.ML.Data; +using Microsoft.ML.Experimental; using static Microsoft.ML.DataOperationsCatalog; namespace Samples.Dynamic { public class ImageClassificationDefault2 { - public static void Example() + public static void MyMain() + { + MLContext mlContext = new MLContext(seed: 1); + var kbTask = Task.Run(() => + { + ConsoleListener(mlContext); + }); + + // thread that performs main work + Task.Run(() => { + Example(mlContext); + }); + + Console.WriteLine("Type commands followed by 'ENTER'"); + Console.WriteLine("Enter 'C' to end program."); + Console.WriteLine(); + + // keep Console running until cancellation token is invoked + kbTask.Wait(); + } + + + static void ConsoleListener(MLContext ctx) + { + while (true) + { + string userInput = Console.ReadLine(); + if (userInput == "c") + { + Console.WriteLine("Cancelation Requested"); + ctx.CancelExecution(); + GC.Collect(); + } + else if (userInput == "q") + { + break; + } + else + { + // handle input + Console.WriteLine("Executing user command {0}...", userInput); + } + } + } + + public static void Example(MLContext mlContext) { // Set the path for input images. string assetsRelativePath = @"../../../assets"; @@ -31,8 +77,7 @@ public static void Example() string fullImagesetFolderPath = Path.Combine( imagesDownloadFolderPath, finalImagesFolderName); - MLContext mlContext = new MLContext(seed: 1); - mlContext.Log += MlContext_Log; + // mlContext.Log += MlContext_Log; // Load all the original images info IEnumerable images = LoadImagesFromDirectory( @@ -82,35 +127,45 @@ public static void Example() // ... // [Source=ImageClassificationTrainer; ImageClassificationTrainer, Kind=Trace] Phase: Training, Dataset used: Train, Batch Processed Count: 18, Learning Rate: 0.004759203 Epoch: 25, Accuracy: 1, Cross-Entropy: 0.04848097 // [Source=ImageClassificationTrainer; ImageClassificationTrainer, Kind=Trace] Phase: Training, Dataset used: Train, Batch Processed Count: 18, Learning Rate: 0.004473651 Epoch: 26, Accuracy: 1, Cross-Entropy: 0.04930306 - var trainedModel = pipeline.Fit(trainDataset); - Console.WriteLine("Training with transfer learning finished."); + try + { + var trainedModel = pipeline.Fit(trainDataset); + + Console.WriteLine("Training with transfer learning finished."); - // Save the trained model. - mlContext.Model.Save(trainedModel, shuffledFullImagesDataset.Schema, - "model.zip"); + // Save the trained model. + mlContext.Model.Save(trainedModel, shuffledFullImagesDataset.Schema, + "model.zip"); - // Load the trained and saved model for prediction. - ITransformer loadedModel; - DataViewSchema schema; - using (var file = File.OpenRead("model.zip")) - loadedModel = mlContext.Model.Load(file, out schema); + // Load the trained and saved model for prediction. + ITransformer loadedModel; + DataViewSchema schema; + using (var file = File.OpenRead("model.zip")) + loadedModel = mlContext.Model.Load(file, out schema); - // Evaluate the model on the test dataset. - // Sample output: - // Making bulk predictions and evaluating model's quality... - // Micro-accuracy: 0.925925925925926,macro-accuracy = 0.933333333333333 - EvaluateModel(mlContext, testDataset, loadedModel); + // Evaluate the model on the test dataset. + // Sample output: + // Making bulk predictions and evaluating model's quality... + // Micro-accuracy: 0.925925925925926,macro-accuracy = 0.933333333333333 + EvaluateModel(mlContext, testDataset, loadedModel); - // Predict on a single image class using an in-memory image. - // Sample output: - // Scores : [0.8657553,0.006911285,1.46484E-05,0.1266835,0.0006352618], Predicted Label : daisy - TrySinglePrediction(fullImagesetFolderPath, mlContext, loadedModel); + // Predict on a single image class using an in-memory image. + // Sample output: + // Scores : [0.8657553,0.006911285,1.46484E-05,0.1266835,0.0006352618], Predicted Label : daisy + TrySinglePrediction(fullImagesetFolderPath, mlContext, loadedModel); - Console.WriteLine("Prediction on a single image finished."); + Console.WriteLine("Prediction on a single image finished."); + } + catch(OperationCanceledException e) + { + Console.WriteLine($"{nameof(OperationCanceledException)} thrown with message: {e.Message}"); + } + finally + { + Console.WriteLine("Exiting Example Method"); + } - Console.WriteLine("Press any key to finish"); - Console.ReadKey(); } private static void MlContext_Log(object sender, LoggingEventArgs e) diff --git a/docs/samples/Microsoft.ML.Samples/Microsoft.ML.Samples.csproj b/docs/samples/Microsoft.ML.Samples/Microsoft.ML.Samples.csproj index da47c54d21..cc67129067 100644 --- a/docs/samples/Microsoft.ML.Samples/Microsoft.ML.Samples.csproj +++ b/docs/samples/Microsoft.ML.Samples/Microsoft.ML.Samples.csproj @@ -25,6 +25,7 @@ + diff --git a/docs/samples/Microsoft.ML.Samples/Program.cs b/docs/samples/Microsoft.ML.Samples/Program.cs index 61c0172aae..7d2a342b3d 100644 --- a/docs/samples/Microsoft.ML.Samples/Program.cs +++ b/docs/samples/Microsoft.ML.Samples/Program.cs @@ -10,7 +10,7 @@ public static class Program internal static void RunAll() { - ImageClassificationDefault2.Example(); + ImageClassificationDefault2.MyMain(); } } } From 9a5afbda13e4138046ab5fa1d1c2785b8d0e5e50 Mon Sep 17 00:00:00 2001 From: Antonio Velazquez Date: Mon, 13 Jan 2020 11:22:42 -0800 Subject: [PATCH 04/11] Added CheckAlive() Checkpoints --- src/Microsoft.ML.Vision/ImageClassificationTrainer.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Microsoft.ML.Vision/ImageClassificationTrainer.cs b/src/Microsoft.ML.Vision/ImageClassificationTrainer.cs index eba8029fdb..14753c0439 100644 --- a/src/Microsoft.ML.Vision/ImageClassificationTrainer.cs +++ b/src/Microsoft.ML.Vision/ImageClassificationTrainer.cs @@ -647,6 +647,8 @@ private protected override MulticlassPredictionTransformer Date: Mon, 13 Jan 2020 14:43:31 -0800 Subject: [PATCH 05/11] Restore Samples' Program.cs back to original and delete test sample --- .../ImageClassificationDefault2.cs | 399 ------------------ docs/samples/Microsoft.ML.Samples/Program.cs | 18 +- 2 files changed, 16 insertions(+), 401 deletions(-) delete mode 100644 docs/samples/Microsoft.ML.Samples/ImageClassificationDefault2.cs diff --git a/docs/samples/Microsoft.ML.Samples/ImageClassificationDefault2.cs b/docs/samples/Microsoft.ML.Samples/ImageClassificationDefault2.cs deleted file mode 100644 index 157aa1be73..0000000000 --- a/docs/samples/Microsoft.ML.Samples/ImageClassificationDefault2.cs +++ /dev/null @@ -1,399 +0,0 @@ - -using System; -using System.Collections.Generic; -using System.IO; -using System.IO.Compression; -using System.Linq; -using System.Net; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.ML; -using Microsoft.ML.Data; -using Microsoft.ML.Experimental; -using static Microsoft.ML.DataOperationsCatalog; - -namespace Samples.Dynamic -{ - public class ImageClassificationDefault2 - { - public static void MyMain() - { - MLContext mlContext = new MLContext(seed: 1); - var kbTask = Task.Run(() => - { - ConsoleListener(mlContext); - }); - - // thread that performs main work - Task.Run(() => { - Example(mlContext); - }); - - Console.WriteLine("Type commands followed by 'ENTER'"); - Console.WriteLine("Enter 'C' to end program."); - Console.WriteLine(); - - // keep Console running until cancellation token is invoked - kbTask.Wait(); - } - - - static void ConsoleListener(MLContext ctx) - { - while (true) - { - string userInput = Console.ReadLine(); - if (userInput == "c") - { - Console.WriteLine("Cancelation Requested"); - ctx.CancelExecution(); - GC.Collect(); - } - else if (userInput == "q") - { - break; - } - else - { - // handle input - Console.WriteLine("Executing user command {0}...", userInput); - } - } - } - - public static void Example(MLContext mlContext) - { - // Set the path for input images. - string assetsRelativePath = @"../../../assets"; - string assetsPath = GetAbsolutePath(assetsRelativePath); - - string imagesDownloadFolderPath = Path.Combine(assetsPath, "inputs", - "images"); - - //Download the image set and unzip, set the path to image folder. - string finalImagesFolderName = DownloadImageSet( - imagesDownloadFolderPath); - - string fullImagesetFolderPath = Path.Combine( - imagesDownloadFolderPath, finalImagesFolderName); - - // mlContext.Log += MlContext_Log; - - // Load all the original images info - IEnumerable images = LoadImagesFromDirectory( - folder: fullImagesetFolderPath, useFolderNameAsLabel: true); - - // Shuffle images. - IDataView shuffledFullImagesDataset = mlContext.Data.ShuffleRows( - mlContext.Data.LoadFromEnumerable(images)); - - // Apply transforms to the input dataset: - // MapValueToKey : map 'string' type labels to keys - // LoadImages : load raw images to "Image" column - shuffledFullImagesDataset = mlContext.Transforms.Conversion - .MapValueToKey("Label", keyOrdinality: Microsoft.ML.Transforms - .ValueToKeyMappingEstimator.KeyOrdinality.ByValue) - .Append(mlContext.Transforms.LoadRawImageBytes("Image", - fullImagesetFolderPath, "ImagePath")) - .Fit(shuffledFullImagesDataset) - .Transform(shuffledFullImagesDataset); - - // Split the data 90:10 into train and test sets. - TrainTestData trainTestData = mlContext.Data.TrainTestSplit( - shuffledFullImagesDataset, testFraction: 0.1, seed: 1); - - IDataView trainDataset = trainTestData.TrainSet; - IDataView testDataset = trainTestData.TestSet; - - // Create the ImageClassification pipeline by just passing the - // input feature and label column name. - var pipeline = mlContext.MulticlassClassification.Trainers - .ImageClassification(featureColumnName:"Image") - .Append(mlContext.Transforms.Conversion.MapKeyToValue( - outputColumnName: "PredictedLabel", - inputColumnName: "PredictedLabel")); - - Console.WriteLine("*** Training the image classification model " + - "with DNN Transfer Learning on top of the selected " + - "pre-trained model/architecture ***"); - - // Train the model. - // This involves calculating the bottleneck values, and then - // training the final layerSample output is: - // [Source=ImageClassificationTrainer; ImageClassificationTrainer, Kind=Trace] Phase: Bottleneck Computation, Dataset used: Train, Image Index: 1 - // [Source=ImageClassificationTrainer; ImageClassificationTrainer, Kind=Trace] Phase: Bottleneck Computation, Dataset used: Train, Image Index: 2 - // ... - // [Source=ImageClassificationTrainer; ImageClassificationTrainer, Kind=Trace] Phase: Training, Dataset used: Train, Batch Processed Count: 18, Learning Rate: 0.01 Epoch: 0, Accuracy: 0.9, Cross-Entropy: 0.481340 - // ... - // [Source=ImageClassificationTrainer; ImageClassificationTrainer, Kind=Trace] Phase: Training, Dataset used: Train, Batch Processed Count: 18, Learning Rate: 0.004759203 Epoch: 25, Accuracy: 1, Cross-Entropy: 0.04848097 - // [Source=ImageClassificationTrainer; ImageClassificationTrainer, Kind=Trace] Phase: Training, Dataset used: Train, Batch Processed Count: 18, Learning Rate: 0.004473651 Epoch: 26, Accuracy: 1, Cross-Entropy: 0.04930306 - - try - { - var trainedModel = pipeline.Fit(trainDataset); - - Console.WriteLine("Training with transfer learning finished."); - - // Save the trained model. - mlContext.Model.Save(trainedModel, shuffledFullImagesDataset.Schema, - "model.zip"); - - // Load the trained and saved model for prediction. - ITransformer loadedModel; - DataViewSchema schema; - using (var file = File.OpenRead("model.zip")) - loadedModel = mlContext.Model.Load(file, out schema); - - // Evaluate the model on the test dataset. - // Sample output: - // Making bulk predictions and evaluating model's quality... - // Micro-accuracy: 0.925925925925926,macro-accuracy = 0.933333333333333 - EvaluateModel(mlContext, testDataset, loadedModel); - - // Predict on a single image class using an in-memory image. - // Sample output: - // Scores : [0.8657553,0.006911285,1.46484E-05,0.1266835,0.0006352618], Predicted Label : daisy - TrySinglePrediction(fullImagesetFolderPath, mlContext, loadedModel); - - Console.WriteLine("Prediction on a single image finished."); - } - catch(OperationCanceledException e) - { - Console.WriteLine($"{nameof(OperationCanceledException)} thrown with message: {e.Message}"); - } - finally - { - Console.WriteLine("Exiting Example Method"); - } - - } - - private static void MlContext_Log(object sender, LoggingEventArgs e) - { - if (e.Message.StartsWith("[Source=ImageClassificationTrainer;")) - { - Console.WriteLine(e.Message); - } - } - - // Predict on a single image. - private static void TrySinglePrediction(string imagesForPredictions, - MLContext mlContext, ITransformer trainedModel) - { - // Create prediction function to try one prediction. - var predictionEngine = mlContext.Model - .CreatePredictionEngine(trainedModel); - - // Load test images. - IEnumerable testImages = - LoadInMemoryImagesFromDirectory(imagesForPredictions, false); - - // Create an in-memory image object from the first image in the test data. - InMemoryImageData imageToPredict = new InMemoryImageData - { - Image = testImages.First().Image - }; - - // Predict on the single image. - var prediction = predictionEngine.Predict(imageToPredict); - - Console.WriteLine($"Scores : [{string.Join(",", prediction.Score)}], " + - $"Predicted Label : {prediction.PredictedLabel}"); - } - - // Evaluate the trained model on the passed test dataset. - private static void EvaluateModel(MLContext mlContext, - IDataView testDataset, ITransformer trainedModel) - { - Console.WriteLine("Making bulk predictions and evaluating model's " + - "quality..."); - - // Evaluate the model on the test data and get the evaluation metrics. - IDataView predictions = trainedModel.Transform(testDataset); - var metrics = mlContext.MulticlassClassification.Evaluate(predictions); - - Console.WriteLine($"Micro-accuracy: {metrics.MicroAccuracy}," + - $"macro-accuracy = {metrics.MacroAccuracy}"); - - Console.WriteLine("Predicting and Evaluation complete."); - } - - //Load the Image Data from input directory. - public static IEnumerable LoadImagesFromDirectory(string folder, - bool useFolderNameAsLabel = true) - { - var files = Directory.GetFiles(folder, "*", - searchOption: SearchOption.AllDirectories); - foreach (var file in files) - { - if (Path.GetExtension(file) != ".jpg") - continue; - - var label = Path.GetFileName(file); - if (useFolderNameAsLabel) - label = Directory.GetParent(file).Name; - else - { - for (int index = 0; index < label.Length; index++) - { - if (!char.IsLetter(label[index])) - { - label = label.Substring(0, index); - break; - } - } - } - - yield return new ImageData() - { - ImagePath = file, - Label = label - }; - - } - } - - // Load In memory raw images from directory. - public static IEnumerable - LoadInMemoryImagesFromDirectory(string folder, - bool useFolderNameAsLabel = true) - { - var files = Directory.GetFiles(folder, "*", - searchOption: SearchOption.AllDirectories); - foreach (var file in files) - { - if (Path.GetExtension(file) != ".jpg") - continue; - - var label = Path.GetFileName(file); - if (useFolderNameAsLabel) - label = Directory.GetParent(file).Name; - else - { - for (int index = 0; index < label.Length; index++) - { - if (!char.IsLetter(label[index])) - { - label = label.Substring(0, index); - break; - } - } - } - - yield return new InMemoryImageData() - { - Image = File.ReadAllBytes(file), - Label = label - }; - - } - } - - // Download and unzip the image dataset. - public static string DownloadImageSet(string imagesDownloadFolder) - { - // get a set of images to teach the network about the new classes - - //SINGLE SMALL FLOWERS IMAGESET (200 files) - string fileName = "flower_photos_small_set.zip"; - string url = $"https://aka.ms/mlnet-resources/datasets/flower_photos_small_set.zip"; - - Download(url, imagesDownloadFolder, fileName); - UnZip(Path.Combine(imagesDownloadFolder, fileName), imagesDownloadFolder); - - return Path.GetFileNameWithoutExtension(fileName); - } - - // Download file to destination directory from input URL. - public static bool Download(string url, string destDir, string destFileName) - { - if (destFileName == null) - destFileName = url.Split(Path.DirectorySeparatorChar).Last(); - - Directory.CreateDirectory(destDir); - - string relativeFilePath = Path.Combine(destDir, destFileName); - - if (File.Exists(relativeFilePath)) - { - Console.WriteLine($"{relativeFilePath} already exists."); - return false; - } - - var wc = new WebClient(); - Console.WriteLine($"Downloading {relativeFilePath}"); - var download = Task.Run(() => wc.DownloadFile(url, relativeFilePath)); - while (!download.IsCompleted) - { - Thread.Sleep(1000); - Console.Write("."); - } - Console.WriteLine(""); - Console.WriteLine($"Downloaded {relativeFilePath}"); - - return true; - } - - // Unzip the file to destination folder. - public static void UnZip(String gzArchiveName, String destFolder) - { - var flag = gzArchiveName.Split(Path.DirectorySeparatorChar) - .Last() - .Split('.') - .First() + ".bin"; - - if (File.Exists(Path.Combine(destFolder, flag))) return; - - Console.WriteLine($"Extracting."); - ZipFile.ExtractToDirectory(gzArchiveName, destFolder); - - File.Create(Path.Combine(destFolder, flag)); - Console.WriteLine(""); - Console.WriteLine("Extracting is completed."); - } - - // Get absolute path from relative path. - public static string GetAbsolutePath(string relativePath) - { - FileInfo _dataRoot = new FileInfo(typeof( - ImageClassificationDefault).Assembly.Location); - - string assemblyFolderPath = _dataRoot.Directory.FullName; - - string fullPath = Path.Combine(assemblyFolderPath, relativePath); - - return fullPath; - } - - // InMemoryImageData class holding the raw image byte array and label. - public class InMemoryImageData - { - [LoadColumn(0)] - public byte[] Image; - - [LoadColumn(1)] - public string Label; - } - - // ImageData class holding the imagepath and label. - public class ImageData - { - [LoadColumn(0)] - public string ImagePath; - - [LoadColumn(1)] - public string Label; - } - - // ImagePrediction class holding the score and predicted label metrics. - public class ImagePrediction - { - [ColumnName("Score")] - public float[] Score; - - [ColumnName("PredictedLabel")] - public string PredictedLabel; - } - } -} diff --git a/docs/samples/Microsoft.ML.Samples/Program.cs b/docs/samples/Microsoft.ML.Samples/Program.cs index 7d2a342b3d..5c57931b30 100644 --- a/docs/samples/Microsoft.ML.Samples/Program.cs +++ b/docs/samples/Microsoft.ML.Samples/Program.cs @@ -10,7 +10,21 @@ public static class Program internal static void RunAll() { - ImageClassificationDefault2.MyMain(); + int samples = 0; + var types = Assembly.GetExecutingAssembly().GetTypes(); + foreach (var type in types) + { + var sample = type.GetMethod("Example", BindingFlags.Public | BindingFlags.Static | BindingFlags.FlattenHierarchy); + + if (sample != null) + { + Console.WriteLine(type.Name); + sample.Invoke(null, null); + samples++; + } + } + + Console.WriteLine("Number of samples that ran without any exception: " + samples); } } -} +} \ No newline at end of file From 8785b8f4c5139fa197a2d94969c59585f214636d Mon Sep 17 00:00:00 2001 From: Antonio Velazquez Date: Mon, 13 Jan 2020 14:55:36 -0800 Subject: [PATCH 06/11] Samples back to original state --- .../Microsoft.ML.Samples/Microsoft.ML.Samples.csproj | 9 ++++----- docs/samples/Microsoft.ML.Samples/Program.cs | 3 +-- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/docs/samples/Microsoft.ML.Samples/Microsoft.ML.Samples.csproj b/docs/samples/Microsoft.ML.Samples/Microsoft.ML.Samples.csproj index cc67129067..9ad892da1e 100644 --- a/docs/samples/Microsoft.ML.Samples/Microsoft.ML.Samples.csproj +++ b/docs/samples/Microsoft.ML.Samples/Microsoft.ML.Samples.csproj @@ -11,7 +11,7 @@ - + @@ -25,7 +25,6 @@ - @@ -953,18 +952,18 @@ - + DnnImageModels\ResNet18Onnx\ResNet18.onnx PreserveNewest - + DnnImageModels\ResNetPrepOnnx\ResNetPreprocess.onnx PreserveNewest - + \ No newline at end of file diff --git a/docs/samples/Microsoft.ML.Samples/Program.cs b/docs/samples/Microsoft.ML.Samples/Program.cs index 5c57931b30..24674651c7 100644 --- a/docs/samples/Microsoft.ML.Samples/Program.cs +++ b/docs/samples/Microsoft.ML.Samples/Program.cs @@ -11,8 +11,7 @@ public static class Program internal static void RunAll() { int samples = 0; - var types = Assembly.GetExecutingAssembly().GetTypes(); - foreach (var type in types) + foreach (var type in Assembly.GetExecutingAssembly().GetTypes()) { var sample = type.GetMethod("Example", BindingFlags.Public | BindingFlags.Static | BindingFlags.FlattenHierarchy); From f6042ed30ab66e00e761eff3389a7adc09574150 Mon Sep 17 00:00:00 2001 From: Antonio Velazquez Date: Mon, 13 Jan 2020 15:03:25 -0800 Subject: [PATCH 07/11] Reset to original state of samples --- .../Microsoft.ML.Samples/Microsoft.ML.Samples.csproj | 8 ++++---- docs/samples/Microsoft.ML.Samples/Program.cs | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/samples/Microsoft.ML.Samples/Microsoft.ML.Samples.csproj b/docs/samples/Microsoft.ML.Samples/Microsoft.ML.Samples.csproj index 9ad892da1e..da47c54d21 100644 --- a/docs/samples/Microsoft.ML.Samples/Microsoft.ML.Samples.csproj +++ b/docs/samples/Microsoft.ML.Samples/Microsoft.ML.Samples.csproj @@ -11,7 +11,7 @@ - + @@ -952,18 +952,18 @@ - + DnnImageModels\ResNet18Onnx\ResNet18.onnx PreserveNewest - + DnnImageModels\ResNetPrepOnnx\ResNetPreprocess.onnx PreserveNewest - \ No newline at end of file + diff --git a/docs/samples/Microsoft.ML.Samples/Program.cs b/docs/samples/Microsoft.ML.Samples/Program.cs index 24674651c7..4c46399421 100644 --- a/docs/samples/Microsoft.ML.Samples/Program.cs +++ b/docs/samples/Microsoft.ML.Samples/Program.cs @@ -26,4 +26,4 @@ internal static void RunAll() Console.WriteLine("Number of samples that ran without any exception: " + samples); } } -} \ No newline at end of file +} From eef112cbedf1d41a16fc16a57c96374e841f8a76 Mon Sep 17 00:00:00 2001 From: Antonio Velazquez Date: Tue, 14 Jan 2020 11:42:05 -0800 Subject: [PATCH 08/11] Removed checkpoints from unnecessary places --- src/Microsoft.ML.Vision/ImageClassificationTrainer.cs | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/Microsoft.ML.Vision/ImageClassificationTrainer.cs b/src/Microsoft.ML.Vision/ImageClassificationTrainer.cs index 14753c0439..214d2124c2 100644 --- a/src/Microsoft.ML.Vision/ImageClassificationTrainer.cs +++ b/src/Microsoft.ML.Vision/ImageClassificationTrainer.cs @@ -647,8 +647,6 @@ private protected override MulticlassPredictionTransformer Date: Tue, 14 Jan 2020 11:47:44 -0800 Subject: [PATCH 09/11] Adding CheckAlive method with exception handling --- .../ImageClassificationTrainer.cs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/Microsoft.ML.Vision/ImageClassificationTrainer.cs b/src/Microsoft.ML.Vision/ImageClassificationTrainer.cs index 214d2124c2..1d3a59129b 100644 --- a/src/Microsoft.ML.Vision/ImageClassificationTrainer.cs +++ b/src/Microsoft.ML.Vision/ImageClassificationTrainer.cs @@ -1120,6 +1120,19 @@ private void TrainAndEvaluateClassificationLayerCore(int epoch, float learningRa } } + private void CheckAlive() + { + try + { + Host.CheckAlive(); + } + catch(OperationCanceledException e) + { + TryCleanupTemporaryWorkspace(); + throw; + } + } + private void TryCleanupTemporaryWorkspace() { if (_cleanupWorkspace && Directory.Exists(_options.WorkspacePath)) From 53b737b4bb7802347bfd4438a037cf12f5098dae Mon Sep 17 00:00:00 2001 From: Antonio Velazquez Date: Tue, 14 Jan 2020 11:59:14 -0800 Subject: [PATCH 10/11] Added checkpoints with new CheckAlive method --- src/Microsoft.ML.Vision/ImageClassificationTrainer.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.ML.Vision/ImageClassificationTrainer.cs b/src/Microsoft.ML.Vision/ImageClassificationTrainer.cs index 1d3a59129b..5d910cc921 100644 --- a/src/Microsoft.ML.Vision/ImageClassificationTrainer.cs +++ b/src/Microsoft.ML.Vision/ImageClassificationTrainer.cs @@ -835,6 +835,7 @@ private void CacheFeaturizedImagesToDisk(IDataView input, string labelColumnName metrics.Bottleneck.DatasetUsed = dataset; while (cursor.MoveNext()) { + CheckAlive(); labelGetter(ref label); imageGetter(ref image); if (image.Length <= 0) @@ -888,6 +889,7 @@ private void CreateFeaturizedCacheFile(string cacheFilePath, int examples, int f foreach (var row in featurizedImages) { + CheckAlive(); writer.WriteLine(row.Item1 + "," + string.Join(",", row.Item2)); labels[0] = row.Item1; for (int index = 0; index < sizeof(long); index++) @@ -992,7 +994,7 @@ private void TrainAndEvaluateClassificationLayer(string trainBottleneckFilePath, for (int epoch = 0; epoch < epochs; epoch += 1) { - Host.CheckAlive(); + CheckAlive(); // Train. TrainAndEvaluateClassificationLayerCore(epoch, learningRate, featureFileStartOffset, metrics, labelTensorShape, featureTensorShape, batchSize, From 681c13c2cbb10267f5eef4c14d3a0dd9fd643732 Mon Sep 17 00:00:00 2001 From: Antonio Velazquez Date: Tue, 14 Jan 2020 13:02:46 -0800 Subject: [PATCH 11/11] Removed unused exception variable "e" --- src/Microsoft.ML.Vision/ImageClassificationTrainer.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.ML.Vision/ImageClassificationTrainer.cs b/src/Microsoft.ML.Vision/ImageClassificationTrainer.cs index 5d910cc921..4f7b8fab08 100644 --- a/src/Microsoft.ML.Vision/ImageClassificationTrainer.cs +++ b/src/Microsoft.ML.Vision/ImageClassificationTrainer.cs @@ -1128,7 +1128,7 @@ private void CheckAlive() { Host.CheckAlive(); } - catch(OperationCanceledException e) + catch(OperationCanceledException) { TryCleanupTemporaryWorkspace(); throw;