Modify image classification sample to take in-memory image for prediction. (#4310)

* Modify image classification sample to take in-memory image for prediction.

* misc.

Author: codemzs

docs/samples/Microsoft.ML.Samples/Dynamic/ImageClassification/ResnetV2101TransferLearningTrainTestSplit.cs

@@ -46,40 +46,40 @@ public static void Example()
                     mlContext.Data.LoadFromEnumerable(images));
 
                 shuffledFullImagesDataset = mlContext.Transforms.Conversion
-                    .MapValueToKey("Label")
+                        .MapValueToKey("Label")
+                    .Append(mlContext.Transforms.LoadImages("Image", 
+                                fullImagesetFolderPath, false, "ImagePath"))
                     .Fit(shuffledFullImagesDataset)
                     .Transform(shuffledFullImagesDataset);
 
-                // Split the data 90:10 into train and test sets, train and evaluate.
+                // Split the data 90:10 into train and test sets, train and
+                // evaluate.
                 TrainTestData trainTestData = mlContext.Data.TrainTestSplit(
                     shuffledFullImagesDataset, testFraction: 0.1, seed: 1);
 
                 IDataView trainDataset = trainTestData.TrainSet;
                 IDataView testDataset = trainTestData.TestSet;
 
-                var validationSet = mlContext.Transforms.LoadImages("Image", fullImagesetFolderPath, false, "ImagePath") // false indicates we want the image as a VBuffer<byte>
-                    .Fit(testDataset)
-                    .Transform(testDataset);
-
-                var pipeline = mlContext.Transforms.LoadImages("Image", fullImagesetFolderPath, false, "ImagePath") // false indicates we want the image as a VBuffer<byte>
-                    .Append(mlContext.Model.ImageClassification(
+                var pipeline = mlContext.Model.ImageClassification(
                         "Image", "Label",
                         // Just by changing/selecting InceptionV3 here instead of 
-                        // ResnetV2101 you can try a different architecture/pre-trained 
-                        // model. 
+                        // ResnetV2101 you can try a different architecture/
+                        // pre-trained model. 
                         arch: ImageClassificationEstimator.Architecture.ResnetV2101,
                         epoch: 50,
                         batchSize: 10,
                         learningRate: 0.01f,
                         metricsCallback: (metrics) => Console.WriteLine(metrics),
-                        validationSet: validationSet,
+                        validationSet: testDataset,
                         disableEarlyStopping: true)
-                    .Append(mlContext.Transforms.Conversion.MapKeyToValue(outputColumnName: "PredictedLabel", inputColumnName: "PredictedLabel")));
+                    .Append(mlContext.Transforms.Conversion.MapKeyToValue(
+                        outputColumnName: "PredictedLabel", 
+                        inputColumnName: "PredictedLabel"));
 
 
-                Console.WriteLine("*** Training the image classification model with " +
-                    "DNN Transfer Learning on top of the selected pre-trained " +
-                    "model/architecture ***");
+                Console.WriteLine("*** Training the image classification model " +
+                    "with DNN Transfer Learning on top of the selected " +
+                    "pre-trained model/architecture ***");
 
                 // Measuring training time
                 var watch = System.Diagnostics.Stopwatch.StartNew();
@@ -104,6 +104,7 @@ public static void Example()
 
                 watch = System.Diagnostics.Stopwatch.StartNew();
 
+                // Predict image class using an in-memory image.
                 TrySinglePrediction(fullImagesetFolderPath, mlContext, loadedModel);
 
                 watch.Stop();
@@ -126,21 +127,19 @@ private static void TrySinglePrediction(string imagesForPredictions,
         {
             // Create prediction function to try one prediction
             var predictionEngine = mlContext.Model
-                .CreatePredictionEngine<ImageData, ImagePrediction>(trainedModel);
+                .CreatePredictionEngine<InMemoryImageData, ImagePrediction>(trainedModel);
 
-            IEnumerable<ImageData> testImages = LoadImagesFromDirectory(
+            IEnumerable<InMemoryImageData> testImages = LoadInMemoryImagesFromDirectory(
                 imagesForPredictions, false);
 
-            ImageData imageToPredict = new ImageData
+            InMemoryImageData imageToPredict = new InMemoryImageData
             {
-                ImagePath = testImages.First().ImagePath
+                Image = testImages.First().Image
             };
 
             var prediction = predictionEngine.Predict(imageToPredict);
 
-            Console.WriteLine($"ImageFile : " +
-                $"[{Path.GetFileName(imageToPredict.ImagePath)}], " +
-                $"Scores : [{string.Join(",", prediction.Score)}], " +
+            Console.WriteLine($"Scores : [{string.Join(",", prediction.Score)}], " +
                 $"Predicted Label : {prediction.PredictedLabel}");
         }
 
@@ -201,6 +200,41 @@ public static IEnumerable<ImageData> LoadImagesFromDirectory(string folder,
             }
         }
 
+        public static IEnumerable<InMemoryImageData> 
+            LoadInMemoryImagesFromDirectory(string folder, 
+                bool useFolderNameAsLabel = true)
+        {
+            var files = Directory.GetFiles(folder, "*",
+                searchOption: SearchOption.AllDirectories);
+            foreach (var file in files)
+            {
+                if (Path.GetExtension(file) != ".jpg")
+                    continue;
+
+                var label = Path.GetFileName(file);
+                if (useFolderNameAsLabel)
+                    label = Directory.GetParent(file).Name;
+                else
+                {
+                    for (int index = 0; index < label.Length; index++)
+                    {
+                        if (!char.IsLetter(label[index]))
+                        {
+                            label = label.Substring(0, index);
+                            break;
+                        }
+                    }
+                }
+
+                yield return new InMemoryImageData()
+                {
+                    Image = File.ReadAllBytes(file),
+                    Label = label
+                };
+
+            }
+        }
+
         public static string DownloadImageSet(string imagesDownloadFolder)
         {
             // get a set of images to teach the network about the new classes
@@ -285,6 +319,15 @@ public static string GetAbsolutePath(string relativePath)
             return fullPath;
         }
 
+        public class InMemoryImageData
+        {
+            [LoadColumn(0)]
+            public byte[] Image;
+
+            [LoadColumn(1)]
+            public string Label;
+        }
+
         public class ImageData
         {
             [LoadColumn(0)]

src/Microsoft.ML.Dnn/ImageClassificationTransform.cs

@@ -1165,11 +1165,6 @@ public sealed class BottleneckMetrics
             /// </summary>
             public ImageClassificationMetrics.Dataset DatasetUsed { get; set; }
 
-            /// <summary>
-            /// Name of the input image.
-            /// </summary>
-            public string Name { get; set; }
-
             /// <summary>
             /// Index of the input image.
             /// </summary>
@@ -1178,7 +1173,7 @@ public sealed class BottleneckMetrics
             /// <summary>
             /// String representation of the metrics.
             /// </summary>
-            public override string ToString() => $"Phase: Bottleneck Computation, Dataset used: {DatasetUsed.ToString(),10}, Image Index: {Index,3}, Image Name: {Name}";
+            public override string ToString() => $"Phase: Bottleneck Computation, Dataset used: {DatasetUsed.ToString(),10}, Image Index: {Index,3}";
         }
 
         /// <summary>