Add a DNN Image Featurizer Transform Estimator

Microsoft.ML.sln

@@ -135,6 +135,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.SamplesUtils",
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.Recommender", "src\Microsoft.ML.Recommender\Microsoft.ML.Recommender.csproj", "{C8E1772B-DFD9-4A4D-830D-6AAB1C668BB3}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.DnnImageFeaturizer.ResNet18", "src\Microsoft.ML.DnnImageFeaturizer.ResNet18\Microsoft.ML.DnnImageFeaturizer.ResNet18.csproj", "{9222FC9D-599A-49A5-B685-08CC9A5C81D7}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -511,6 +513,14 @@ Global
 		{C8E1772B-DFD9-4A4D-830D-6AAB1C668BB3}.Release|Any CPU.Build.0 = Release|Any CPU
 		{C8E1772B-DFD9-4A4D-830D-6AAB1C668BB3}.Release-Intrinsics|Any CPU.ActiveCfg = Release-Intrinsics|Any CPU
 		{C8E1772B-DFD9-4A4D-830D-6AAB1C668BB3}.Release-Intrinsics|Any CPU.Build.0 = Release-Intrinsics|Any CPU
+		{9222FC9D-599A-49A5-B685-08CC9A5C81D7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{9222FC9D-599A-49A5-B685-08CC9A5C81D7}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{9222FC9D-599A-49A5-B685-08CC9A5C81D7}.Debug-Intrinsics|Any CPU.ActiveCfg = Debug-Intrinsics|Any CPU
+		{9222FC9D-599A-49A5-B685-08CC9A5C81D7}.Debug-Intrinsics|Any CPU.Build.0 = Debug-Intrinsics|Any CPU
+		{9222FC9D-599A-49A5-B685-08CC9A5C81D7}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{9222FC9D-599A-49A5-B685-08CC9A5C81D7}.Release|Any CPU.Build.0 = Release|Any CPU
+		{9222FC9D-599A-49A5-B685-08CC9A5C81D7}.Release-Intrinsics|Any CPU.ActiveCfg = Release-Intrinsics|Any CPU
+		{9222FC9D-599A-49A5-B685-08CC9A5C81D7}.Release-Intrinsics|Any CPU.Build.0 = Release-Intrinsics|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -567,6 +577,7 @@ Global
 		{ECB71297-9DF1-48CE-B93A-CD969221F9B6} = {DA452A53-2E94-4433-B08C-041EDEC729E6}
 		{11A5210E-2EA7-42F1-80DB-827762E9C781} = {09EADF06-BE25-4228-AB53-95AE3E15B530}
 		{C8E1772B-DFD9-4A4D-830D-6AAB1C668BB3} = {09EADF06-BE25-4228-AB53-95AE3E15B530}
+		{9222FC9D-599A-49A5-B685-08CC9A5C81D7} = {09EADF06-BE25-4228-AB53-95AE3E15B530}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {41165AF1-35BB-4832-A189-73060F82B01D}

pkg/Microsoft.ML.DnnImageFeaturizer.ResNet18/Microsoft.ML.DnnImageFeaturizer.ResNet18.nupkgproj

@@ -0,0 +1,17 @@
+<Project Sdk="Microsoft.NET.Sdk" DefaultTargets="Pack">
+
+  <PropertyGroup>
+    <TargetFramework>netstandard2.0</TargetFramework>
+    <PackageDescription>ML.NET component for pretrained ResNet18 image featurization</PackageDescription>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="../Microsoft.ML.OnnxTransform/Microsoft.ML.OnnxTransform.nupkgproj" />
+  </ItemGroup>
+
+  <ItemGroup>  
+    <Content Include="$(LocalAppData)\mlnet-resources\ResNet18Onnx\ResNet18.onnx" Pack="true" PackagePath=".\lib\netstandard2.0\ResNet18Onnx" /> 
+    <Content Include="$(LocalAppData)\mlnet-resources\ResNetPrepOnnx\ResNetPreprocess.onnx" Pack="true" PackagePath=".\lib\netstandard2.0\ResNetPrepOnnx" /> 
+  </ItemGroup>  
+
+</Project>

pkg/Microsoft.ML.DnnImageFeaturizer.ResNet18/Microsoft.ML.DnnImageFeaturizer.ResNet18.symbols.nupkgproj

@@ -0,0 +1,5 @@
+<Project DefaultTargets="Pack">
+
+  <Import Project="Microsoft.ML.DnnImageFeaturizer.ResNet18.nupkgproj" />
+
+</Project>

src/Microsoft.ML.DnnImageFeaturizer.ResNet18/Microsoft.ML.DnnImageFeaturizer.ResNet18.csproj

@@ -0,0 +1,50 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <UsingTask TaskName="DownloadFilesFromUrl" AssemblyFile="$(ToolsDir)Microsoft.DotNet.Build.Tasks.dll"/>
+
+  <PropertyGroup>
+    <TargetFramework>netstandard2.0</TargetFramework>
+    <IncludeInPackage>Microsoft.ML.DnnImageFeaturizer.ResNet18</IncludeInPackage>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Microsoft.ML.OnnxTransform\Microsoft.ML.OnnxTransform.csproj" />
+  </ItemGroup>
+
+  <PropertyGroup>
+    <LocalAppDataDir>$(LocalAppData)</LocalAppDataDir>
+    <MlnetAppDataDir>$(LocalAppData)/mlnet-resources</MlnetAppDataDir>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PreprocessFile Include="$(MlnetAppDataDir)/ResNetPrepOnnx/ResNetPreprocess.onnx"
+    Url="https://express-tlcresources.azureedge.net/image/ResNetPrepOnnx/ResNetPreprocess.onnx "
+    DestinationFile="$(MlnetAppDataDir)/ResNetPrepOnnx/ResNetPreprocess.onnx"
+    DestinationDir="$(MlnetAppDataDir)/ResNetPrepOnnx" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ModelFile Include="$(MlnetAppDataDir)/ResNet18Onnx/ResNet18.onnx"
+    Url="https://express-tlcresources.azureedge.net/image/ResNet18Onnx/ResNet18.onnx"
+    DestinationFile="$(MlnetAppDataDir)/ResNet18Onnx/ResNet18.onnx"
+    DestinationDir="$(MlnetAppDataDir)/ResNet18Onnx" />
+  </ItemGroup>
+
+  <Target Name="DownloadPrepModels" Inputs="@(PreprocessFile)" Outputs="%(PreprocessFile.DestinationFile)">
+    <Message Importance="High" Text="Downloading external model files... %(PreprocessFile.DestinationFile)" />
+    <MakeDir Directories="$(MlnetAppDataDir)" />
+    <MakeDir Directories="$(LocalAppDataDir)" />
+    <MakeDir Directories="%(PreprocessFile.DestinationDir)" />
+    <DownloadFilesFromUrl Items="@(PreprocessFile)"/>
+  </Target>
+
+  <Target Name="DownloadMainModels" 
+          DependsOnTargets="DownloadPrepModels"
+          AfterTargets="Build"
+          Inputs="@(ModelFile)" Outputs="%(ModelFile.DestinationFile)">
+    <Message Importance="High" Text="Downloading external model files... %(ModelFile.DestinationFile)" />
+    <MakeDir Directories="%(ModelFile.DestinationDir)" />
+    <DownloadFilesFromUrl Items="@(ModelFile)"/>
+  </Target>
+
+</Project>

src/Microsoft.ML.DnnImageFeaturizer.ResNet18/ResNet18.cs

@@ -0,0 +1,50 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using Microsoft.ML.Runtime.Data;
+using System.IO;
+using System.Reflection;
+
+namespace Microsoft.ML.Transforms
+{
+    // This is an extension method to be used with the DnnImageFeaturizerTransform in order to use a pretrained ResNet18 model.
+    // The NuGet containing this extension is also guaranteed to include the binary model file. Note that when building the project
+    // containing this extension method, the corresponding binary model will be downloaded from the CDN at
+    // https://express-tlcresources.azureedge.net/image/ResNetPrepOnnx/ResNetPreprocess.onnx and
+    // https://express-tlcresources.azureedge.net/image/ResNet18Onnx/ResNet18.onnx  and placed into the local app directory
+    // folder under mlnet-resources.
+    public static class ResNet18Extension
+    {
+        // If including this through a NuGet, the location of the model will be the same as of this file. This looks for the model there.
+        // This should be the default way to use ResNet18 if importing the model from a NuGet.
+        public static EstimatorChain<OnnxTransform> ResNet18(this DnnImageModelSelector model)
+        {
+            var modelChain = new EstimatorChain<OnnxTransform>();
+            var tempCol = "onnxDnnPrep";
+            var execDir = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
+
+            var prepEstimator = new OnnxScoringEstimator(model.Env, Path.Combine(execDir, "ResNetPrepOnnx", "ResNetPreprocess.onnx"), model.Input, tempCol);
+            var mainEstimator = new OnnxScoringEstimator(model.Env, Path.Combine(execDir, "ResNet18Onnx", "ResNet18.onnx"), tempCol, model.Output);
+            modelChain = modelChain.Append(prepEstimator);
+            modelChain = modelChain.Append(mainEstimator);
+            return modelChain;
+        }
+
+        // This allows a custom model location to be specified. This is mainly useful for other code inside of ML.NET (such as tests),
+        // but could also be used if one wishes to change the directory of the model for whatever reason. Note that because Onnx models
+        // must be in a directory all by themsleves for the OnnxTransform to work, this method appends a ResNet18Onnx/ResNetPrepOnnx subdirectory
+        // to the passed in directory to prevent having to make that directory manually each time.
+        public static EstimatorChain<OnnxTransform> ResNet18(this DnnImageModelSelector model, string modelDir)
+        {
+            var modelChain = new EstimatorChain<OnnxTransform>();
+            var tempCol = "onnxDnnPrep";
+
+            var prepEstimator = new OnnxScoringEstimator(model.Env, Path.Combine(modelDir, "ResNetPrepOnnx", "ResNetPreprocess.onnx"), model.Input, tempCol);
+            var mainEstimator = new OnnxScoringEstimator(model.Env, Path.Combine(modelDir, "ResNet18Onnx", "ResNet18.onnx"), tempCol, model.Output);
+            modelChain = modelChain.Append(prepEstimator);
+            modelChain = modelChain.Append(mainEstimator);
+            return modelChain;
+        }
+    }
+}

src/Microsoft.ML.OnnxTransform/DnnImageFeaturizerTransform.cs

@@ -0,0 +1,101 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using Microsoft.ML.Core.Data;
+using Microsoft.ML.Runtime;
+using Microsoft.ML.Runtime.Data;
+using Microsoft.ML.Runtime.Internal.Utilities;
+using Microsoft.ML.StaticPipe;
+using Microsoft.ML.StaticPipe.Runtime;
+using System;
+using System.Collections.Generic;
+
+namespace Microsoft.ML.Transforms
+{
+    // This is a helper class that is required to use the DnnImageFeaturizer estimator.
+    // Note that by default, it is not usable as it does not have any valid methods that return an EstimatorChain that
+    // is used by the DnnImageFeaturizeEstimator.
+    // In order to use this, at least one model project with the corresponding extension methods must by included.
+    // See Microsoft.ML.DNNImageFeaturizer.ResNet18 for an example.
+    public class DnnImageModelSelector
+    {
+        public readonly IHostEnvironment Env;
+        public readonly string Input;
+        public readonly string Output;
+
+        public DnnImageModelSelector(IHostEnvironment env, string input, string output)
+        {
+            Env = env;
+            Input = input;
+            Output = output;
+        }
+    }
+
+    // The Dnn Image Featurizer is just a wrapper around two OnnxTransforms with present pretrained DNN models.
+    // Note that because of this, it only works on Windows machines as that is a constraint of the OnnxTransform.
+    public sealed class DnnImageFeaturizerEstimator : IEstimator<TransformerChain<OnnxTransform>>
+    {
+        private readonly EstimatorChain<OnnxTransform> _modelChain;
+
+        // The function passed in to this method is expected to be an extension method on the DnnImageModelSelector class
+        // that creates a chain of two OnnxTransforms with specific models included together with that extension method.
+        // For an example, see Microsoft.ML.DnnImageFeaturizer.ResNet18.
+        public DnnImageFeaturizerEstimator(IHostEnvironment env, Func<DnnImageModelSelector, EstimatorChain<OnnxTransform>> model, string input, string output)
+        {
+            _modelChain = model(new DnnImageModelSelector(env, input, output));
+        }
+
+        public TransformerChain<OnnxTransform> Fit(IDataView input)
+        {
+            return _modelChain.Fit(input);
+        }
+
+        public SchemaShape GetOutputSchema(SchemaShape inputSchema)
+        {
+            return _modelChain.GetOutputSchema(inputSchema);
+        }
+    }
+
+    public static class DnnImageFeaturizerStaticExtensions
+    {
+        private sealed class OutColumn : Vector<float>
+        {
+            public PipelineColumn Input { get; }
+
+            public OutColumn(Vector<float> input, Func<DnnImageModelSelector, EstimatorChain<OnnxTransform>> model)
+                : base(new Reconciler(model), input)
+            {
+                Input = input;
+            }
+        }
+
+        private sealed class Reconciler : EstimatorReconciler
+        {
+            private readonly Func<DnnImageModelSelector, EstimatorChain<OnnxTransform>> _model;
+
+            public Reconciler(Func<DnnImageModelSelector, EstimatorChain<OnnxTransform>> model)
+            {
+                _model = model;
+            }
+
+            public override IEstimator<ITransformer> Reconcile(IHostEnvironment env,
+                PipelineColumn[] toOutput,
+                IReadOnlyDictionary<PipelineColumn, string> inputNames,
+                IReadOnlyDictionary<PipelineColumn, string> outputNames,
+                IReadOnlyCollection<string> usedNames)
+            {
+                Contracts.Assert(toOutput.Length == 1);
+
+                var outCol = (OutColumn)toOutput[0];
+                return new DnnImageFeaturizerEstimator(env, _model, inputNames[outCol.Input], outputNames[outCol]);
+            }
+        }
+
+        public static Vector<float> DnnImageFeaturizer(this Vector<float> input, Func<DnnImageModelSelector, EstimatorChain<OnnxTransform>> model)
+        {
+            Contracts.CheckValue(input, nameof(input));
+            return new OutColumn(input, model);
+        }
+    }
+}