Add AutoML Interactive Extension

Microsoft.ML.sln

@@ -121,6 +121,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.AutoML", "src\
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.AutoML.Samples", "docs\samples\Microsoft.ML.AutoML.Samples\Microsoft.ML.AutoML.Samples.csproj", "{A6924919-9E37-4023-8B7F-E85C8E3CC9B3}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.AutoML.Interactive", "src\Microsoft.ML.AutoML.Interactive\Microsoft.ML.AutoML.Interactive.csproj", "{3B00090A-B5E4-4570-BCD0-B4CD5D499394}"
+EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.Samples.GPU", "docs\samples\Microsoft.ML.Samples.GPU\Microsoft.ML.Samples.GPU.csproj", "{3C8F910B-7F23-4D25-B521-6D5AC9570ADD}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.Featurizers", "src\Microsoft.ML.Featurizers\Microsoft.ML.Featurizers.csproj", "{E2DD0721-5B0F-4606-8182-4C7EFB834518}"
@@ -155,7 +157,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.SearchSpace.Te
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.AutoML.SourceGenerator", "tools-local\Microsoft.ML.AutoML.SourceGenerator\Microsoft.ML.AutoML.SourceGenerator.csproj", "{C804B990-390E-41D7-8FF1-6774495D70E2}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.ML.TorchSharp", "src\Microsoft.ML.TorchSharp\Microsoft.ML.TorchSharp.csproj", "{FF0BD187-4451-4A3B-934B-2AE3454896E2}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.TorchSharp", "src\Microsoft.ML.TorchSharp\Microsoft.ML.TorchSharp.csproj", "{FF0BD187-4451-4A3B-934B-2AE3454896E2}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -597,6 +599,14 @@ Global
 		{A6924919-9E37-4023-8B7F-E85C8E3CC9B3}.Release|Any CPU.Build.0 = Release|Any CPU
 		{A6924919-9E37-4023-8B7F-E85C8E3CC9B3}.Release|x64.ActiveCfg = Release|Any CPU
 		{A6924919-9E37-4023-8B7F-E85C8E3CC9B3}.Release|x64.Build.0 = Release|Any CPU
+		{3B00090A-B5E4-4570-BCD0-B4CD5D499394}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{3B00090A-B5E4-4570-BCD0-B4CD5D499394}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{3B00090A-B5E4-4570-BCD0-B4CD5D499394}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{3B00090A-B5E4-4570-BCD0-B4CD5D499394}.Debug|x64.Build.0 = Debug|Any CPU
+		{3B00090A-B5E4-4570-BCD0-B4CD5D499394}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{3B00090A-B5E4-4570-BCD0-B4CD5D499394}.Release|Any CPU.Build.0 = Release|Any CPU
+		{3B00090A-B5E4-4570-BCD0-B4CD5D499394}.Release|x64.ActiveCfg = Release|Any CPU
+		{3B00090A-B5E4-4570-BCD0-B4CD5D499394}.Release|x64.Build.0 = Release|Any CPU
 		{3C8F910B-7F23-4D25-B521-6D5AC9570ADD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{3C8F910B-7F23-4D25-B521-6D5AC9570ADD}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{3C8F910B-7F23-4D25-B521-6D5AC9570ADD}.Debug|x64.ActiveCfg = Debug|Any CPU
@@ -796,6 +806,7 @@ Global
 		{C2652287-CD6D-40FB-B042-95FB56D09DB8} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
 		{E48285BF-F49A-4EA3-AED0-1BDDBF77EB80} = {09EADF06-BE25-4228-AB53-95AE3E15B530}
 		{A6924919-9E37-4023-8B7F-E85C8E3CC9B3} = {DA452A53-2E94-4433-B08C-041EDEC729E6}
+		{3B00090A-B5E4-4570-BCD0-B4CD5D499394} = {09EADF06-BE25-4228-AB53-95AE3E15B530}
 		{3C8F910B-7F23-4D25-B521-6D5AC9570ADD} = {DA452A53-2E94-4433-B08C-041EDEC729E6}
 		{E2DD0721-5B0F-4606-8182-4C7EFB834518} = {09EADF06-BE25-4228-AB53-95AE3E15B530}
 		{56CB0850-7341-4D71-9AE4-9EFC472D93DD} = {09EADF06-BE25-4228-AB53-95AE3E15B530}

eng/Versions.props

@@ -35,15 +35,16 @@
     <LightGBMVersion>2.3.1</LightGBMVersion>
     <MicrosoftCodeAnalysisAnalyzersVersion>3.3.0</MicrosoftCodeAnalysisAnalyzersVersion>
     <MicrosoftCodeAnalysisCSharpVersion>3.9.0</MicrosoftCodeAnalysisCSharpVersion>
-    <MicrosoftDotNetInteractiveFormattingVersion>1.0.0-beta.22103.1</MicrosoftDotNetInteractiveFormattingVersion>
-    <MicrosoftDotNetInteractiveVersion>1.0.0-beta.22103.1</MicrosoftDotNetInteractiveVersion>
+    <MicrosoftDotNetInteractiveFormattingVersion>1.0.0-beta.22314.1</MicrosoftDotNetInteractiveFormattingVersion>
+    <MicrosoftDotNetInteractiveVersion>1.0.0-beta.22314.1</MicrosoftDotNetInteractiveVersion>
     <MicrosoftMLFeaturizersVersion>0.4.1</MicrosoftMLFeaturizersVersion>
     <MicrosoftMLOnnxRuntimeVersion>1.10.0</MicrosoftMLOnnxRuntimeVersion>
     <MlNetMklDepsVersion>0.0.0.12</MlNetMklDepsVersion>
     <MSTestTestAdapterVersion>2.1.0</MSTestTestAdapterVersion>
     <MSTestTestFrameworkVersion>2.1.0</MSTestTestFrameworkVersion>
     <NewtonsoftJsonVersion>10.0.3</NewtonsoftJsonVersion>
     <ParquetDotNetVersion>2.1.3</ParquetDotNetVersion>
+    <PlotlyNETCSharpVersion>0.0.1</PlotlyNETCSharpVersion>
     <SharpZipLibVersion>1.3.3</SharpZipLibVersion>
     <TensorflowDotNETVersion>0.20.1</TensorflowDotNETVersion>
     <TensorFlowMajorVersion>2</TensorFlowMajorVersion>

src/Microsoft.ML.AutoML.Interactive/ActionThrottler.cs

@@ -0,0 +1,50 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Microsoft.ML.AutoML
+{
+    internal class ActionThrottler
+    {
+        private readonly Action _action;
+        private readonly TimeSpan _minDelay;
+
+        private DateTime _nextUpdateTime = DateTime.MinValue;
+        private int _updatePending = 0;
+
+        /// <summary>
+        /// This constructor initializes an ActionThrottler that ensures <paramref name="action"/> runs no more than once per <paramref name="minDelay"/>.
+        /// </summary>
+        /// <param name="action">The action to thorttle.</param>
+        /// <param name="minDelay">Timespan to indicate the minimum delay between each time action is executed.</param>
+        public ActionThrottler(Action action, TimeSpan minDelay)
+        {
+            _minDelay = minDelay;
+            _action = action;
+        }
+
+
+        public async Task ExecuteAsync()
+        {
+            if (Interlocked.CompareExchange(ref _updatePending, 1, 0) == 0) // _updatePending is int initialized with 0
+            {
+                DateTime currentTime = DateTime.UtcNow;
+
+                if (_nextUpdateTime > currentTime)
+                {
+                    await Task.Delay(_nextUpdateTime - currentTime);
+                }
+                _action();
+                _nextUpdateTime = DateTime.UtcNow + _minDelay;
+                _updatePending = 0;
+            }
+        }
+    }
+}

src/Microsoft.ML.AutoML.Interactive/AutoMLMonitorKernelExtension.cs

@@ -0,0 +1,103 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using Microsoft.AspNetCore.Html;
+using Microsoft.Data.Analysis;
+using Microsoft.DotNet.Interactive;
+using Microsoft.DotNet.Interactive.Commands;
+using Microsoft.DotNet.Interactive.Formatting;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text.Json;
+using System.Threading.Tasks;
+using Plotly.NET.CSharp;
+using static Microsoft.DotNet.Interactive.Formatting.PocketViewTags;
+
+
+namespace Microsoft.ML.AutoML
+{
+    public class AutoMLMonitorKernelExtension : IKernelExtension
+    {
+        public async Task OnLoadAsync(Kernel kernel)
+        {
+            Formatter.Register<NotebookMonitor>((monitor, writer) =>
+            {
+                WriteSummary(monitor, writer);
+                WriteChart(monitor, writer);
+                WriteTable(monitor, writer);
+            }, "text/html");
+
+            if (Kernel.Root?.FindKernel("csharp") is { } csKernel)
+            {
+                await LoadExtensionApiAsync(csKernel);
+            }
+        }
+
+        private static async Task LoadExtensionApiAsync(Kernel cSharpKernel)
+        {
+            await cSharpKernel.SendAsync(new SubmitCode($@"#r ""{typeof(AutoMLMonitorKernelExtension).Assembly.Location}""
+using {typeof(NotebookMonitor).Namespace};"));
+        }
+
+        private static void WriteSummary(NotebookMonitor monitor, TextWriter writer)
+        {
+
+            var summary = new List<IHtmlContent>();
+
+            if (monitor.BestTrial != null)
+            {
+                var bestTrialParam = JsonSerializer.Serialize(monitor.BestTrial.TrialSettings.Parameter, new JsonSerializerOptions() { WriteIndented = true, });
+                summary.Add(h3("Best Trial"));
+                summary.Add(p($"Id: {monitor.BestTrial.TrialSettings.TrialId}"));
+                summary.Add(p($"Trainer: {monitor.BestTrial.TrialSettings.Pipeline}".Replace("Unknown=>", "")));
+                summary.Add(p($"Parameters: {bestTrialParam}"));
+            }
+            if (monitor.ActiveTrial != null)
+            {
+
+                var activeTrialParam = JsonSerializer.Serialize(monitor.ActiveTrial.Parameter, new JsonSerializerOptions() { WriteIndented = true, });
+
+                summary.Add(h3("Active Trial"));
+                summary.Add(p($"Id: {monitor.ActiveTrial.TrialId}"));
+                summary.Add(p($"Trainer: {monitor.ActiveTrial.Pipeline}".Replace("Unknown=>", "")));
+                summary.Add(p($"Parameters: {activeTrialParam}"));
+            }
+
+            writer.Write(div(summary));
+        }
+
+        private static void WriteChart(NotebookMonitor monitor, TextWriter writer)
+        {
+            var x = monitor.CompletedTrials.Select(x => x.TrialSettings.TrialId);
+            var y = monitor.CompletedTrials.Select(x => x.Metric);
+
+            var chart = Chart.Point<int, double, string>(x, y, "Plot Metrics over Trials.")
+            .WithTraceInfo(ShowLegend: false)
+            .WithXAxisStyle<double, double, string>(TitleText: "Trial", ShowGrid: false)
+            .WithYAxisStyle<double, double, string>(TitleText: "Metric", ShowGrid: false);
+
+            var chartHeader = new List<IHtmlContent>();
+            chartHeader.Add(h3("Plot Metrics over Trials"));
+            writer.Write(div(chartHeader));
+
+
+            Formatter.GetPreferredFormatterFor(typeof(Plotly.NET.GenericChart.GenericChart), "text/html").Format(chart, writer);
+
+            // Works around issue with earlier versions of Plotly.NET - https://github.com/plotly/Plotly.NET/pull/305
+            if (writer.ToString().EndsWith("</div    \r\n"))
+            {
+                writer.Write(">");
+            }
+        }
+
+        private static void WriteTable(NotebookMonitor notebookMonitor, TextWriter writer)
+        {
+            var tableHeader = new List<IHtmlContent>();
+            tableHeader.Add(h3("All Trials Table"));
+            writer.Write(div(tableHeader));
+            Formatter.GetPreferredFormatterFor(typeof(DataFrame), "text/html").Format(notebookMonitor.TrialData, writer);
+        }
+    }
+}

src/Microsoft.ML.AutoML.Interactive/Microsoft.ML.AutoML.Interactive.csproj

@@ -0,0 +1,20 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net6.0</TargetFramework>
+    <IsPackable>false</IsPackable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.DotNet.Interactive" Version="$(MicrosoftDotNetInteractiveVersion)" />
+    <PackageReference Include="Microsoft.DotNet.Interactive.Formatting" Version="$(MicrosoftDotNetInteractiveFormattingVersion)" />
+    <PackageReference Include="Plotly.NET.CSharp" Version="$(PlotlyNETCSharpVersion)" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Microsoft.ML.SearchSpace\Microsoft.ML.SearchSpace.csproj" />
+    <ProjectReference Include="..\Microsoft.Data.Analysis\Microsoft.Data.Analysis.csproj" />
+    <ProjectReference Include="..\Microsoft.ML.AutoML\Microsoft.ML.AutoML.csproj" />
+  </ItemGroup>
+
+</Project>

src/Microsoft.ML.AutoML.Interactive/NotebookMonitor.cs

@@ -0,0 +1,86 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using Microsoft.DotNet.Interactive;
+using System.Collections.Generic;
+using Microsoft.Data.Analysis;
+using System;
+using System.Threading.Tasks;
+using System.Text.Json;
+using System.Threading;
+
+namespace Microsoft.ML.AutoML
+{
+    public class NotebookMonitor : IMonitor
+    {
+        private readonly ActionThrottler _updateThrottler;
+        private DisplayedValue _valueToUpdate;
+
+        public TrialResult BestTrial { get; set; }
+        public TrialResult MostRecentTrial { get; set; }
+        public TrialSettings ActiveTrial { get; set; }
+        public List<TrialResult> CompletedTrials { get; set; }
+        public DataFrame TrialData { get; set; }
+
+        public NotebookMonitor()
+        {
+            CompletedTrials = new List<TrialResult>();
+            TrialData = new DataFrame(new PrimitiveDataFrameColumn<int>("Trial"), new PrimitiveDataFrameColumn<float>("Metric"), new StringDataFrameColumn("Trainer"), new StringDataFrameColumn("Parameters"));
+            _updateThrottler = new ActionThrottler(Update, TimeSpan.FromSeconds(5));
+        }
+
+        public void ReportBestTrial(TrialResult result)
+        {
+            BestTrial = result;
+
+            ThrottledUpdate();
+        }
+
+        public void ReportCompletedTrial(TrialResult result)
+        {
+            MostRecentTrial = result;
+            CompletedTrials.Add(result);
+
+            var activeRunParam = JsonSerializer.Serialize(result.TrialSettings.Parameter, new JsonSerializerOptions() { WriteIndented = false, });
+
+            TrialData.Append(new List<KeyValuePair<string, object>>()
+            {
+                new KeyValuePair<string, object>("Trial",result.TrialSettings.TrialId),
+                new KeyValuePair<string, object>("Metric", result.Metric),
+                new KeyValuePair<string, object>("Trainer",result.TrialSettings.Pipeline.ToString().Replace("Unknown=>","")),
+                new KeyValuePair<string, object>("Parameters",activeRunParam),
+            }, true);
+
+            ThrottledUpdate();
+        }
+
+        public void ReportFailTrial(TrialResult result)
+        {
+            // TODO figure out what to do with failed trials.
+            ThrottledUpdate();
+        }
+
+        public void ReportRunningTrial(TrialSettings setting)
+        {
+            ActiveTrial = setting;
+            ThrottledUpdate();
+        }
+
+        private void ThrottledUpdate()
+        {
+            Task.Run(async () => await _updateThrottler.ExecuteAsync());
+        }
+
+        public void Update()
+        {
+            _valueToUpdate.Update(this);
+        }
+
+        public void SetUpdate(DisplayedValue valueToUpdate)
+        {
+            _valueToUpdate = valueToUpdate;
+            ThrottledUpdate();
+        }
+    }
+}

src/Microsoft.ML.AutoML/Microsoft.ML.AutoML.csproj

@@ -6,8 +6,29 @@
     <IncludeInPackage>Microsoft.ML.AutoML</IncludeInPackage>
     <PackageDescription>ML.NET AutoML: Optimizes an ML pipeline for your dataset, by automatically locating the best feature engineering, model, and hyperparameters</PackageDescription>
     <TargetsForTfmSpecificBuildOutput>$(TargetsForTfmSpecificBuildOutput);CopyProjectReferencesToPackage</TargetsForTfmSpecificBuildOutput>
+
+    <!-- 
+      1591: Documentation warnings
+      NU5100: Warning that gets triggered because a .dll is not placed under lib folder on package. This is by design as we want AutoML Interactive to be under interactive-extensions folder.
+       -->
+    <NoWarn>$(NoWarn);1591;NU5100</NoWarn>
+    <TargetsForTfmSpecificContentInPackage>$(TargetsForTfmSpecificContentInPackage);AddAutoMLInteractiveToInteractiveExtensionsFolder</TargetsForTfmSpecificContentInPackage>
+
   </PropertyGroup>
 
+  <!-- The following properties are set to package AutoML Interactive with the AutoML nuget package. If AutoML Interactive undergoes TFM or dependency changes, we need to update the TargetFramework passed in below-->
+  <Target Name="AddAutoMLInteractiveToInteractiveExtensionsFolder">
+    <MSBuild Projects="./../Microsoft.ML.AutoML.Interactive/Microsoft.ML.AutoML.Interactive.csproj" Targets="_GetBuildOutputFilesWithTfm" Properties="TargetFramework=net6.0">
+      <!-- Manually hardcoding the TargetFramework to net6.0 as that is the one that AutoML Interactive targets -->
+      <Output TaskParameter="TargetOutputs" ItemName="_ItemsToIncludeForInteractive" />
+    </MSBuild>
+
+    <ItemGroup>
+      <_ItemsToIncludeForInteractive Update="@(_ItemsToIncludeForInteractive)" PackagePath="interactive-extensions/dotnet" />
+      <TfmSpecificPackageFile Include="@(_ItemsToIncludeForInteractive)" />
+    </ItemGroup>
+  </Target>
+
   <ItemGroup>
     <ProjectReference Include="..\..\tools-local\Microsoft.ML.AutoML.SourceGenerator\Microsoft.ML.AutoML.SourceGenerator.csproj" OutputItemType="Analyzer" ReferenceOutputAssembly="false" />
     <ProjectReference Include="..\Microsoft.ML.Core\Microsoft.ML.Core.csproj">