Add continuous resource monitoring to AutoML.IMonitor

src/Microsoft.ML.AutoML.Interactive/NotebookMonitor.cs

@@ -24,6 +24,8 @@ public class NotebookMonitor : IMonitor
         public List<TrialResult> CompletedTrials { get; set; }
         public DataFrame TrialData { get; set; }
 
+        public int ResourceUsageCheckInterval => 5000;
+
         public NotebookMonitor(SweepablePipeline pipeline)
         {
             CompletedTrials = new List<TrialResult>();
@@ -84,5 +86,9 @@ public void SetUpdate(DisplayedValue valueToUpdate)
             _valueToUpdate = valueToUpdate;
             ThrottledUpdate();
         }
+
+        public void ReportTrialResourceUsage(TrialSettings setting)
+        {
+        }
     }
 }

src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs

@@ -3,6 +3,7 @@
 // See the LICENSE file in the project root for more information.
 
 using System;
+using System.Diagnostics;
 using System.Linq;
 using System.Text.Json;
 using System.Threading;
@@ -249,19 +250,37 @@ public async Task<TrialResult> RunAsync(CancellationToken ct = default)
             var trialNum = trialResultManager?.GetAllTrialResults().Max(t => t.TrialSettings?.TrialId) + 1 ?? 0;
             var tuner = serviceProvider.GetService<ITuner>();
             Contracts.Assert(tuner != null, "tuner can't be null");
+
             while (!aggregateTrainingStopManager.IsStopTrainingRequested())
             {
-                var setting = new TrialSettings()
+                var trialSettings = new TrialSettings()
                 {
                     TrialId = trialNum++,
                     Parameter = Parameter.CreateNestedParameter(),
+                    StartedAtUtc = DateTime.UtcNow,
+                    CancellationTokenSource = null,
+                    PerformanceMetrics = new TrialPerformanceMetrics(),
                 };
-                var parameter = tuner.Propose(setting);
-                setting.Parameter = parameter;
+                var parameter = tuner.Propose(trialSettings);
+                trialSettings.Parameter = parameter;
 
-                monitor?.ReportRunningTrial(setting);
                 using (var trialCancellationTokenSource = new CancellationTokenSource())
                 {
+                    trialSettings.CancellationTokenSource = trialCancellationTokenSource;
+                    monitor?.ReportRunningTrial(trialSettings);
+
+                    System.Timers.Timer resourceUsageTimer = null;
+                    if ((monitor != null) && (monitor?.ResourceUsageCheckInterval > 0))
+                    {
+                        resourceUsageTimer = new System.Timers.Timer(monitor.ResourceUsageCheckInterval);
+                        resourceUsageTimer.Elapsed += (o, e) =>
+                        {
+                            monitor?.ReportTrialResourceUsage(trialSettings);
+                        };
+                        resourceUsageTimer.AutoReset = true;
+                        resourceUsageTimer.Enabled = false;
+                    }
+
                     void handler(object o, EventArgs e)
                     {
                         // only force-canceling running trials when there's completed trials.
@@ -276,21 +295,25 @@ void handler(object o, EventArgs e)
                         {
                             aggregateTrainingStopManager.OnStopTraining += handler;
 
-                            performanceMonitor.MemoryUsageInMegaByte += (o, m) =>
+                            performanceMonitor.PerformanceMetricsUpdated += (o, metrics) =>
                             {
-                                if (_settings.MaximumMemoryUsageInMegaByte is double d && m > d && !trialCancellationTokenSource.IsCancellationRequested)
+                                trialSettings.PerformanceMetrics = metrics;
+
+                                if (_settings.MaximumMemoryUsageInMegaByte is double d && metrics.PeakMemoryUsage > d && !trialCancellationTokenSource.IsCancellationRequested)
                                 {
-                                    logger.Trace($"cancel current trial {setting.TrialId} because it uses {m} mb memory and the maximum memory usage is {d}");
+                                    logger.Trace($"cancel current trial {trialSettings.TrialId} because it uses {metrics.PeakMemoryUsage} mb memory and the maximum memory usage is {d}");
                                     trialCancellationTokenSource.Cancel();
 
-                                    GC.AddMemoryPressure(Convert.ToInt64(m) * 1024 * 1024);
+                                    GC.AddMemoryPressure(Convert.ToInt64(metrics.PeakMemoryUsage) * 1024 * 1024);
                                     GC.Collect();
                                 }
                             };
 
+                            var trialTask = runner.RunAsync(trialSettings, trialCancellationTokenSource.Token);
                             performanceMonitor.Start();
-                            logger.Trace($"trial setting - {JsonSerializer.Serialize(setting)}");
-                            var trialResult = await runner.RunAsync(setting, trialCancellationTokenSource.Token);
+                            resourceUsageTimer?.Start();
+                            logger.Trace($"trial setting - {JsonSerializer.Serialize(trialSettings)}");
+                            var trialResult = await trialTask;
 
                             var peakCpu = performanceMonitor?.GetPeakCpuUsage();
                             var peakMemoryInMB = performanceMonitor?.GetPeakMemoryUsageInMegaByte();
@@ -313,10 +336,10 @@ void handler(object o, EventArgs e)
                     }
                     catch (OperationCanceledException ex) when (aggregateTrainingStopManager.IsStopTrainingRequested() == false)
                     {
-                        monitor?.ReportFailTrial(setting, ex);
+                        monitor?.ReportFailTrial(trialSettings, ex);
                         var result = new TrialResult
                         {
-                            TrialSettings = setting,
+                            TrialSettings = trialSettings,
                             Loss = double.MaxValue,
                         };
 
@@ -329,7 +352,7 @@ void handler(object o, EventArgs e)
                     }
                     catch (Exception ex)
                     {
-                        monitor?.ReportFailTrial(setting, ex);
+                        monitor?.ReportFailTrial(trialSettings, ex);
 
                         if (!aggregateTrainingStopManager.IsStopTrainingRequested() && _bestTrialResult == null)
                         {
@@ -343,7 +366,7 @@ void handler(object o, EventArgs e)
                     finally
                     {
                         aggregateTrainingStopManager.OnStopTraining -= handler;
-
+                        resourceUsageTimer?.Stop();
                     }
                 }
             }

src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs

@@ -13,13 +13,15 @@ namespace Microsoft.ML.AutoML
     /// </summary>
     public interface IMonitor
     {
+        /// <summary>
+        /// Interval in milliseconds to report resource usage.
+        /// </summary>
+        int ResourceUsageCheckInterval { get; }
         void ReportCompletedTrial(TrialResult result);
-
         void ReportBestTrial(TrialResult result);
-
         void ReportFailTrial(TrialSettings settings, Exception exception = null);
-
-        void ReportRunningTrial(TrialSettings setting);
+        void ReportRunningTrial(TrialSettings settings);
+        void ReportTrialResourceUsage(TrialSettings settings);
     }
 
     /// <summary>
@@ -30,11 +32,14 @@ internal class MLContextMonitor : IMonitor
         private readonly IChannel _logger;
         private readonly List<TrialResult> _completedTrials;
         private readonly SweepablePipeline _pipeline;
-        public MLContextMonitor(IChannel logger, SweepablePipeline pipeline)
+        public int ResourceUsageCheckInterval { get; private set; }
+
+        public MLContextMonitor(IChannel logger, SweepablePipeline pipeline, int resourceUsageCheckInterval = 5000)
         {
             _logger = logger;
             _completedTrials = new List<TrialResult>();
             _pipeline = pipeline;
+            ResourceUsageCheckInterval = resourceUsageCheckInterval;
         }
 
         public virtual void ReportBestTrial(TrialResult result)
@@ -55,7 +60,11 @@ public virtual void ReportFailTrial(TrialSettings settings, Exception exception
 
         public virtual void ReportRunningTrial(TrialSettings setting)
         {
-            _logger.Info($"Update Running Trial - Id: {setting.TrialId} - Pipeline: {_pipeline.ToString(setting.Parameter)}");
+            _logger.Info($"Update Running Trial - Id: {setting.TrialId}");
+        }
+
+        public void ReportTrialResourceUsage(TrialSettings setting)
+        {
         }
     }
 

src/Microsoft.ML.AutoML/AutoMLExperiment/IPerformanceMonitor.cs

@@ -5,6 +5,8 @@
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
+using System.IO;
+using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
 using System.Timers;
@@ -22,9 +24,7 @@ internal interface IPerformanceMonitor : IDisposable
 
         double? GetPeakCpuUsage();
 
-        public event EventHandler<double> CpuUsage;
-
-        public event EventHandler<double> MemoryUsageInMegaByte;
+        public event EventHandler<TrialPerformanceMetrics> PerformanceMetricsUpdated;
     }
 
     internal class DefaultPerformanceMonitor : IPerformanceMonitor
@@ -43,9 +43,7 @@ public DefaultPerformanceMonitor(IChannel logger, int checkIntervalInMillisecond
         }
 
 
-        public event EventHandler<double> CpuUsage;
-
-        public event EventHandler<double> MemoryUsageInMegaByte;
+        public event EventHandler<TrialPerformanceMetrics> PerformanceMetricsUpdated;
 
 
         public void Dispose()
@@ -110,9 +108,18 @@ private void SampleCpuAndMemoryUsage()
                 // calculate Memory Usage in MB
                 var memoryUsage = process.PrivateMemorySize64 * 1.0 / (1024 * 1024);
                 _peakMemoryUsage = Math.Max(memoryUsage, _peakMemoryUsage ?? 0);
+
+                var metrics = new TrialPerformanceMetrics()
+                {
+                    CpuUsage = cpuUsageInTotal,
+                    MemoryUsage = memoryUsage,
+                    PeakCpuUsage = _peakCpuUsage,
+                    PeakMemoryUsage = _peakMemoryUsage
+                };
+
                 _logger?.Trace($"current CPU: {cpuUsageInTotal}, current Memory(mb): {memoryUsage}");
-                MemoryUsageInMegaByte?.Invoke(this, memoryUsage);
-                CpuUsage?.Invoke(this, cpuUsageInTotal);
+
+                PerformanceMetricsUpdated?.Invoke(this, metrics);
             }
         }
     }

src/Microsoft.ML.AutoML/AutoMLExperiment/TrialPerformanceMetrics.cs

@@ -0,0 +1,33 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace Microsoft.ML.AutoML
+{
+    /// <summary>
+    /// Performance metrics for a trial.
+    /// </summary>
+    public class TrialPerformanceMetrics
+    {
+        /// <summary>
+        /// Peak memory usage during the trial in megabytes
+        /// </summary>
+        public double? PeakMemoryUsage { get; set; }
+        /// <summary>
+        /// Peak CPU usage during the trial
+        /// </summary>
+        public double? PeakCpuUsage { get; set; }
+        /// <summary>
+        /// Current CPU usage of the runner process
+        /// </summary>
+        public double CpuUsage { get; internal set; }
+        /// <summary>
+        /// Current memory usage of the runner process in megabytes
+        /// </summary>
+        public double MemoryUsage { get; internal set; }
+    }
+}

src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettings.cs

@@ -2,14 +2,39 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
+using System;
+using System.Runtime.Serialization;
+using System.Text.Json.Serialization;
+using System.Threading;
 using Microsoft.ML.SearchSpace;
 
 namespace Microsoft.ML.AutoML
 {
+    /// <summary>
+    /// Settings used for the trial
+    /// </summary>
     public class TrialSettings
     {
+        /// <summary>
+        /// Identifier of the trial
+        /// </summary>
         public int TrialId { get; set; }
-
+        /// <summary>
+        /// Parameters for the pipeline used in this trial
+        /// </summary>
         public Parameter Parameter { get; set; }
+        /// <summary>
+        /// Cancellation token source to have the ability to cancel the trial
+        /// </summary>
+        [JsonIgnore]
+        public CancellationTokenSource CancellationTokenSource { get; set; }
+        /// <summary>
+        /// Performance metrics of the trial
+        /// </summary>
+        public TrialPerformanceMetrics PerformanceMetrics { get; internal set; }
+        /// <summary>
+        /// The time when the trial started (UTC)
+        /// </summary>
+        public DateTime StartedAtUtc { get; internal set; }
     }
 }

src/Microsoft.ML.SearchSpace/Parameter.cs

@@ -50,7 +50,7 @@ public enum ParameterType
     }
 
     /// <summary>
-    /// <see cref="Parameter"/> is used to save sweeping result from tuner and is used to restore mlnet pipeline from sweepable pipline.
+    /// <see cref="Parameter"/> is used to save sweeping result from tuner and is used to restore mlnet pipeline from sweepable pipeline.
     /// </summary>
     [JsonConverter(typeof(ParameterConverter))]
     public sealed class Parameter : IDictionary<string, Parameter>, IEquatable<Parameter>, IEqualityComparer<Parameter>

test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs

@@ -407,9 +407,7 @@ public DummyPeformanceMonitor()
             _checkIntervalInMilliseconds = 1000;
         }
 
-        public event EventHandler<double> CpuUsage;
-
-        public event EventHandler<double> MemoryUsageInMegaByte;
+        public event EventHandler<TrialPerformanceMetrics> PerformanceMetricsUpdated;
 
         public void Dispose()
         {
@@ -432,8 +430,7 @@ public void Start()
                 _timer = new System.Timers.Timer(_checkIntervalInMilliseconds);
                 _timer.Elapsed += (o, e) =>
                 {
-                    CpuUsage?.Invoke(this, 100);
-                    MemoryUsageInMegaByte?.Invoke(this, 1000);
+                    PerformanceMetricsUpdated?.Invoke(this, new TrialPerformanceMetrics() { PeakCpuUsage = 100, PeakMemoryUsage = 1000 });
                 };
 
                 _timer.AutoReset = true;