CLI -- consume logs from AutoML SDK (dotnet#349)

Author: daholste

src/Microsoft.ML.Auto/DebugLogger.cs

@@ -6,12 +6,12 @@ namespace Microsoft.ML.Auto
 {
     internal interface IDebugLogger
     {
-        void Log(DebugStream stream, string message);
+        void Log(LogSeverity logLevel, string message);
     }
 
-    internal enum DebugStream
+    internal enum LogSeverity
     {
-        Exception,
-        RunResult
+        Error,
+        Debug
     }
 }

src/Microsoft.ML.Auto/Experiment/Experiment.cs

@@ -68,7 +68,7 @@ public IList<TRunDetails> Execute()
                 }
 
                 // evaluate pipeline
-                WriteDebugLog(DebugStream.RunResult, $"Evaluating pipeline {pipeline.ToString()}");
+                Log(LogSeverity.Debug, $"Evaluating pipeline {pipeline.ToString()}");
                 (SuggestedPipelineRunDetails suggestedPipelineRunDetails, TRunDetails runDetails)
                     = _runner.Run(pipeline, _modelDirectory, _history.Count + 1);
                 _history.Add(suggestedPipelineRunDetails);
@@ -128,27 +128,23 @@ private void ReportProgress(TRunDetails iterationResult)
             }
             catch (Exception ex)
             {
-                WriteDebugLog(DebugStream.Exception, $"Progress report callback reported exception {ex}");
+                Log(LogSeverity.Error, $"Progress report callback reported exception {ex}");
             }
         }
 
         private void WriteIterationLog(SuggestedPipeline pipeline, SuggestedPipelineRunDetails runResult, Stopwatch stopwatch)
         {
-            // debug log pipeline result
-            if (runResult.RunSucceded)
-            {
-                WriteDebugLog(DebugStream.RunResult, $"{_history.Count}\t{runResult.Score}\t{stopwatch.Elapsed}\t{pipeline.ToString()}");
-            }
+            Log(LogSeverity.Debug, $"{_history.Count}\t{runResult.Score}\t{stopwatch.Elapsed}\t{pipeline.ToString()}");
         }
 
-        private void WriteDebugLog(DebugStream stream, string message)
+        private void Log(LogSeverity severity, string message)
         {
             if(_experimentSettings?.DebugLogger == null)
             {
                 return;
             }
 
-            _experimentSettings.DebugLogger.Log(stream, message);
+            _experimentSettings.DebugLogger.Log(severity, message);
         }
     }
 }

src/Microsoft.ML.Auto/Experiment/Runners/RunnerUtil.cs

@@ -44,7 +44,7 @@ public static (ModelContainer model, TMetrics metrics, Exception exception, doub
             }
             catch (Exception ex)
             {
-                logger?.Log(DebugStream.Exception, $"Pipeline crashed: {pipeline.ToString()} . Exception: {ex}");
+                logger?.Log(LogSeverity.Error, $"Pipeline crashed: {pipeline.ToString()} . Exception: {ex}");
                 return (null, null, ex, double.NaN);
             }
         }

src/mlnet/AutoML/AutoMLDebugLogger.cs

@@ -0,0 +1,21 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using Microsoft.ML.Auto;
+using NLog;
+
+namespace Microsoft.ML.CLI.AutoML
+{
+    internal class AutoMLDebugLogger: IDebugLogger
+    {
+        public static AutoMLDebugLogger Instance = new AutoMLDebugLogger();
+
+        private static Logger logger = LogManager.GetCurrentClassLogger();
+
+        public void Log(LogSeverity severity, string message)
+        {
+            logger.Log(LogLevel.Trace, message);
+        }
+    }
+}

src/mlnet/AutoML/AutoMLEngine.cs

@@ -2,9 +2,9 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
-using System;
 using System.Collections.Generic;
 using Microsoft.ML.Auto;
+using Microsoft.ML.CLI.AutoML;
 using Microsoft.ML.CLI.Data;
 using Microsoft.ML.CLI.ShellProgressBar;
 using Microsoft.ML.CLI.Utilities;
@@ -53,7 +53,8 @@ IEnumerable<RunDetails<BinaryClassificationMetrics>> IAutoMLEngine.ExploreBinary
                 {
                     MaxExperimentTimeInSeconds = settings.MaxExplorationTime,
                     CacheBeforeTrainer = this.enableCaching,
-                    OptimizingMetric = optimizationMetric
+                    OptimizingMetric = optimizationMetric,
+                    DebugLogger = AutoMLDebugLogger.Instance
                 })
                 .Execute(trainData, validationData, columnInformation, progressHandler: progressReporter);
             logger.Log(LogLevel.Trace, Strings.RetrieveBestPipeline);
@@ -68,7 +69,8 @@ IEnumerable<RunDetails<RegressionMetrics>> IAutoMLEngine.ExploreRegressionModels
                 {
                     MaxExperimentTimeInSeconds = settings.MaxExplorationTime,
                     OptimizingMetric = optimizationMetric,
-                    CacheBeforeTrainer = this.enableCaching
+                    CacheBeforeTrainer = this.enableCaching,
+                    DebugLogger = AutoMLDebugLogger.Instance
                 }).Execute(trainData, validationData, columnInformation, progressHandler: progressReporter);
             logger.Log(LogLevel.Trace, Strings.RetrieveBestPipeline);
             return result;
@@ -82,7 +84,8 @@ IEnumerable<RunDetails<MulticlassClassificationMetrics>> IAutoMLEngine.ExploreMu
                 {
                     MaxExperimentTimeInSeconds = settings.MaxExplorationTime,
                     CacheBeforeTrainer = this.enableCaching,
-                    OptimizingMetric = optimizationMetric
+                    OptimizingMetric = optimizationMetric,
+                    DebugLogger = AutoMLDebugLogger.Instance
                 }).Execute(trainData, validationData, columnInformation, progressHandler: progressReporter);
             logger.Log(LogLevel.Trace, Strings.RetrieveBestPipeline);
             return result;