Change test to validate (#6599)

zewditu · web-flow · commit ebb578946fa6 · 2023-04-21T10:49:51.000-07:00
diff --git a/src/Microsoft.ML.AutoML/API/AutoMLExperimentExtension.cs b/src/Microsoft.ML.AutoML/API/AutoMLExperimentExtension.cs
@@ -27,10 +27,10 @@ public static class AutoMLExperimentExtension
         /// <returns><see cref="AutoMLExperiment"/></returns>
         public static AutoMLExperiment SetDataset(this AutoMLExperiment experiment, IDataView train, IDataView validation)
         {
-            var datasetManager = new TrainTestDatasetManager()
+            var datasetManager = new TrainValidateDatasetManager()
             {
                 TrainDataset = train,
-                TestDataset = validation
+                ValidateDataset = validation
             };
 
             experiment.ServiceCollection.AddSingleton<IDatasetManager>(datasetManager);
diff --git a/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs b/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs
@@ -400,12 +400,12 @@ public TrialResult Run(TrialSettings settings)
                     };
                 }
 
-                if (_datasetManager is ITrainTestDatasetManager trainTestDatasetManager)
+                if (_datasetManager is ITrainValidateDatasetManager trainTestDatasetManager)
                 {
                     var stopWatch = new Stopwatch();
                     stopWatch.Start();
                     var model = pipeline.Fit(trainTestDatasetManager.TrainDataset);
-                    var eval = model.Transform(trainTestDatasetManager.TestDataset);
+                    var eval = model.Transform(trainTestDatasetManager.ValidateDataset);
                     var metrics = _context.BinaryClassification.EvaluateNonCalibrated(eval, metricManager.LabelColumn, predictedLabelColumnName: metricManager.PredictedColumn);
                     var metric = GetMetric(metricManager.Metric, metrics);
                     var loss = metricManager.IsMaximize ? -metric : metric;
@@ -426,7 +426,7 @@ public TrialResult Run(TrialSettings settings)
                 }
             }
 
-            throw new ArgumentException($"The runner metric manager is of type {_metricManager.GetType()} which expected to be of type {typeof(ITrainTestDatasetManager)} or {typeof(ICrossValidateDatasetManager)}");
+            throw new ArgumentException($"The runner metric manager is of type {_metricManager.GetType()} which expected to be of type {typeof(ITrainValidateDatasetManager)} or {typeof(ICrossValidateDatasetManager)}");
         }
 
         public Task<TrialResult> RunAsync(TrialSettings settings, CancellationToken ct)
diff --git a/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs b/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs
@@ -394,12 +394,12 @@ public TrialResult Run(TrialSettings settings)
                     };
                 }
 
-                if (_datasetManager is ITrainTestDatasetManager trainTestDatasetManager)
+                if (_datasetManager is ITrainValidateDatasetManager trainTestDatasetManager)
                 {
                     var stopWatch = new Stopwatch();
                     stopWatch.Start();
                     var model = pipeline.Fit(trainTestDatasetManager.TrainDataset);
-                    var eval = model.Transform(trainTestDatasetManager.TestDataset);
+                    var eval = model.Transform(trainTestDatasetManager.ValidateDataset);
                     var metrics = _context.MulticlassClassification.Evaluate(eval, metricManager.LabelColumn, predictedLabelColumnName: metricManager.PredictedColumn);
                     var metric = GetMetric(metricManager.Metric, metrics);
                     var loss = metricManager.IsMaximize ? -metric : metric;
@@ -420,7 +420,7 @@ public TrialResult Run(TrialSettings settings)
                 }
             }
 
-            throw new ArgumentException($"The runner metric manager is of type {_metricManager.GetType()} which expected to be of type {typeof(ITrainTestDatasetManager)} or {typeof(ICrossValidateDatasetManager)}");
+            throw new ArgumentException($"The runner metric manager is of type {_metricManager.GetType()} which expected to be of type {typeof(ITrainValidateDatasetManager)} or {typeof(ICrossValidateDatasetManager)}");
         }
 
         public Task<TrialResult> RunAsync(TrialSettings settings, CancellationToken ct)
diff --git a/src/Microsoft.ML.AutoML/API/RegressionExperiment.cs b/src/Microsoft.ML.AutoML/API/RegressionExperiment.cs
@@ -421,12 +421,12 @@ public Task<TrialResult> RunAsync(TrialSettings settings, CancellationToken ct)
                             } as TrialResult);
                         }
 
-                        if (_datasetManager is ITrainTestDatasetManager trainTestDatasetManager)
+                        if (_datasetManager is ITrainValidateDatasetManager trainTestDatasetManager)
                         {
                             var stopWatch = new Stopwatch();
                             stopWatch.Start();
                             var model = pipeline.Fit(trainTestDatasetManager.TrainDataset);
-                            var eval = model.Transform(trainTestDatasetManager.TestDataset);
+                            var eval = model.Transform(trainTestDatasetManager.ValidateDataset);
                             var metrics = _context.Regression.Evaluate(eval, metricManager.LabelColumn, scoreColumnName: metricManager.ScoreColumn);
                             var metric = GetMetric(metricManager.Metric, metrics);
                             var loss = metricManager.IsMaximize ? -metric : metric;
@@ -447,7 +447,7 @@ public Task<TrialResult> RunAsync(TrialSettings settings, CancellationToken ct)
                         }
                     }
 
-                    throw new ArgumentException($"The runner metric manager is of type {_metricManager.GetType()} which expected to be of type {typeof(ITrainTestDatasetManager)} or {typeof(ICrossValidateDatasetManager)}");
+                    throw new ArgumentException($"The runner metric manager is of type {_metricManager.GetType()} which expected to be of type {typeof(ITrainValidateDatasetManager)} or {typeof(ICrossValidateDatasetManager)}");
                 }
             }
             catch (Exception ex) when (ct.IsCancellationRequested)
diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/IDatasetManager.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/IDatasetManager.cs
@@ -19,18 +19,18 @@ internal interface ICrossValidateDatasetManager
         IDataView Dataset { get; set; }
     }
 
-    internal interface ITrainTestDatasetManager
+    internal interface ITrainValidateDatasetManager
     {
         IDataView TrainDataset { get; set; }
 
-        IDataView TestDataset { get; set; }
+        IDataView ValidateDataset { get; set; }
     }
 
-    internal class TrainTestDatasetManager : IDatasetManager, ITrainTestDatasetManager
+    internal class TrainValidateDatasetManager : IDatasetManager, ITrainValidateDatasetManager
     {
         public IDataView TrainDataset { get; set; }
 
-        public IDataView TestDataset { get; set; }
+        public IDataView ValidateDataset { get; set; }
     }
 
     internal class CrossValidateDatasetManager : IDatasetManager, ICrossValidateDatasetManager
diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/SweepablePipelineRunner.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/SweepablePipelineRunner.cs
@@ -66,10 +66,10 @@ public TrialResult Run(TrialSettings settings)
                 };
             }
 
-            if (_datasetManager is ITrainTestDatasetManager trainTestDatasetManager)
+            if (_datasetManager is ITrainValidateDatasetManager trainTestDatasetManager)
             {
                 var model = mlnetPipeline.Fit(trainTestDatasetManager.TrainDataset);
-                var eval = model.Transform(trainTestDatasetManager.TestDataset);
+                var eval = model.Transform(trainTestDatasetManager.ValidateDataset);
                 var metric = _metricManager.Evaluate(_mLContext, eval);
                 stopWatch.Stop();
                 var loss = _metricManager.IsMaximize ? -metric : metric;

Original file line number	Diff line number	Diff line change
`@@ -27,10 +27,10 @@ public static class AutoMLExperimentExtension`
`27`	`27`	`/// <returns><see cref="AutoMLExperiment"/></returns>`
`28`	`28`	`public static AutoMLExperiment SetDataset(this AutoMLExperiment experiment, IDataView train, IDataView validation)`
`29`	`29`	`{`
`30`		`- var datasetManager = new TrainTestDatasetManager()`
	`30`	`+ var datasetManager = new TrainValidateDatasetManager()`
`31`	`31`	`{`
`32`	`32`	`TrainDataset = train,`
`33`		`- TestDataset = validation`
	`33`	`+ ValidateDataset = validation`
`34`	`34`	`};`
`35`	`35`
`36`	`36`	`experiment.ServiceCollection.AddSingleton<IDatasetManager>(datasetManager);`
Original file line number	Diff line number	Diff line change
`@@ -400,12 +400,12 @@ public TrialResult Run(TrialSettings settings)`
`400`	`400`	`};`
`401`	`401`	`}`
`402`	`402`
`403`		`- if (_datasetManager is ITrainTestDatasetManager trainTestDatasetManager)`
	`403`	`+ if (_datasetManager is ITrainValidateDatasetManager trainTestDatasetManager)`
`404`	`404`	`{`
`405`	`405`	`var stopWatch = new Stopwatch();`
`406`	`406`	`stopWatch.Start();`
`407`	`407`	`var model = pipeline.Fit(trainTestDatasetManager.TrainDataset);`
`408`		`- var eval = model.Transform(trainTestDatasetManager.TestDataset);`
	`408`	`+ var eval = model.Transform(trainTestDatasetManager.ValidateDataset);`
`409`	`409`	`var metrics = _context.BinaryClassification.EvaluateNonCalibrated(eval, metricManager.LabelColumn, predictedLabelColumnName: metricManager.PredictedColumn);`
`410`	`410`	`var metric = GetMetric(metricManager.Metric, metrics);`
`411`	`411`	`var loss = metricManager.IsMaximize ? -metric : metric;`
`@@ -426,7 +426,7 @@ public TrialResult Run(TrialSettings settings)`
`426`	`426`	`}`
`427`	`427`	`}`
`428`	`428`
`429`		`- throw new ArgumentException($"The runner metric manager is of type {_metricManager.GetType()} which expected to be of type {typeof(ITrainTestDatasetManager)} or {typeof(ICrossValidateDatasetManager)}");`
	`429`	`+ throw new ArgumentException($"The runner metric manager is of type {_metricManager.GetType()} which expected to be of type {typeof(ITrainValidateDatasetManager)} or {typeof(ICrossValidateDatasetManager)}");`
`430`	`430`	`}`
`431`	`431`
`432`	`432`	`public Task<TrialResult> RunAsync(TrialSettings settings, CancellationToken ct)`
Original file line number	Diff line number	Diff line change
`@@ -394,12 +394,12 @@ public TrialResult Run(TrialSettings settings)`
`394`	`394`	`};`
`395`	`395`	`}`
`396`	`396`
`397`		`- if (_datasetManager is ITrainTestDatasetManager trainTestDatasetManager)`
	`397`	`+ if (_datasetManager is ITrainValidateDatasetManager trainTestDatasetManager)`
`398`	`398`	`{`
`399`	`399`	`var stopWatch = new Stopwatch();`
`400`	`400`	`stopWatch.Start();`
`401`	`401`	`var model = pipeline.Fit(trainTestDatasetManager.TrainDataset);`
`402`		`- var eval = model.Transform(trainTestDatasetManager.TestDataset);`
	`402`	`+ var eval = model.Transform(trainTestDatasetManager.ValidateDataset);`
`403`	`403`	`var metrics = _context.MulticlassClassification.Evaluate(eval, metricManager.LabelColumn, predictedLabelColumnName: metricManager.PredictedColumn);`
`404`	`404`	`var metric = GetMetric(metricManager.Metric, metrics);`
`405`	`405`	`var loss = metricManager.IsMaximize ? -metric : metric;`
`@@ -420,7 +420,7 @@ public TrialResult Run(TrialSettings settings)`
`420`	`420`	`}`
`421`	`421`	`}`
`422`	`422`
`423`		`- throw new ArgumentException($"The runner metric manager is of type {_metricManager.GetType()} which expected to be of type {typeof(ITrainTestDatasetManager)} or {typeof(ICrossValidateDatasetManager)}");`
	`423`	`+ throw new ArgumentException($"The runner metric manager is of type {_metricManager.GetType()} which expected to be of type {typeof(ITrainValidateDatasetManager)} or {typeof(ICrossValidateDatasetManager)}");`
`424`	`424`	`}`
`425`	`425`
`426`	`426`	`public Task<TrialResult> RunAsync(TrialSettings settings, CancellationToken ct)`
Original file line number	Diff line number	Diff line change
`@@ -421,12 +421,12 @@ public Task<TrialResult> RunAsync(TrialSettings settings, CancellationToken ct)`
`421`	`421`	`} as TrialResult);`
`422`	`422`	`}`
`423`	`423`
`424`		`- if (_datasetManager is ITrainTestDatasetManager trainTestDatasetManager)`
	`424`	`+ if (_datasetManager is ITrainValidateDatasetManager trainTestDatasetManager)`
`425`	`425`	`{`
`426`	`426`	`var stopWatch = new Stopwatch();`
`427`	`427`	`stopWatch.Start();`
`428`	`428`	`var model = pipeline.Fit(trainTestDatasetManager.TrainDataset);`
`429`		`- var eval = model.Transform(trainTestDatasetManager.TestDataset);`
	`429`	`+ var eval = model.Transform(trainTestDatasetManager.ValidateDataset);`
`430`	`430`	`var metrics = _context.Regression.Evaluate(eval, metricManager.LabelColumn, scoreColumnName: metricManager.ScoreColumn);`
`431`	`431`	`var metric = GetMetric(metricManager.Metric, metrics);`
`432`	`432`	`var loss = metricManager.IsMaximize ? -metric : metric;`
`@@ -447,7 +447,7 @@ public Task<TrialResult> RunAsync(TrialSettings settings, CancellationToken ct)`
`447`	`447`	`}`
`448`	`448`	`}`
`449`	`449`
`450`		`- throw new ArgumentException($"The runner metric manager is of type {_metricManager.GetType()} which expected to be of type {typeof(ITrainTestDatasetManager)} or {typeof(ICrossValidateDatasetManager)}");`
	`450`	`+ throw new ArgumentException($"The runner metric manager is of type {_metricManager.GetType()} which expected to be of type {typeof(ITrainValidateDatasetManager)} or {typeof(ICrossValidateDatasetManager)}");`
`451`	`451`	`}`
`452`	`452`	`}`
`453`	`453`	`catch (Exception ex) when (ct.IsCancellationRequested)`
Original file line number	Diff line number	Diff line change
`@@ -19,18 +19,18 @@ internal interface ICrossValidateDatasetManager`
`19`	`19`	`IDataView Dataset { get; set; }`
`20`	`20`	`}`
`21`	`21`
`22`		`- internal interface ITrainTestDatasetManager`
	`22`	`+ internal interface ITrainValidateDatasetManager`
`23`	`23`	`{`
`24`	`24`	`IDataView TrainDataset { get; set; }`
`25`	`25`
`26`		`- IDataView TestDataset { get; set; }`
	`26`	`+ IDataView ValidateDataset { get; set; }`
`27`	`27`	`}`
`28`	`28`
`29`		`- internal class TrainTestDatasetManager : IDatasetManager, ITrainTestDatasetManager`
	`29`	`+ internal class TrainValidateDatasetManager : IDatasetManager, ITrainValidateDatasetManager`
`30`	`30`	`{`
`31`	`31`	`public IDataView TrainDataset { get; set; }`
`32`	`32`
`33`		`- public IDataView TestDataset { get; set; }`
	`33`	`+ public IDataView ValidateDataset { get; set; }`
`34`	`34`	`}`
`35`	`35`
`36`	`36`	`internal class CrossValidateDatasetManager : IDatasetManager, ICrossValidateDatasetManager`
Original file line number	Diff line number	Diff line change
`@@ -66,10 +66,10 @@ public TrialResult Run(TrialSettings settings)`
`66`	`66`	`};`
`67`	`67`	`}`
`68`	`68`
`69`		`- if (_datasetManager is ITrainTestDatasetManager trainTestDatasetManager)`
	`69`	`+ if (_datasetManager is ITrainValidateDatasetManager trainTestDatasetManager)`
`70`	`70`	`{`
`71`	`71`	`var model = mlnetPipeline.Fit(trainTestDatasetManager.TrainDataset);`
`72`		`- var eval = model.Transform(trainTestDatasetManager.TestDataset);`
	`72`	`+ var eval = model.Transform(trainTestDatasetManager.ValidateDataset);`
`73`	`73`	`var metric = _metricManager.Evaluate(_mLContext, eval);`
`74`	`74`	`stopWatch.Stop();`
`75`	`75`	`var loss = _metricManager.IsMaximize ? -metric : metric;`