Modify API for advanced settings (RandomizedPcaTrainer)

src/Microsoft.ML.Data/Evaluators/AnomalyDetectionEvaluator.cs

@@ -10,6 +10,7 @@
 using Microsoft.ML;
 using Microsoft.ML.CommandLine;
 using Microsoft.ML.Data;
+using Microsoft.ML.Data.Evaluators.Metrics;
 using Microsoft.ML.EntryPoints;
 using Microsoft.ML.Internal.Utilities;
 using Microsoft.ML.Transforms;
@@ -576,6 +577,43 @@ public void Finish()
                 FinishOtherMetrics();
             }
         }
+
+        /// <summary>
+        /// Evaluates scored anomaly detection data.
+        /// </summary>
+        /// <param name="data">The scored data.</param>
+        /// <param name="label">The name of the label column in <paramref name="data"/>.</param>
+        /// <param name="score">The name of the score column in <paramref name="data"/>.</param>
+        /// <param name="predictedLabel">The name of the predicted label column in <paramref name="data"/>.</param>
+        /// <returns>The evaluation results for these outputs.</returns>
+        public AnomalyDetectionMetrics Evaluate(IDataView data, string label, string score, string predictedLabel)
+        {
+            Host.CheckValue(data, nameof(data));
+            Host.CheckNonEmpty(label, nameof(label));
+            Host.CheckNonEmpty(score, nameof(score));
+            Host.CheckNonEmpty(predictedLabel, nameof(predictedLabel));
+
+            var roles = new RoleMappedData(data, opt: false,
+                RoleMappedSchema.ColumnRole.Label.Bind(label),
+                RoleMappedSchema.CreatePair(MetadataUtils.Const.ScoreValueKind.Score, score),
+                RoleMappedSchema.CreatePair(MetadataUtils.Const.ScoreValueKind.PredictedLabel, predictedLabel));
+
+            var resultDict = ((IEvaluator)this).Evaluate(roles);
+            Host.Assert(resultDict.ContainsKey(MetricKinds.OverallMetrics));
+            var overall = resultDict[MetricKinds.OverallMetrics];
+
+            AnomalyDetectionMetrics result;
+            using (var cursor = overall.GetRowCursorForAllColumns())
+            {
+                var moved = cursor.MoveNext();
+                Host.Assert(moved);
+                result = new AnomalyDetectionMetrics(Host, cursor);
+                moved = cursor.MoveNext();
+                Host.Assert(!moved);
+            }
+            return result;
+        }
+
     }
 
     [BestFriend]

src/Microsoft.ML.Data/Evaluators/Metrics/AnomalyDetectionMetrics.cs

@@ -0,0 +1,37 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using Microsoft.Data.DataView;
+
+namespace Microsoft.ML.Data.Evaluators.Metrics
+{
+    public sealed class AnomalyDetectionMetrics
+    {
+        public double Auc { get; }
+        public double DrAtK { get; }
+        public double DrAtPFpr { get; }
+        public double DrAtNumPos { get; }
+        public double NumAnomalies { get; }
+        public double ThreshAtK { get; }
+        public double ThreshAtP { get; }
+        public double ThreshAtNumPos { get; }
+
+        internal AnomalyDetectionMetrics(IExceptionContext ectx, Row overallResult)
+        {
+            long FetchInt(string name) => RowCursorUtils.Fetch<long>(ectx, overallResult, name);
+            float FetchFloat(string name) => RowCursorUtils.Fetch<float>(ectx, overallResult, name);
+            double FetchDouble(string name) => RowCursorUtils.Fetch<double>(ectx, overallResult, name);
+
+            Auc = FetchDouble(BinaryClassifierEvaluator.Auc);
+            DrAtK = FetchDouble(AnomalyDetectionEvaluator.OverallMetrics.DrAtK);
+            DrAtPFpr = FetchDouble(AnomalyDetectionEvaluator.OverallMetrics.DrAtPFpr);
+            DrAtNumPos = FetchDouble(AnomalyDetectionEvaluator.OverallMetrics.DrAtNumPos);
+            NumAnomalies = FetchInt(AnomalyDetectionEvaluator.OverallMetrics.NumAnomalies);
+            ThreshAtK = FetchFloat(AnomalyDetectionEvaluator.OverallMetrics.ThreshAtK);
+            ThreshAtP = FetchFloat(AnomalyDetectionEvaluator.OverallMetrics.ThreshAtP);
+            ThreshAtNumPos = FetchFloat(AnomalyDetectionEvaluator.OverallMetrics.ThreshAtNumPos);
+        }
+    }
+}

src/Microsoft.ML.Data/MLContext.cs

@@ -40,6 +40,11 @@ public sealed class MLContext : IHostEnvironment
         /// </summary>
         public RankingCatalog Ranking { get; }
 
+        /// <summary>
+        /// Trainers and tasks specific to anomaly detection problems.
+        /// </summary>
+        public AnomalyDetectionCatalog AnomalyDetection { get; }
+
         /// <summary>
         /// Data processing operations.
         /// </summary>
@@ -83,6 +88,7 @@ public MLContext(int? seed = null, int conc = 0)
             Regression = new RegressionCatalog(_env);
             Clustering = new ClusteringCatalog(_env);
             Ranking = new RankingCatalog(_env);
+            AnomalyDetection = new AnomalyDetectionCatalog(_env);
             Transforms = new TransformsCatalog(_env);
             Model = new ModelOperationsCatalog(_env);
             Data = new DataOperationsCatalog(_env);

src/Microsoft.ML.Data/TrainCatalog.cs

@@ -8,6 +8,7 @@
 using Microsoft.Data.DataView;
 using Microsoft.ML.Core.Data;
 using Microsoft.ML.Data;
+using Microsoft.ML.Data.Evaluators.Metrics;
 using Microsoft.ML.Transforms;
 using Microsoft.ML.Transforms.Conversions;
 
@@ -564,4 +565,51 @@ public RankerMetrics Evaluate(IDataView data, string label, string groupId, stri
             return eval.Evaluate(data, label, groupId, score);
         }
     }
+
+    /// <summary>
+    /// The central catalog for anomaly detection tasks and trainers.
+    /// </summary>
+    public sealed class AnomalyDetectionCatalog : TrainCatalogBase
+    {
+        /// <summary>
+        /// The list of trainers for anomaly detection.
+        /// </summary>
+        public AnomalyDetectionTrainers Trainers { get; }
+
+        internal AnomalyDetectionCatalog(IHostEnvironment env)
+            : base(env, nameof(AnomalyDetectionCatalog))
+        {
+            Trainers = new AnomalyDetectionTrainers(this);
+        }
+
+        public sealed class AnomalyDetectionTrainers : CatalogInstantiatorBase
+        {
+            internal AnomalyDetectionTrainers(AnomalyDetectionCatalog catalog)
+                : base(catalog)
+            {
+            }
+        }
+
+        /// <summary>
+        /// Evaluates scored anomaly detection data.
+        /// </summary>
+        /// <param name="data">The scored data.</param>
+        /// <param name="label">The name of the label column in <paramref name="data"/>.</param>
+        /// <param name="score">The name of the score column in <paramref name="data"/>.</param>
+        /// <param name="predictedLabel">The name of the predicted label column in <paramref name="data"/>.</param>
+        /// <returns>The evaluation results for these calibrated outputs.</returns>
+        public AnomalyDetectionMetrics Evaluate(IDataView data, string label = DefaultColumnNames.Label, string score = DefaultColumnNames.Score,
+            string predictedLabel = DefaultColumnNames.PredictedLabel)
+        {
+            Host.CheckValue(data, nameof(data));
+            Host.CheckNonEmpty(label, nameof(label));
+            Host.CheckNonEmpty(score, nameof(score));
+            Host.CheckNonEmpty(predictedLabel, nameof(predictedLabel));
+
+            var args = new AnomalyDetectionEvaluator.Arguments();
+
+            var eval = new AnomalyDetectionEvaluator(Host, args);
+            return eval.Evaluate(data, label, score, predictedLabel);
+        }
+    }
 }

src/Microsoft.ML.PCA/PCACatalog.cs

@@ -3,13 +3,14 @@
 // See the LICENSE file in the project root for more information.
 
 using Microsoft.ML.Data;
+using Microsoft.ML.Trainers.PCA;
 using Microsoft.ML.Transforms.Projections;
+using static Microsoft.ML.Trainers.PCA.RandomizedPcaTrainer;
 
 namespace Microsoft.ML
 {
     public static class PcaCatalog
     {
-
         /// <summary>Initializes a new instance of <see cref="PrincipalComponentAnalysisEstimator"/>.</summary>
         /// <param name="catalog">The transform's catalog.</param>
         /// <param name="outputColumnName">Name of the column resulting from the transformation of <paramref name="inputColumnName"/>.</param>
@@ -35,5 +36,25 @@ public static PrincipalComponentAnalysisEstimator ProjectToPrincipalComponents(t
         /// <param name="columns">Input columns to apply PrincipalComponentAnalysis on.</param>
         public static PrincipalComponentAnalysisEstimator ProjectToPrincipalComponents(this TransformsCatalog.ProjectionTransforms catalog, params PrincipalComponentAnalysisEstimator.ColumnInfo[] columns)
             => new PrincipalComponentAnalysisEstimator(CatalogUtils.GetEnvironment(catalog), columns);
+
+        public static RandomizedPcaTrainer RandomizedPca(this AnomalyDetectionCatalog.AnomalyDetectionTrainers catalog,
+            string featureColumn = DefaultColumnNames.Features,
+            string weights = null,
+            int rank = 20,
+            int oversampling = 20,
+            bool center = true,
+            int? seed = null)
+        {
+            Contracts.CheckValue(catalog, nameof(catalog));
+            var env = CatalogUtils.GetEnvironment(catalog);
+            return new RandomizedPcaTrainer(env, featureColumn, weights, rank, oversampling, center, seed);
+        }
+
+        public static RandomizedPcaTrainer RandomizedPca(this AnomalyDetectionCatalog.AnomalyDetectionTrainers catalog, Options options)
+        {
+            Contracts.CheckValue(catalog, nameof(catalog));
+            var env = CatalogUtils.GetEnvironment(catalog);
+            return new RandomizedPcaTrainer(env, options);
+        }
     }
 }

src/Microsoft.ML.PCA/PcaTrainer.cs

@@ -19,7 +19,7 @@
 using Microsoft.ML.Trainers.PCA;
 using Microsoft.ML.Training;
 
-[assembly: LoadableClass(RandomizedPcaTrainer.Summary, typeof(RandomizedPcaTrainer), typeof(RandomizedPcaTrainer.Arguments),
+[assembly: LoadableClass(RandomizedPcaTrainer.Summary, typeof(RandomizedPcaTrainer), typeof(RandomizedPcaTrainer.Options),
     new[] { typeof(SignatureAnomalyDetectorTrainer), typeof(SignatureTrainer) },
     RandomizedPcaTrainer.UserNameValue,
     RandomizedPcaTrainer.LoadNameValue,
@@ -49,7 +49,7 @@ public sealed class RandomizedPcaTrainer : TrainerEstimatorBase<AnomalyPredictio
         internal const string Summary = "This algorithm trains an approximate PCA using Randomized SVD algorithm. "
             + "This PCA can be made into Kernel PCA by using Random Fourier Features transform.";
 
-        public class Arguments : UnsupervisedLearnerInputBaseWithWeight
+        public class Options : UnsupervisedLearnerInputBaseWithWeight
         {
             [Argument(ArgumentType.AtMostOnce, HelpText = "The number of components in the PCA", ShortName = "k", SortOrder = 50)]
             [TGUI(SuggestedSweeps = "10,20,40,80")]
@@ -91,7 +91,7 @@ public class Arguments : UnsupervisedLearnerInputBaseWithWeight
         /// <param name="oversampling">Oversampling parameter for randomized PCA training.</param>
         /// <param name="center">If enabled, data is centered to be zero mean.</param>
         /// <param name="seed">The seed for random number generation.</param>
-        public RandomizedPcaTrainer(IHostEnvironment env,
+        internal RandomizedPcaTrainer(IHostEnvironment env,
             string features,
             string weights = null,
             int rank = 20,
@@ -103,23 +103,23 @@ public RandomizedPcaTrainer(IHostEnvironment env,
 
         }
 
-        internal RandomizedPcaTrainer(IHostEnvironment env, Arguments args)
-            :this(env, args, args.FeatureColumn, args.WeightColumn)
+        internal RandomizedPcaTrainer(IHostEnvironment env, Options options)
+            :this(env, options, options.FeatureColumn, options.WeightColumn)
         {
 
         }
 
-        private RandomizedPcaTrainer(IHostEnvironment env, Arguments args, string featureColumn, string weightColumn,
+        private RandomizedPcaTrainer(IHostEnvironment env, Options options, string featureColumn, string weightColumn,
             int rank = 20, int oversampling = 20, bool center = true, int? seed = null)
             : base(Contracts.CheckRef(env, nameof(env)).Register(LoadNameValue), TrainerUtils.MakeR4VecFeature(featureColumn), default, TrainerUtils.MakeR4ScalarWeightColumn(weightColumn))
         {
             // if the args are not null, we got here from maml, and the internal ctor.
-            if (args != null)
+            if (options != null)
             {
-                _rank = args.Rank;
-                _center = args.Center;
-                _oversampling = args.Oversampling;
-                _seed = args.Seed ?? Host.Rand.Next();
+                _rank = options.Rank;
+                _center = options.Center;
+                _oversampling = options.Oversampling;
+                _seed = options.Seed ?? Host.Rand.Next();
             }
             else
             {
@@ -347,14 +347,14 @@ protected override AnomalyPredictionTransformer<PcaModelParameters> MakeTransfor
             Desc = "Train an PCA Anomaly model.",
             UserName = UserNameValue,
             ShortName = ShortName)]
-        internal static CommonOutputs.AnomalyDetectionOutput TrainPcaAnomaly(IHostEnvironment env, Arguments input)
+        internal static CommonOutputs.AnomalyDetectionOutput TrainPcaAnomaly(IHostEnvironment env, Options input)
         {
             Contracts.CheckValue(env, nameof(env));
             var host = env.Register("TrainPCAAnomaly");
             host.CheckValue(input, nameof(input));
             EntryPointUtils.CheckInputArgs(host, input);
 
-            return LearnerEntryPointsUtils.Train<Arguments, CommonOutputs.AnomalyDetectionOutput>(host, input,
+            return LearnerEntryPointsUtils.Train<Options, CommonOutputs.AnomalyDetectionOutput>(host, input,
                 () => new RandomizedPcaTrainer(host, input),
                 getWeight: () => LearnerEntryPointsUtils.FindColumn(host, input.TrainingData.Schema, input.WeightColumn));
         }

src/Microsoft.ML.PCA/Properties/AssemblyInfo.cs

@@ -5,6 +5,7 @@
 using System.Runtime.CompilerServices;
 using Microsoft.ML;
 
+[assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.Tests" + PublicKey.TestValue)]
 [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.StaticPipe" + PublicKey.Value)]
 [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.Core.Tests" + PublicKey.TestValue)]
 

test/BaselineOutput/Common/EntryPoints/core_ep-list.tsv

@@ -63,7 +63,7 @@ Trainers.LogisticRegressionClassifier	Logistic Regression is a method in statist
 Trainers.NaiveBayesClassifier	Train a MultiClassNaiveBayesTrainer.	Microsoft.ML.Trainers.MultiClassNaiveBayesTrainer	TrainMultiClassNaiveBayesTrainer	Microsoft.ML.Trainers.MultiClassNaiveBayesTrainer+Arguments	Microsoft.ML.EntryPoints.CommonOutputs+MulticlassClassificationOutput
 Trainers.OnlineGradientDescentRegressor	Train a Online gradient descent perceptron.	Microsoft.ML.Trainers.Online.OnlineGradientDescentTrainer	TrainRegression	Microsoft.ML.Trainers.Online.OnlineGradientDescentTrainer+Options	Microsoft.ML.EntryPoints.CommonOutputs+RegressionOutput
 Trainers.OrdinaryLeastSquaresRegressor	Train an OLS regression model.	Microsoft.ML.Trainers.HalLearners.OlsLinearRegressionTrainer	TrainRegression	Microsoft.ML.Trainers.HalLearners.OlsLinearRegressionTrainer+Options	Microsoft.ML.EntryPoints.CommonOutputs+RegressionOutput
-Trainers.PcaAnomalyDetector	Train an PCA Anomaly model.	Microsoft.ML.Trainers.PCA.RandomizedPcaTrainer	TrainPcaAnomaly	Microsoft.ML.Trainers.PCA.RandomizedPcaTrainer+Arguments	Microsoft.ML.EntryPoints.CommonOutputs+AnomalyDetectionOutput
+Trainers.PcaAnomalyDetector	Train an PCA Anomaly model.	Microsoft.ML.Trainers.PCA.RandomizedPcaTrainer	TrainPcaAnomaly	Microsoft.ML.Trainers.PCA.RandomizedPcaTrainer+Options	Microsoft.ML.EntryPoints.CommonOutputs+AnomalyDetectionOutput
 Trainers.PoissonRegressor	Train an Poisson regression model.	Microsoft.ML.Trainers.PoissonRegression	TrainRegression	Microsoft.ML.Trainers.PoissonRegression+Options	Microsoft.ML.EntryPoints.CommonOutputs+RegressionOutput
 Trainers.StochasticDualCoordinateAscentBinaryClassifier	Train an SDCA binary model.	Microsoft.ML.Trainers.Sdca	TrainBinary	Microsoft.ML.Trainers.SdcaBinaryTrainer+Options	Microsoft.ML.EntryPoints.CommonOutputs+BinaryClassificationOutput
 Trainers.StochasticDualCoordinateAscentClassifier	The SDCA linear multi-class classification trainer.	Microsoft.ML.Trainers.Sdca	TrainMultiClass	Microsoft.ML.Trainers.SdcaMultiClassTrainer+Options	Microsoft.ML.EntryPoints.CommonOutputs+MulticlassClassificationOutput

test/BaselineOutput/SingleDebug/Rff/featurized.tsv

@@ -6,7 +6,7 @@
 #@   col=RffVectorFloat:R4:9-14
 #@ }
 15	8:Label
-5	1	1	1	2	1	3	1	0	-0.157029659	-0.555585265	0.490177631	-0.305056125	0.35670203	-0.453979075
-5	4	4	5	7	10	3	2	0	-0.375955045	0.43816793	-0.5670244	0.108704455	-0.271485656	-0.5095379
-3	1	1	1	2	2	3	1	0	-0.08380841	-0.571235061	0.4856296	-0.312245429	0.389987826	-0.4257262
-6	8	8	1	3	4	3	7	0	-0.2813567	0.504154444	-0.266616732	-0.512102365	-0.5723418	-0.07588247
+5	1	1	1	2	1	3	1	0	0.494028777	0.298778981	0.533874154	-0.219799235	0.5505202	-0.173956484
+5	4	4	5	7	10	3	2	0	0.363161922	-0.4488282	0.5746647	-0.0556217246	0.276754946	0.5066952
+3	1	1	1	2	2	3	1	0	0.5167663	0.257460445	0.5747438	-0.05479813	0.5746628	0.0556417964
+6	8	8	1	3	4	3	7	0	-0.5738443	-0.0635295138	0.3556944	-0.454768956	-0.4006303	-0.415726721

test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs

@@ -3416,7 +3416,7 @@ public void EntryPointPcaPredictorSummary()
                     InputFile = inputFile,
                 }).Data;
 
-                var pcaInput = new RandomizedPcaTrainer.Arguments
+                var pcaInput = new RandomizedPcaTrainer.Options
                 {
                     TrainingData = dataView,
                 };

test/Microsoft.ML.Tests/AnomalyDetectionTests.cs

@@ -0,0 +1,67 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Drawing;
+using System.Drawing.Imaging;
+using System.IO;
+using System.Linq;
+using Microsoft.Data.DataView;
+using Microsoft.ML.Data;
+using Microsoft.ML.ImageAnalytics;
+using Microsoft.ML.Model;
+using Microsoft.ML.RunTests;
+using Xunit;
+using Xunit.Abstractions;
+
+namespace Microsoft.ML.Tests
+{
+    public class AnomalyDetectionTests : TestDataPipeBase
+    {
+        public AnomalyDetectionTests(ITestOutputHelper output) : base(output)
+        {
+        }
+
+        /// <summary>
+        /// RandomizedPcaTrainer test 
+        /// </summary>
+        [Fact]
+        public void RandomizedPcaTrainer()
+        {
+            var mlContext = new MLContext(seed: 1, conc: 1);
+            string featureColumn = "NumericFeatures";
+
+            var reader = new TextLoader(Env, new TextLoader.Arguments()
+            {
+                HasHeader = true,
+                Separator = "\t",
+                Columns = new[]
+                {
+                    new TextLoader.Column("Label", DataKind.R4, 0),
+                    new TextLoader.Column(featureColumn, DataKind.R4, new [] { new TextLoader.Range(1, 784) })
+                }
+            });
+
+            var trainData = reader.Read(GetDataPath(TestDatasets.mnistOneClass.trainFilename));
+            var testData = reader.Read(GetDataPath(TestDatasets.mnistOneClass.testFilename));
+
+            var pipeline = ML.AnomalyDetection.Trainers.RandomizedPca(featureColumn);
+
+            var transformer = pipeline.Fit(trainData);
+            var transformedData = transformer.Transform(testData);
+
+            // Evaluate
+            var metrics = ML.AnomalyDetection.Evaluate(transformedData);
+
+            Assert.Equal(0.99, metrics.Auc, 2);
+            Assert.Equal(0.90, metrics.DrAtK, 2);
+            Assert.Equal(0.90, metrics.DrAtPFpr, 2);
+            Assert.Equal(0.90, metrics.DrAtNumPos, 2);
+            Assert.Equal(10, metrics.NumAnomalies);
+            Assert.Equal(0.57, metrics.ThreshAtK, 2);
+            Assert.Equal(0.63, metrics.ThreshAtP, 2);
+            Assert.Equal(0.65, metrics.ThreshAtNumPos, 2);
+        }
+    }
+}