From 96aa6ece5f81296a785d0688ef99702d69b8f083 Mon Sep 17 00:00:00 2001 From: Zeeshan Siddiqui Date: Wed, 28 Nov 2018 15:05:43 -0800 Subject: [PATCH 1/6] Sample for IID spike and changepoint detection using time series stateful prediction engine. --- .../IidChangePointDetectorTransform.cs | 91 +++++++++++++++++++ .../Dynamic/IidSpikeDetectorTransform.cs | 83 +++++++++++++++++ 2 files changed, 174 insertions(+) diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/IidChangePointDetectorTransform.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/IidChangePointDetectorTransform.cs index 179dda7444..17a0e680f7 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/IidChangePointDetectorTransform.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/IidChangePointDetectorTransform.cs @@ -4,6 +4,10 @@ using Microsoft.ML.Runtime.Data; using Microsoft.ML.Runtime.Api; using Microsoft.ML.Runtime.TimeSeriesProcessing; +using Microsoft.ML.Core.Data; +using Microsoft.ML.TimeSeries; +using System.IO; +using Microsoft.ML.Data; namespace Microsoft.ML.Samples.Dynamic { @@ -88,5 +92,92 @@ public static void IidChangePointDetectorTransform() // 7 0 7.00 0.50 0.00 // 7 0 7.00 0.50 0.00 } + + // This example creates a time series (list of Data with the i-th element corresponding to the i-th time slot). + // IidChangePointDetector is applied then to identify points where data distribution changed using time series + // prediction engine. The engine is checkpointed and then loaded back from disk into memory and used for prediction. + public static void IidChangePointDetectorPrediction() + { + // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, + // as well as the source of randomness. + var ml = new MLContext(); + + // Generate sample series data with a change + const int size = 16; + var data = new List(size); + for (int i = 0; i < size / 2; i++) + data.Add(new IidChangePointData(5)); + // This is a change point + for (int i = 0; i < size / 2; i++) + data.Add(new IidChangePointData(7)); + + // Convert data to IDataView. + var dataView = ml.CreateStreamingDataView(data); + + // Setup IidSpikeDetector arguments + string outputColumnName = "Prediction"; + string inputColumnName = "Value"; + var args = new IidChangePointDetector.Arguments() + { + Source = inputColumnName, + Name = outputColumnName, + Confidence = 95, // The confidence for spike detection in the range [0, 100] + ChangeHistoryLength = size / 4, // The length of the sliding window on p-values for computing the martingale score. + }; + + // Time Series model. + ITransformer model = new IidChangePointEstimator(ml, args).Fit(dataView); + + // Create a time series prediction engine from the model. + var engine = model.CreateTimeSeriesPredictionFunction(ml); + for(int index = 0; index < 8; index++) + { + // Anomaly change point detection. + var prediction = engine.Predict(new IidChangePointData(5)); + Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}\t{4:0.00}", 5, prediction.Prediction[0], + prediction.Prediction[1], prediction.Prediction[2], prediction.Prediction[3]); + } + + // Change point + var changePointPrediction = engine.Predict(new IidChangePointData(7)); + Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}\t{4:0.00}", 7, changePointPrediction.Prediction[0], + changePointPrediction.Prediction[1], changePointPrediction.Prediction[2], changePointPrediction.Prediction[3]); + + // Checkpoint the model. + var modelPath = "temp.zip"; + engine.CheckPoint(ml, modelPath); + + // Load the model. + using (var file = File.OpenRead(modelPath)) + model = TransformerChain.LoadFrom(ml, file); + + // Create a time series prediction engine from the model. + engine = model.CreateTimeSeriesPredictionFunction(ml); + for (int index = 0; index < 8; index++) + { + // Anomaly change point detection. + var prediction = engine.Predict(new IidChangePointData(7)); + Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}\t{4:0.00}", 7, prediction.Prediction[0], + prediction.Prediction[1], prediction.Prediction[2], prediction.Prediction[3]); + } + + // Data Alert Score P-Value Martingale value + // 5 0 5.00 0.50 0.00 + // 5 0 5.00 0.50 0.00 + // 5 0 5.00 0.50 0.00 + // 5 0 5.00 0.50 0.00 + // 5 0 5.00 0.50 0.00 + // 5 0 5.00 0.50 0.00 + // 5 0 5.00 0.50 0.00 + // 5 0 5.00 0.50 0.00 + // 7 1 7.00 0.00 10298.67 <-- alert is on, predicted changepoint (and model is checkpointed). + // 7 0 7.00 0.13 33950.16 <-- model loaded back from disk and prediction is made. + // 7 0 7.00 0.26 60866.34 + // 7 0 7.00 0.38 78362.04 + // 7 0 7.00 0.50 0.01 + // 7 0 7.00 0.50 0.00 + // 7 0 7.00 0.50 0.00 + // 7 0 7.00 0.50 0.00 + } } } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/IidSpikeDetectorTransform.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/IidSpikeDetectorTransform.cs index 3c6ae7a9c9..78d5d91dd2 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/IidSpikeDetectorTransform.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/IidSpikeDetectorTransform.cs @@ -4,6 +4,10 @@ using Microsoft.ML.Runtime.Data; using Microsoft.ML.Runtime.Api; using Microsoft.ML.Runtime.TimeSeriesProcessing; +using Microsoft.ML.Core.Data; +using Microsoft.ML.TimeSeries; +using System.IO; +using Microsoft.ML.Data; namespace Microsoft.ML.Samples.Dynamic { @@ -83,5 +87,84 @@ public static void IidSpikeDetectorTransform() // 0 5.00 0.50 // 0 5.00 0.50 } + + public static void IidSpikeDetectorPrediction() + { + // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, + // as well as the source of randomness. + var ml = new MLContext(); + + // Generate sample series data with a spike + const int size = 10; + var data = new List(size); + for (int i = 0; i < size / 2; i++) + data.Add(new IidSpikeData(5)); + // This is a spike + data.Add(new IidSpikeData(10)); + for (int i = 0; i < size / 2; i++) + data.Add(new IidSpikeData(5)); + + // Convert data to IDataView. + var dataView = ml.CreateStreamingDataView(data); + + // Setup IidSpikeDetector arguments + string outputColumnName = "Prediction"; + string inputColumnName = "Value"; + var args = new IidSpikeDetector.Arguments() + { + Source = inputColumnName, + Name = outputColumnName, + Confidence = 95, // The confidence for spike detection in the range [0, 100] + PvalueHistoryLength = size / 4 // The size of the sliding window for computing the p-value + }; + + // The transformed model. + ITransformer model = new IidSpikeEstimator(ml, args).Fit(dataView); + + // Create a time series prediction engine from the model. + var engine = model.CreateTimeSeriesPredictionFunction(ml); + for (int index = 0; index < 5; index++) + { + // Anomaly spike detection. + var prediction = engine.Predict(new IidSpikeData(5)); + Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}", 5, prediction.Prediction[0], + prediction.Prediction[1], prediction.Prediction[2]); + } + + // Spike. + var spikePrediction = engine.Predict(new IidSpikeData(10)); + Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}", 10, spikePrediction.Prediction[0], + spikePrediction.Prediction[1], spikePrediction.Prediction[2]); + + // Checkpoint the model. + var modelPath = "temp.zip"; + engine.CheckPoint(ml, modelPath); + + // Load the model. + using (var file = File.OpenRead(modelPath)) + model = TransformerChain.LoadFrom(ml, file); + + for (int index = 0; index < 5; index++) + { + // Anomaly spike detection. + var prediction = engine.Predict(new IidSpikeData(5)); + Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}", 5, prediction.Prediction[0], + prediction.Prediction[1], prediction.Prediction[2]); + } + + // Prediction column obtained post-transformation. + // Datat Alert Score P-Value + // 5 0 5.00 0.50 + // 5 0 5.00 0.50 + // 5 0 5.00 0.50 + // 5 0 5.00 0.50 + // 5 0 5.00 0.50 + // 10 1 10.00 0.00 <-- alert is on, predicted spike (check-point model) + // 5 0 5.00 0.26 <-- load model from disk. + // 5 0 5.00 0.26 + // 5 0 5.00 0.50 + // 5 0 5.00 0.50 + // 5 0 5.00 0.50 + } } } From d5ba57870cdcbc96b6bdd83f3e2499b22104c981 Mon Sep 17 00:00:00 2001 From: Zeeshan Siddiqui Date: Thu, 29 Nov 2018 15:50:43 -0800 Subject: [PATCH 2/6] PR feedback. --- .../IidChangePointDetectorTransform.cs | 28 +++++++++---------- .../Dynamic/IidSpikeDetectorTransform.cs | 3 +- .../IidChangePointDetector.cs | 3 +- .../IidSpikeDetector.cs | 3 +- .../PredictionFunction.cs | 27 ++++++++++++++++++ .../SsaChangePointDetector.cs | 3 +- .../SsaSpikeDetector.cs | 3 +- 7 files changed, 50 insertions(+), 20 deletions(-) diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/IidChangePointDetectorTransform.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/IidChangePointDetectorTransform.cs index 17a0e680f7..771eb26ee5 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/IidChangePointDetectorTransform.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/IidChangePointDetectorTransform.cs @@ -38,26 +38,26 @@ public static void IidChangePointDetectorTransform() var ml = new MLContext(); // Generate sample series data with a change - const int size = 16; - var data = new List(size); - for (int i = 0; i < size / 2; i++) + const int Size = 16; + var data = new List(Size); + for (int i = 0; i < Size / 2; i++) data.Add(new IidChangePointData(5)); // This is a change point - for (int i = 0; i < size / 2; i++) + for (int i = 0; i < Size / 2; i++) data.Add(new IidChangePointData(7)); // Convert data to IDataView. var dataView = ml.CreateStreamingDataView(data); // Setup IidSpikeDetector arguments - string outputColumnName = "Prediction"; - string inputColumnName = "Value"; + string outputColumnName = nameof(ChangePointPrediction.Prediction); + string inputColumnName = nameof(IidChangePointData.Value); var args = new IidChangePointDetector.Arguments() { Source = inputColumnName, Name = outputColumnName, Confidence = 95, // The confidence for spike detection in the range [0, 100] - ChangeHistoryLength = size / 4, // The length of the sliding window on p-values for computing the martingale score. + ChangeHistoryLength = Size / 4, // The length of the sliding window on p-values for computing the martingale score. }; // The transformed data. @@ -103,26 +103,26 @@ public static void IidChangePointDetectorPrediction() var ml = new MLContext(); // Generate sample series data with a change - const int size = 16; - var data = new List(size); - for (int i = 0; i < size / 2; i++) + const int Size = 16; + var data = new List(Size); + for (int i = 0; i < Size / 2; i++) data.Add(new IidChangePointData(5)); // This is a change point - for (int i = 0; i < size / 2; i++) + for (int i = 0; i < Size / 2; i++) data.Add(new IidChangePointData(7)); // Convert data to IDataView. var dataView = ml.CreateStreamingDataView(data); // Setup IidSpikeDetector arguments - string outputColumnName = "Prediction"; - string inputColumnName = "Value"; + string outputColumnName = nameof(ChangePointPrediction.Prediction); + string inputColumnName = nameof(IidChangePointData.Value); var args = new IidChangePointDetector.Arguments() { Source = inputColumnName, Name = outputColumnName, Confidence = 95, // The confidence for spike detection in the range [0, 100] - ChangeHistoryLength = size / 4, // The length of the sliding window on p-values for computing the martingale score. + ChangeHistoryLength = Size / 4, // The length of the sliding window on p-values for computing the martingale score. }; // Time Series model. diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/IidSpikeDetectorTransform.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/IidSpikeDetectorTransform.cs index 78d5d91dd2..3297739cc8 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/IidSpikeDetectorTransform.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/IidSpikeDetectorTransform.cs @@ -152,8 +152,7 @@ public static void IidSpikeDetectorPrediction() prediction.Prediction[1], prediction.Prediction[2]); } - // Prediction column obtained post-transformation. - // Datat Alert Score P-Value + // Data Alert Score P-Value // 5 0 5.00 0.50 // 5 0 5.00 0.50 // 5 0 5.00 0.50 diff --git a/src/Microsoft.ML.TimeSeries/IidChangePointDetector.cs b/src/Microsoft.ML.TimeSeries/IidChangePointDetector.cs index 487a01a4d4..7497fef593 100644 --- a/src/Microsoft.ML.TimeSeries/IidChangePointDetector.cs +++ b/src/Microsoft.ML.TimeSeries/IidChangePointDetector.cs @@ -212,7 +212,8 @@ public sealed class IidChangePointEstimator : TrivialEstimator /// Host Environment. /// Name of the input column. - /// Name of the output column. + /// Name of the output column. Name of the output column. Column is a vector of type double and size 4. + /// The vector contains Alert, Raw Score, P-Value and Martingale score as first four values. /// The confidence for change point detection in the range [0, 100]. /// The length of the sliding window on p-values for computing the martingale score. /// The martingale used for scoring. diff --git a/src/Microsoft.ML.TimeSeries/IidSpikeDetector.cs b/src/Microsoft.ML.TimeSeries/IidSpikeDetector.cs index 7bc8c6b7a2..3fdf968025 100644 --- a/src/Microsoft.ML.TimeSeries/IidSpikeDetector.cs +++ b/src/Microsoft.ML.TimeSeries/IidSpikeDetector.cs @@ -191,7 +191,8 @@ public sealed class IidSpikeEstimator : TrivialEstimator /// /// Host Environment. /// Name of the input column. - /// Name of the output column. + /// Name of the output column. Column is a vector of type double and size 3. + /// The vector contains Alert, Raw Score, P-Value as first three values. /// The confidence for spike detection in the range [0, 100]. /// The size of the sliding window for computing the p-value. /// The argument that determines whether to detect positive or negative anomalies, or both. diff --git a/src/Microsoft.ML.TimeSeries/PredictionFunction.cs b/src/Microsoft.ML.TimeSeries/PredictionFunction.cs index 114aa0e059..603a81c7ce 100644 --- a/src/Microsoft.ML.TimeSeries/PredictionFunction.cs +++ b/src/Microsoft.ML.TimeSeries/PredictionFunction.cs @@ -53,6 +53,12 @@ public sealed class TimeSeriesPredictionFunction : PredictionEngineB private long _rowPosition; private ITransformer InputTransformer { get; set; } + /// + /// Checkpoints to disk with the updated + /// state. + /// + /// Usually + /// Path to file on disk where the updated model needs to be saved. public void CheckPoint(IHostEnvironment env, string modelPath) { using (var file = File.Create(modelPath)) @@ -246,6 +252,27 @@ public override void Predict(TSrc example, ref TDst prediction) public static class PredictionFunctionExtensions { + /// + /// Creates a prediction function/engine for a time series pipeline. + /// It updates the state of time series model with observations seen at prediction phase and allows checkpointing the model./> + /// + /// Class describing input schema to the model. + /// Class describing the output schema of the prediction. + /// The time series pipeline in the form of a + /// Usually + /// + /// + /// + /// + ///

Example code can be found by searching for TimeSeriesPredictionFunction in ML.NET.

+ /// + /// + /// + /// + /// public static TimeSeriesPredictionFunction CreateTimeSeriesPredictionFunction(this ITransformer transformer, IHostEnvironment env, bool ignoreMissingColumns = false, SchemaDefinition inputSchemaDefinition = null, SchemaDefinition outputSchemaDefinition = null) where TSrc : class diff --git a/src/Microsoft.ML.TimeSeries/SsaChangePointDetector.cs b/src/Microsoft.ML.TimeSeries/SsaChangePointDetector.cs index 856ba1a0e3..9171c34662 100644 --- a/src/Microsoft.ML.TimeSeries/SsaChangePointDetector.cs +++ b/src/Microsoft.ML.TimeSeries/SsaChangePointDetector.cs @@ -225,7 +225,8 @@ public sealed class SsaChangePointEstimator : IEstimator /// /// Host Environment. /// Name of the input column. - /// Name of the output column. + /// Name of the output column. Column is a vector of type double and size 4. + /// The vector contains Alert, Raw Score, P-Value and Martingale score as first four values. /// The confidence for change point detection in the range [0, 100]. /// The number of points from the beginning of the sequence used for training. /// The size of the sliding window for computing the p-value. diff --git a/src/Microsoft.ML.TimeSeries/SsaSpikeDetector.cs b/src/Microsoft.ML.TimeSeries/SsaSpikeDetector.cs index 9c3d72cd2f..f98c8bf34b 100644 --- a/src/Microsoft.ML.TimeSeries/SsaSpikeDetector.cs +++ b/src/Microsoft.ML.TimeSeries/SsaSpikeDetector.cs @@ -206,7 +206,8 @@ public sealed class SsaSpikeEstimator : IEstimator /// /// Host Environment. /// Name of the input column. - /// Name of the output column. + /// Name of the output column. Column is a vector of type double and size 3. + /// The vector contains Alert, Raw Score, P-Value as first three values. /// The confidence for spike detection in the range [0, 100]. /// The size of the sliding window for computing the p-value. /// The number of points from the beginning of the sequence used for training. From 5ed8fbec1f22a17f9b0c11bc9aca172815a18c8a Mon Sep 17 00:00:00 2001 From: Zeeshan Siddiqui Date: Fri, 30 Nov 2018 12:57:26 -0800 Subject: [PATCH 3/6] cleanup. --- .../Dynamic/IidSpikeDetectorTransform.cs | 28 +++++++++---------- .../PredictionFunction.cs | 4 +-- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/IidSpikeDetectorTransform.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/IidSpikeDetectorTransform.cs index 3297739cc8..caa6b737fb 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/IidSpikeDetectorTransform.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/IidSpikeDetectorTransform.cs @@ -38,27 +38,27 @@ public static void IidSpikeDetectorTransform() var ml = new MLContext(); // Generate sample series data with a spike - const int size = 10; - var data = new List(size); - for (int i = 0; i < size / 2; i++) + const int Size = 10; + var data = new List(Size); + for (int i = 0; i < Size / 2; i++) data.Add(new IidSpikeData(5)); // This is a spike data.Add(new IidSpikeData(10)); - for (int i = 0; i < size / 2; i++) + for (int i = 0; i < Size / 2; i++) data.Add(new IidSpikeData(5)); // Convert data to IDataView. var dataView = ml.CreateStreamingDataView(data); // Setup IidSpikeDetector arguments - string outputColumnName = "Prediction"; - string inputColumnName = "Value"; + string outputColumnName = nameof(IidSpikePrediction.Prediction); + string inputColumnName = nameof(IidSpikeData.Value); var args = new IidSpikeDetector.Arguments() { Source = inputColumnName, Name = outputColumnName, Confidence = 95, // The confidence for spike detection in the range [0, 100] - PvalueHistoryLength = size / 4 // The size of the sliding window for computing the p-value + PvalueHistoryLength = Size / 4 // The size of the sliding window for computing the p-value }; // The transformed data. @@ -95,27 +95,27 @@ public static void IidSpikeDetectorPrediction() var ml = new MLContext(); // Generate sample series data with a spike - const int size = 10; - var data = new List(size); - for (int i = 0; i < size / 2; i++) + const int Size = 10; + var data = new List(Size); + for (int i = 0; i < Size / 2; i++) data.Add(new IidSpikeData(5)); // This is a spike data.Add(new IidSpikeData(10)); - for (int i = 0; i < size / 2; i++) + for (int i = 0; i < Size / 2; i++) data.Add(new IidSpikeData(5)); // Convert data to IDataView. var dataView = ml.CreateStreamingDataView(data); // Setup IidSpikeDetector arguments - string outputColumnName = "Prediction"; - string inputColumnName = "Value"; + string outputColumnName = nameof(IidSpikePrediction.Prediction); + string inputColumnName = nameof(IidSpikeData.Value); var args = new IidSpikeDetector.Arguments() { Source = inputColumnName, Name = outputColumnName, Confidence = 95, // The confidence for spike detection in the range [0, 100] - PvalueHistoryLength = size / 4 // The size of the sliding window for computing the p-value + PvalueHistoryLength = Size / 4 // The size of the sliding window for computing the p-value }; // The transformed model. diff --git a/src/Microsoft.ML.TimeSeries/PredictionFunction.cs b/src/Microsoft.ML.TimeSeries/PredictionFunction.cs index 603a81c7ce..82b408dc07 100644 --- a/src/Microsoft.ML.TimeSeries/PredictionFunction.cs +++ b/src/Microsoft.ML.TimeSeries/PredictionFunction.cs @@ -253,8 +253,8 @@ public override void Predict(TSrc example, ref TDst prediction) public static class PredictionFunctionExtensions { /// - /// Creates a prediction function/engine for a time series pipeline. - /// It updates the state of time series model with observations seen at prediction phase and allows checkpointing the model./> + /// creates a prediction function/engine for a time series pipeline + /// It updates the state of time series model with observations seen at prediction phase and allows checkpointing the model. /// /// Class describing input schema to the model. /// Class describing the output schema of the prediction. From ce6e9b15cba87a662b53dc2388c8e35a879bf848 Mon Sep 17 00:00:00 2001 From: Zeeshan Siddiqui Date: Fri, 30 Nov 2018 14:14:22 -0800 Subject: [PATCH 4/6] PR feedback. --- .../Microsoft.ML.Samples/Dynamic/IidSpikeDetectorTransform.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/IidSpikeDetectorTransform.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/IidSpikeDetectorTransform.cs index caa6b737fb..9b484f932a 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/IidSpikeDetectorTransform.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/IidSpikeDetectorTransform.cs @@ -58,7 +58,7 @@ public static void IidSpikeDetectorTransform() Source = inputColumnName, Name = outputColumnName, Confidence = 95, // The confidence for spike detection in the range [0, 100] - PvalueHistoryLength = Size / 4 // The size of the sliding window for computing the p-value + PvalueHistoryLength = Size / 4 // The size of the sliding window for computing the p-value; shorter windows are more sensitive to spikes. }; // The transformed data. @@ -115,7 +115,7 @@ public static void IidSpikeDetectorPrediction() Source = inputColumnName, Name = outputColumnName, Confidence = 95, // The confidence for spike detection in the range [0, 100] - PvalueHistoryLength = Size / 4 // The size of the sliding window for computing the p-value + PvalueHistoryLength = Size / 4 // The size of the sliding window for computing the p-value; shorter windows are more sensitive to spikes. }; // The transformed model. From fdb09cdca88ba84a6b83e076f8be06fd1e22aa1e Mon Sep 17 00:00:00 2001 From: Zeeshan Siddiqui Date: Mon, 3 Dec 2018 09:42:27 -0800 Subject: [PATCH 5/6] PR feedback. --- .../IidChangePointDetectorTransform.cs | 30 +++++++++++++++++-- .../Dynamic/IidSpikeDetectorTransform.cs | 4 +-- .../PredictionFunction.cs | 11 ++++--- 3 files changed, 34 insertions(+), 11 deletions(-) diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/IidChangePointDetectorTransform.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/IidChangePointDetectorTransform.cs index 771eb26ee5..52729c6c97 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/IidChangePointDetectorTransform.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/IidChangePointDetectorTransform.cs @@ -147,11 +147,15 @@ public static void IidChangePointDetectorPrediction() var modelPath = "temp.zip"; engine.CheckPoint(ml, modelPath); + // Reference to current time series engine because in the next step "engine" will point to the + // checkpointed model being loaded from disk. + var timeseries1 = engine; + // Load the model. using (var file = File.OpenRead(modelPath)) model = TransformerChain.LoadFrom(ml, file); - // Create a time series prediction engine from the model. + // Create a time series prediction engine from the checkpointed model. engine = model.CreateTimeSeriesPredictionFunction(ml); for (int index = 0; index < 8; index++) { @@ -161,8 +165,19 @@ public static void IidChangePointDetectorPrediction() prediction.Prediction[1], prediction.Prediction[2], prediction.Prediction[3]); } + // Prediction from the original time series engine should match the prediction from + // check pointed model. + engine = timeseries1; + for (int index = 0; index < 8; index++) + { + // Anomaly change point detection. + var prediction = engine.Predict(new IidChangePointData(7)); + Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}\t{4:0.00}", 7, prediction.Prediction[0], + prediction.Prediction[1], prediction.Prediction[2], prediction.Prediction[3]); + } + // Data Alert Score P-Value Martingale value - // 5 0 5.00 0.50 0.00 + // 5 0 5.00 0.50 0.00 <-- Time Series 1. // 5 0 5.00 0.50 0.00 // 5 0 5.00 0.50 0.00 // 5 0 5.00 0.50 0.00 @@ -171,7 +186,16 @@ public static void IidChangePointDetectorPrediction() // 5 0 5.00 0.50 0.00 // 5 0 5.00 0.50 0.00 // 7 1 7.00 0.00 10298.67 <-- alert is on, predicted changepoint (and model is checkpointed). - // 7 0 7.00 0.13 33950.16 <-- model loaded back from disk and prediction is made. + + // 7 0 7.00 0.13 33950.16 <-- Time Series 2 : Model loaded back from disk and prediction is made. + // 7 0 7.00 0.26 60866.34 + // 7 0 7.00 0.38 78362.04 + // 7 0 7.00 0.50 0.01 + // 7 0 7.00 0.50 0.00 + // 7 0 7.00 0.50 0.00 + // 7 0 7.00 0.50 0.00 + + // 7 0 7.00 0.13 33950.16 <-- Time Series 1 and prediction is made. // 7 0 7.00 0.26 60866.34 // 7 0 7.00 0.38 78362.04 // 7 0 7.00 0.50 0.01 diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/IidSpikeDetectorTransform.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/IidSpikeDetectorTransform.cs index 9b484f932a..5dcb7e1774 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/IidSpikeDetectorTransform.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/IidSpikeDetectorTransform.cs @@ -1,13 +1,13 @@ using System; +using System.IO; using System.Linq; using System.Collections.Generic; +using Microsoft.ML.Data; using Microsoft.ML.Runtime.Data; using Microsoft.ML.Runtime.Api; using Microsoft.ML.Runtime.TimeSeriesProcessing; using Microsoft.ML.Core.Data; using Microsoft.ML.TimeSeries; -using System.IO; -using Microsoft.ML.Data; namespace Microsoft.ML.Samples.Dynamic { diff --git a/src/Microsoft.ML.TimeSeries/PredictionFunction.cs b/src/Microsoft.ML.TimeSeries/PredictionFunction.cs index 82b408dc07..b488c7b106 100644 --- a/src/Microsoft.ML.TimeSeries/PredictionFunction.cs +++ b/src/Microsoft.ML.TimeSeries/PredictionFunction.cs @@ -57,7 +57,7 @@ public sealed class TimeSeriesPredictionFunction : PredictionEngineB /// Checkpoints to disk with the updated /// state. /// - /// Usually + /// Usually . /// Path to file on disk where the updated model needs to be saved. public void CheckPoint(IHostEnvironment env, string modelPath) { @@ -258,12 +258,11 @@ public static class PredictionFunctionExtensions /// /// Class describing input schema to the model. /// Class describing the output schema of the prediction. - /// The time series pipeline in the form of a + /// The time series pipeline in the form of a . /// Usually - /// - /// - /// - /// + /// To ignore missing columns. Default is false. + /// Input schema definition. Default is null. + /// Output schema definition. Default is null. ///

Example code can be found by searching for TimeSeriesPredictionFunction in ML.NET.

/// /// From 41e4cec06d087c81e12733aa6182539e3a4b9444 Mon Sep 17 00:00:00 2001 From: Zeeshan Siddiqui Date: Mon, 3 Dec 2018 10:33:04 -0800 Subject: [PATCH 6/6] clean up. --- src/Microsoft.ML.TimeSeries/IidChangePointDetector.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.ML.TimeSeries/IidChangePointDetector.cs b/src/Microsoft.ML.TimeSeries/IidChangePointDetector.cs index 7497fef593..3a899f4339 100644 --- a/src/Microsoft.ML.TimeSeries/IidChangePointDetector.cs +++ b/src/Microsoft.ML.TimeSeries/IidChangePointDetector.cs @@ -212,7 +212,7 @@ public sealed class IidChangePointEstimator : TrivialEstimator /// Host Environment. /// Name of the input column. - /// Name of the output column. Name of the output column. Column is a vector of type double and size 4. + /// Name of the output column. Column is a vector of type double and size 4. /// The vector contains Alert, Raw Score, P-Value and Martingale score as first four values. /// The confidence for change point detection in the range [0, 100]. /// The length of the sliding window on p-values for computing the martingale score.