Skip to content

Commit 048d828

Browse files
mengaimsmengaims
authored
Add SrCnn Anomaly Detector (#3693)
* Add SrCnn Anomaly Detector * Add core calculation code to SrCnn * Fix implementation bugs * Add test and sample. * Fix commented problems: 1.Add xml to DetectAnomalyBySrCnn; 2.Add default value to DetectAnomalyBySrCnn; 3.Change class name of SrCnnAnomalyDetectionBaseWrapper; 4.fix AlertThreshold write bug; 5.Add check argument part and remove redundent TODOs 6.Fix slot name filling bug. * Fix a predict bug * 1. Fix build fail problem; 2. Improve samples; 3. Minor change to docs * Add document. * Add equations to doc
1 parent 18a2615 commit 048d828

File tree

7 files changed

+1208
-0
lines changed

7 files changed

+1208
-0
lines changed

‎docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectAnomalyBySrCnn.cs

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.IO;
4+
using Microsoft.ML;
5+
using Microsoft.ML.Data;
6+
using Microsoft.ML.Transforms.TimeSeries;
7+
8+
namespace Samples.Dynamic
9+
{
10+
public static class DetectAnomalyBySrCnn
11+
{
12+
// This example creates a time series (list of Data with the i-th element corresponding to the i-th time slot).
13+
// The estimator is applied then to identify spiking points in the series.
14+
public static void Example()
15+
{
16+
// Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging,
17+
// as well as the source of randomness.
18+
var ml = new MLContext();
19+
20+
// Generate sample series data with an anomaly
21+
var data = new List<TimeSeriesData>();
22+
for (int index = 0; index < 20; index++)
23+
{
24+
data.Add(new TimeSeriesData(5));
25+
}
26+
data.Add(new TimeSeriesData(10));
27+
for (int index = 0; index < 5; index++)
28+
{
29+
data.Add(new TimeSeriesData(5));
30+
}
31+
32+
// Convert data to IDataView.
33+
var dataView = ml.Data.LoadFromEnumerable(data);
34+
35+
// Setup the estimator arguments
36+
string outputColumnName = nameof(SrCnnAnomalyDetection.Prediction);
37+
string inputColumnName = nameof(TimeSeriesData.Value);
38+
39+
// The transformed model.
40+
ITransformer model = ml.Transforms.DetectAnomalyBySrCnn(outputColumnName, inputColumnName, 16, 5, 5, 3, 8, 0.35).Fit(dataView);
41+
42+
// Create a time series prediction engine from the model.
43+
var engine = model.CreateTimeSeriesPredictionFunction<TimeSeriesData, SrCnnAnomalyDetection>(ml);
44+
45+
Console.WriteLine($"{outputColumnName} column obtained post-transformation.");
46+
Console.WriteLine("Data\tAlert\tScore\tMag");
47+
48+
// Prediction column obtained post-transformation.
49+
// Data Alert Score Mag
50+
51+
// Create non-anomalous data and check for anomaly.
52+
for (int index = 0; index < 20; index++)
53+
{
54+
// Anomaly detection.
55+
PrintPrediction(5, engine.Predict(new TimeSeriesData(5)));
56+
}
57+
58+
//5 0 0.00 0.00
59+
//5 0 0.00 0.00
60+
//5 0 0.00 0.00
61+
//5 0 0.00 0.00
62+
//5 0 0.00 0.00
63+
//5 0 0.00 0.00
64+
//5 0 0.00 0.00
65+
//5 0 0.00 0.00
66+
//5 0 0.00 0.00
67+
//5 0 0.00 0.00
68+
//5 0 0.00 0.00
69+
//5 0 0.00 0.00
70+
//5 0 0.00 0.00
71+
//5 0 0.00 0.00
72+
//5 0 0.00 0.00
73+
//5 0 0.03 0.18
74+
//5 0 0.03 0.18
75+
//5 0 0.03 0.18
76+
//5 0 0.03 0.18
77+
//5 0 0.03 0.18
78+
79+
// Anomaly.
80+
PrintPrediction(10, engine.Predict(new TimeSeriesData(10)));
81+
82+
//10 1 0.47 0.93 <-- alert is on, predicted anomaly
83+
84+
// Checkpoint the model.
85+
var modelPath = "temp.zip";
86+
engine.CheckPoint(ml, modelPath);
87+
88+
// Load the model.
89+
using (var file = File.OpenRead(modelPath))
90+
model = ml.Model.Load(file, out DataViewSchema schema);
91+
92+
for (int index = 0; index < 5; index++)
93+
{
94+
// Anomaly detection.
95+
PrintPrediction(5, engine.Predict(new TimeSeriesData(5)));
96+
}
97+
98+
//5 0 0.31 0.50
99+
//5 0 0.05 0.30
100+
//5 0 0.01 0.23
101+
//5 0 0.00 0.21
102+
//5 0 0.01 0.25
103+
}
104+
105+
private static void PrintPrediction(float value, SrCnnAnomalyDetection prediction) =>
106+
Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}", value, prediction.Prediction[0],
107+
prediction.Prediction[1], prediction.Prediction[2]);
108+
109+
private class TimeSeriesData
110+
{
111+
public float Value;
112+
113+
public TimeSeriesData(float value)
114+
{
115+
Value = value;
116+
}
117+
}
118+
119+
private class SrCnnAnomalyDetection
120+
{
121+
[VectorType(3)]
122+
public double[] Prediction { get; set; }
123+
}
124+
}
125+
}

‎docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectAnomalyBySrCnnBatchPrediction.cs

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using Microsoft.ML;
4+
using Microsoft.ML.Data;
5+
6+
namespace Samples.Dynamic
7+
{
8+
public static class DetectAnomalyBySrCnnBatchPrediction
9+
{
10+
public static void Example()
11+
{
12+
// Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging,
13+
// as well as the source of randomness.
14+
var ml = new MLContext();
15+
16+
// Generate sample series data with an anomaly
17+
var data = new List<TimeSeriesData>();
18+
for (int index = 0; index < 20; index++)
19+
{
20+
data.Add(new TimeSeriesData(5));
21+
}
22+
data.Add(new TimeSeriesData(10));
23+
for (int index = 0; index < 5; index++)
24+
{
25+
data.Add(new TimeSeriesData(5));
26+
}
27+
28+
// Convert data to IDataView.
29+
var dataView = ml.Data.LoadFromEnumerable(data);
30+
31+
// Setup the estimator arguments
32+
string outputColumnName = nameof(SrCnnAnomalyDetection.Prediction);
33+
string inputColumnName = nameof(TimeSeriesData.Value);
34+
35+
// The transformed data.
36+
var transformedData = ml.Transforms.DetectAnomalyBySrCnn(outputColumnName, inputColumnName, 16, 5, 5, 3, 8, 0.35).Fit(dataView).Transform(dataView);
37+
38+
// Getting the data of the newly created column as an IEnumerable of SrCnnAnomalyDetection.
39+
var predictionColumn = ml.Data.CreateEnumerable<SrCnnAnomalyDetection>(transformedData, reuseRowObject: false);
40+
41+
Console.WriteLine($"{outputColumnName} column obtained post-transformation.");
42+
Console.WriteLine("Data\tAlert\tScore\tMag");
43+
44+
int k = 0;
45+
foreach (var prediction in predictionColumn)
46+
PrintPrediction(data[k++].Value, prediction);
47+
48+
//Prediction column obtained post-transformation.
49+
//Data Alert Score Mag
50+
//5 0 0.00 0.00
51+
//5 0 0.00 0.00
52+
//5 0 0.00 0.00
53+
//5 0 0.00 0.00
54+
//5 0 0.00 0.00
55+
//5 0 0.00 0.00
56+
//5 0 0.00 0.00
57+
//5 0 0.00 0.00
58+
//5 0 0.00 0.00
59+
//5 0 0.00 0.00
60+
//5 0 0.00 0.00
61+
//5 0 0.00 0.00
62+
//5 0 0.00 0.00
63+
//5 0 0.00 0.00
64+
//5 0 0.00 0.00
65+
//5 0 0.03 0.18
66+
//5 0 0.03 0.18
67+
//5 0 0.03 0.18
68+
//5 0 0.03 0.18
69+
//5 0 0.03 0.18
70+
//10 1 0.47 0.93
71+
//5 0 0.31 0.50
72+
//5 0 0.05 0.30
73+
//5 0 0.01 0.23
74+
//5 0 0.00 0.21
75+
//5 0 0.01 0.25
76+
}
77+
78+
private static void PrintPrediction(float value, SrCnnAnomalyDetection prediction) =>
79+
Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}", value, prediction.Prediction[0],
80+
prediction.Prediction[1], prediction.Prediction[2]);
81+
82+
private class TimeSeriesData
83+
{
84+
public float Value;
85+
86+
public TimeSeriesData(float value)
87+
{
88+
Value = value;
89+
}
90+
}
91+
92+
private class SrCnnAnomalyDetection
93+
{
94+
[VectorType(3)]
95+
public double[] Prediction { get; set; }
96+
}
97+
}
98+
}

‎src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,5 +121,29 @@ public static SsaChangePointEstimator DetectChangePointBySsa(this TransformsCata
121121
public static SsaSpikeEstimator DetectSpikeBySsa(this TransformsCatalog catalog, string outputColumnName, string inputColumnName, int confidence, int pvalueHistoryLength,
122122
int trainingWindowSize, int seasonalityWindowSize, AnomalySide side = AnomalySide.TwoSided, ErrorFunction errorFunction = ErrorFunction.SignedDifference)
123123
=> new SsaSpikeEstimator(CatalogUtils.GetEnvironment(catalog), outputColumnName, confidence, pvalueHistoryLength, trainingWindowSize, seasonalityWindowSize, inputColumnName, side, errorFunction);
124+
125+
/// <summary>
126+
/// Create <see cref="SrCnnAnomalyEstimator"/>, which detects timeseries anomalies using SRCNN algorithm.
127+
/// </summary>
128+
/// <param name="catalog">The transform's catalog.</param>
129+
/// <param name="outputColumnName">Name of the column resulting from the transformation of <paramref name="inputColumnName"/>.
130+
/// The column data is a vector of <see cref="System.Double"/>. The vector contains 3 elements: alert (1 means anomaly while 0 means normal), raw score, and magnitude of spectual residual.</param>
131+
/// <param name="inputColumnName">Name of column to transform. The column data must be <see cref="System.Single"/>.</param>
132+
/// <param name="windowSize">The size of the sliding window for computing spectral residual.</param>
133+
/// <param name="backAddWindowSize">The number of points to add back of training window. No more than windowSize, usually keep default value.</param>
134+
/// <param name="lookaheadWindowSize">The number of pervious points used in prediction. No more than windowSize, usually keep default value.</param>
135+
/// <param name="averageingWindowSize">The size of sliding window to generate a saliency map for the series. No more than windowSize, usually keep default value.</param>
136+
/// <param name="judgementWindowSize">The size of sliding window to calculate the anomaly score for each data point. No more than windowSize.</param>
137+
/// <param name="threshold">The threshold to determine anomaly, score larger than the threshold is considered as anomaly. Should be in (0,1)</param>
138+
/// <example>
139+
/// <format type="text/markdown">
140+
/// <![CDATA[
141+
/// [!code-csharp[DetectAnomalyBySrCnn](~/../docs/samples/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectAnomalyBySrCnn.cs)]
142+
/// ]]>
143+
/// </format>
144+
/// </example>
145+
public static SrCnnAnomalyEstimator DetectAnomalyBySrCnn(this TransformsCatalog catalog, string outputColumnName, string inputColumnName,
146+
int windowSize=64, int backAddWindowSize=5, int lookaheadWindowSize=5, int averageingWindowSize=3, int judgementWindowSize=21, double threshold=0.3)
147+
=> new SrCnnAnomalyEstimator(CatalogUtils.GetEnvironment(catalog), outputColumnName, windowSize, backAddWindowSize, lookaheadWindowSize, averageingWindowSize, judgementWindowSize, threshold, inputColumnName);
124148
}
125149
}

0 commit comments

Comments
 (0)