Skip to content

Commit c70ba3c

Browse files
droberts195droberts195
authored
[ML] Use results retention time for deleting system annotations (elastic#76113)
* [ML] Use results retention time for deleting system annotations In elastic#75617 a new setting, system_annotations_retention_days, was added to control how long system annotations are retained for. We now feel that this setting is redundant and that system annotations should be retained for the same period as results. This is intuitive and defensible, as system annotations can be considered a type of result. Backport of elastic#76096 * Fix one more merge clash
1 parent edd5887 commit c70ba3c

File tree

18 files changed

+29
-205
lines changed

18 files changed

+29
-205
lines changed

‎client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/config/Job.java

Lines changed: 3 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,6 @@ public class Job implements ToXContentObject {
5757
public static final ParseField DAILY_MODEL_SNAPSHOT_RETENTION_AFTER_DAYS =
5858
new ParseField("daily_model_snapshot_retention_after_days");
5959
public static final ParseField RESULTS_RETENTION_DAYS = new ParseField("results_retention_days");
60-
public static final ParseField SYSTEM_ANNOTATIONS_RETENTION_DAYS = new ParseField("system_annotations_retention_days");
6160
public static final ParseField MODEL_SNAPSHOT_ID = new ParseField("model_snapshot_id");
6261
public static final ParseField RESULTS_INDEX_NAME = new ParseField("results_index_name");
6362
public static final ParseField DELETING = new ParseField("deleting");
@@ -88,7 +87,6 @@ public class Job implements ToXContentObject {
8887
PARSER.declareString((builder, val) -> builder.setBackgroundPersistInterval(
8988
TimeValue.parseTimeValue(val, BACKGROUND_PERSIST_INTERVAL.getPreferredName())), BACKGROUND_PERSIST_INTERVAL);
9089
PARSER.declareLong(Builder::setResultsRetentionDays, RESULTS_RETENTION_DAYS);
91-
PARSER.declareLong(Builder::setSystemAnnotationsRetentionDays, SYSTEM_ANNOTATIONS_RETENTION_DAYS);
9290
PARSER.declareLong(Builder::setModelSnapshotRetentionDays, MODEL_SNAPSHOT_RETENTION_DAYS);
9391
PARSER.declareLong(Builder::setDailyModelSnapshotRetentionAfterDays, DAILY_MODEL_SNAPSHOT_RETENTION_AFTER_DAYS);
9492
PARSER.declareField(Builder::setCustomSettings, (p, c) -> p.mapOrdered(), CUSTOM_SETTINGS, ValueType.OBJECT);
@@ -116,7 +114,6 @@ public class Job implements ToXContentObject {
116114
private final Long modelSnapshotRetentionDays;
117115
private final Long dailyModelSnapshotRetentionAfterDays;
118116
private final Long resultsRetentionDays;
119-
private final Long systemAnnotationsRetentionDays;
120117
private final Map<String, Object> customSettings;
121118
private final String modelSnapshotId;
122119
private final String resultsIndexName;
@@ -130,7 +127,7 @@ private Job(String jobId, String jobType, List<String> groups, String descriptio
130127
AnalysisConfig analysisConfig, AnalysisLimits analysisLimits, DataDescription dataDescription,
131128
ModelPlotConfig modelPlotConfig, Long renormalizationWindowDays, TimeValue backgroundPersistInterval,
132129
Long modelSnapshotRetentionDays, Long dailyModelSnapshotRetentionAfterDays, Long resultsRetentionDays,
133-
Long systemAnnotationsRetentionDays, Map<String, Object> customSettings, String modelSnapshotId, String resultsIndexName,
130+
Map<String, Object> customSettings, String modelSnapshotId, String resultsIndexName,
134131
Boolean deleting, Boolean allowLazyOpen, Blocked blocked, DatafeedConfig datafeedConfig) {
135132

136133
this.jobId = jobId;
@@ -148,7 +145,6 @@ private Job(String jobId, String jobType, List<String> groups, String descriptio
148145
this.modelSnapshotRetentionDays = modelSnapshotRetentionDays;
149146
this.dailyModelSnapshotRetentionAfterDays = dailyModelSnapshotRetentionAfterDays;
150147
this.resultsRetentionDays = resultsRetentionDays;
151-
this.systemAnnotationsRetentionDays = systemAnnotationsRetentionDays;
152148
this.customSettings = customSettings == null ? null : Collections.unmodifiableMap(customSettings);
153149
this.modelSnapshotId = modelSnapshotId;
154150
this.resultsIndexName = resultsIndexName;
@@ -276,10 +272,6 @@ public Long getResultsRetentionDays() {
276272
return resultsRetentionDays;
277273
}
278274

279-
public Long getSystemAnnotationsRetentionDays() {
280-
return systemAnnotationsRetentionDays;
281-
}
282-
283275
public Map<String, Object> getCustomSettings() {
284276
return customSettings;
285277
}
@@ -354,9 +346,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
354346
if (resultsRetentionDays != null) {
355347
builder.field(RESULTS_RETENTION_DAYS.getPreferredName(), resultsRetentionDays);
356348
}
357-
if (systemAnnotationsRetentionDays != null) {
358-
builder.field(SYSTEM_ANNOTATIONS_RETENTION_DAYS.getPreferredName(), systemAnnotationsRetentionDays);
359-
}
360349
if (customSettings != null) {
361350
builder.field(CUSTOM_SETTINGS.getPreferredName(), customSettings);
362351
}
@@ -411,7 +400,6 @@ public boolean equals(Object other) {
411400
&& Objects.equals(this.customSettings, that.customSettings)
412401
&& Objects.equals(this.modelSnapshotId, that.modelSnapshotId)
413402
&& Objects.equals(this.resultsIndexName, that.resultsIndexName)
414-
&& Objects.equals(this.systemAnnotationsRetentionDays, that.systemAnnotationsRetentionDays)
415403
&& Objects.equals(this.deleting, that.deleting)
416404
&& Objects.equals(this.allowLazyOpen, that.allowLazyOpen)
417405
&& Objects.equals(this.blocked, that.blocked)
@@ -423,7 +411,7 @@ public int hashCode() {
423411
return Objects.hash(jobId, jobType, groups, description, createTime, finishedTime,
424412
analysisConfig, analysisLimits, dataDescription, modelPlotConfig, renormalizationWindowDays,
425413
backgroundPersistInterval, modelSnapshotRetentionDays, dailyModelSnapshotRetentionAfterDays, resultsRetentionDays,
426-
systemAnnotationsRetentionDays, customSettings, modelSnapshotId, resultsIndexName, deleting, allowLazyOpen, blocked,
414+
customSettings, modelSnapshotId, resultsIndexName, deleting, allowLazyOpen, blocked,
427415
datafeedConfig);
428416
}
429417

@@ -453,7 +441,6 @@ public static class Builder {
453441
private Long modelSnapshotRetentionDays;
454442
private Long dailyModelSnapshotRetentionAfterDays;
455443
private Long resultsRetentionDays;
456-
private Long systemAnnotationsRetentionDays;
457444
private Map<String, Object> customSettings;
458445
private String modelSnapshotId;
459446
private String resultsIndexName;
@@ -485,7 +472,6 @@ public Builder(Job job) {
485472
this.modelSnapshotRetentionDays = job.getModelSnapshotRetentionDays();
486473
this.dailyModelSnapshotRetentionAfterDays = job.getDailyModelSnapshotRetentionAfterDays();
487474
this.resultsRetentionDays = job.getResultsRetentionDays();
488-
this.systemAnnotationsRetentionDays = job.getSystemAnnotationsRetentionDays();
489475
this.customSettings = job.getCustomSettings() == null ? null : new LinkedHashMap<>(job.getCustomSettings());
490476
this.modelSnapshotId = job.getModelSnapshotId();
491477
this.resultsIndexName = job.getResultsIndexNameNoPrefix();
@@ -579,11 +565,6 @@ public Builder setResultsRetentionDays(Long resultsRetentionDays) {
579565
return this;
580566
}
581567

582-
public Builder setSystemAnnotationsRetentionDays(Long systemAnnotationsRetentionDays) {
583-
this.systemAnnotationsRetentionDays = systemAnnotationsRetentionDays;
584-
return this;
585-
}
586-
587568
public Builder setModelSnapshotId(String modelSnapshotId) {
588569
this.modelSnapshotId = modelSnapshotId;
589570
return this;
@@ -626,7 +607,7 @@ public Job build() {
626607
id, jobType, groups, description, createTime, finishedTime,
627608
analysisConfig, analysisLimits, dataDescription, modelPlotConfig, renormalizationWindowDays,
628609
backgroundPersistInterval, modelSnapshotRetentionDays, dailyModelSnapshotRetentionAfterDays, resultsRetentionDays,
629-
systemAnnotationsRetentionDays, customSettings, modelSnapshotId, resultsIndexName, deleting, allowLazyOpen, blocked,
610+
customSettings, modelSnapshotId, resultsIndexName, deleting, allowLazyOpen, blocked,
630611
datafeedConfig == null ? null : datafeedConfig.build());
631612
}
632613
}

‎client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/config/JobUpdate.java

Lines changed: 3 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,6 @@ public class JobUpdate implements ToXContentObject {
4040
TimeValue.parseTimeValue(val, Job.BACKGROUND_PERSIST_INTERVAL.getPreferredName())), Job.BACKGROUND_PERSIST_INTERVAL);
4141
PARSER.declareLong(Builder::setRenormalizationWindowDays, Job.RENORMALIZATION_WINDOW_DAYS);
4242
PARSER.declareLong(Builder::setResultsRetentionDays, Job.RESULTS_RETENTION_DAYS);
43-
PARSER.declareLong(Builder::setSystemAnnotationsRetentionDays, Job.SYSTEM_ANNOTATIONS_RETENTION_DAYS);
4443
PARSER.declareLong(Builder::setModelSnapshotRetentionDays, Job.MODEL_SNAPSHOT_RETENTION_DAYS);
4544
PARSER.declareLong(Builder::setDailyModelSnapshotRetentionAfterDays, Job.DAILY_MODEL_SNAPSHOT_RETENTION_AFTER_DAYS);
4645
PARSER.declareStringArray(Builder::setCategorizationFilters, AnalysisConfig.CATEGORIZATION_FILTERS);
@@ -63,7 +62,6 @@ public class JobUpdate implements ToXContentObject {
6362
private final Long modelSnapshotRetentionDays;
6463
private final Long dailyModelSnapshotRetentionAfterDays;
6564
private final Long resultsRetentionDays;
66-
private final Long systemAnnotationsRetentionDays;
6765
private final List<String> categorizationFilters;
6866
private final PerPartitionCategorizationConfig perPartitionCategorizationConfig;
6967
private final Map<String, Object> customSettings;
@@ -74,7 +72,7 @@ private JobUpdate(String jobId, @Nullable List<String> groups, @Nullable String
7472
@Nullable List<DetectorUpdate> detectorUpdates, @Nullable ModelPlotConfig modelPlotConfig,
7573
@Nullable AnalysisLimits analysisLimits, @Nullable TimeValue backgroundPersistInterval,
7674
@Nullable Long renormalizationWindowDays, @Nullable Long resultsRetentionDays,
77-
@Nullable Long systemAnnotationsRetentionDays, @Nullable Long modelSnapshotRetentionDays,
75+
@Nullable Long modelSnapshotRetentionDays,
7876
@Nullable Long dailyModelSnapshotRetentionAfterDays, @Nullable List<String> categorizationFilters,
7977
@Nullable PerPartitionCategorizationConfig perPartitionCategorizationConfig,
8078
@Nullable Map<String, Object> customSettings, @Nullable Boolean allowLazyOpen, @Nullable TimeValue modelPruneWindow) {
@@ -89,7 +87,6 @@ private JobUpdate(String jobId, @Nullable List<String> groups, @Nullable String
8987
this.modelSnapshotRetentionDays = modelSnapshotRetentionDays;
9088
this.dailyModelSnapshotRetentionAfterDays = dailyModelSnapshotRetentionAfterDays;
9189
this.resultsRetentionDays = resultsRetentionDays;
92-
this.systemAnnotationsRetentionDays = systemAnnotationsRetentionDays;
9390
this.categorizationFilters = categorizationFilters;
9491
this.perPartitionCategorizationConfig = perPartitionCategorizationConfig;
9592
this.customSettings = customSettings;
@@ -137,10 +134,6 @@ public Long getResultsRetentionDays() {
137134
return resultsRetentionDays;
138135
}
139136

140-
public Long getSystemAnnotationsRetentionDays() {
141-
return systemAnnotationsRetentionDays;
142-
}
143-
144137
public List<String> getCategorizationFilters() {
145138
return categorizationFilters;
146139
}
@@ -195,9 +188,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
195188
if (resultsRetentionDays != null) {
196189
builder.field(Job.RESULTS_RETENTION_DAYS.getPreferredName(), resultsRetentionDays);
197190
}
198-
if (systemAnnotationsRetentionDays != null) {
199-
builder.field(Job.SYSTEM_ANNOTATIONS_RETENTION_DAYS.getPreferredName(), systemAnnotationsRetentionDays);
200-
}
201191
if (categorizationFilters != null) {
202192
builder.field(AnalysisConfig.CATEGORIZATION_FILTERS.getPreferredName(), categorizationFilters);
203193
}
@@ -240,7 +230,6 @@ public boolean equals(Object other) {
240230
&& Objects.equals(this.modelSnapshotRetentionDays, that.modelSnapshotRetentionDays)
241231
&& Objects.equals(this.dailyModelSnapshotRetentionAfterDays, that.dailyModelSnapshotRetentionAfterDays)
242232
&& Objects.equals(this.resultsRetentionDays, that.resultsRetentionDays)
243-
&& Objects.equals(this.systemAnnotationsRetentionDays, that.systemAnnotationsRetentionDays)
244233
&& Objects.equals(this.categorizationFilters, that.categorizationFilters)
245234
&& Objects.equals(this.perPartitionCategorizationConfig, that.perPartitionCategorizationConfig)
246235
&& Objects.equals(this.customSettings, that.customSettings)
@@ -252,7 +241,7 @@ public boolean equals(Object other) {
252241
public int hashCode() {
253242
return Objects.hash(jobId, groups, description, detectorUpdates, modelPlotConfig, analysisLimits, renormalizationWindowDays,
254243
backgroundPersistInterval, modelSnapshotRetentionDays, dailyModelSnapshotRetentionAfterDays, resultsRetentionDays,
255-
systemAnnotationsRetentionDays, categorizationFilters, perPartitionCategorizationConfig, customSettings, allowLazyOpen,
244+
categorizationFilters, perPartitionCategorizationConfig, customSettings, allowLazyOpen,
256245
modelPruneWindow);
257246
}
258247

@@ -348,7 +337,6 @@ public static class Builder {
348337
private Long modelSnapshotRetentionDays;
349338
private Long dailyModelSnapshotRetentionAfterDays;
350339
private Long resultsRetentionDays;
351-
private Long systemAnnotationsRetentionDays;
352340
private List<String> categorizationFilters;
353341
private PerPartitionCategorizationConfig perPartitionCategorizationConfig;
354342
private Map<String, Object> customSettings;
@@ -484,18 +472,6 @@ public Builder setResultsRetentionDays(Long resultsRetentionDays) {
484472
return this;
485473
}
486474

487-
/**
488-
* Advanced configuration option. The number of days for which job annotations are retained
489-
*
490-
* Updates the {@link Job#systemAnnotationsRetentionDays} setting
491-
*
492-
* @param systemAnnotationsRetentionDays number of days to keep results.
493-
*/
494-
public Builder setSystemAnnotationsRetentionDays(Long systemAnnotationsRetentionDays) {
495-
this.systemAnnotationsRetentionDays = systemAnnotationsRetentionDays;
496-
return this;
497-
}
498-
499475
/**
500476
* Sets the categorization filters on the {@link Job}
501477
*
@@ -546,7 +522,7 @@ public Builder setModelPruneWindow(TimeValue modelPruneWindow) {
546522

547523
public JobUpdate build() {
548524
return new JobUpdate(jobId, groups, description, detectorUpdates, modelPlotConfig, analysisLimits, backgroundPersistInterval,
549-
renormalizationWindowDays, resultsRetentionDays, systemAnnotationsRetentionDays, modelSnapshotRetentionDays,
525+
renormalizationWindowDays, resultsRetentionDays, modelSnapshotRetentionDays,
550526
dailyModelSnapshotRetentionAfterDays, categorizationFilters, perPartitionCategorizationConfig, customSettings,
551527
allowLazyOpen, modelPruneWindow);
552528
}

‎client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/config/JobTests.java

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -147,9 +147,6 @@ public static Job.Builder createRandomizedJobBuilder() {
147147
if (randomBoolean()) {
148148
builder.setResultsRetentionDays(randomNonNegativeLong());
149149
}
150-
if (randomBoolean()) {
151-
builder.setSystemAnnotationsRetentionDays(randomNonNegativeLong());
152-
}
153150
if (randomBoolean()) {
154151
builder.setCustomSettings(Collections.singletonMap(randomAlphaOfLength(10), randomAlphaOfLength(10)));
155152
}

‎client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/config/JobUpdateTests.java

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -65,9 +65,6 @@ public static JobUpdate createRandom(String jobId) {
6565
if (randomBoolean()) {
6666
update.setResultsRetentionDays(randomNonNegativeLong());
6767
}
68-
if (randomBoolean()) {
69-
update.setSystemAnnotationsRetentionDays(randomNonNegativeLong());
70-
}
7168
if (randomBoolean()) {
7269
update.setCategorizationFilters(Arrays.asList(generateRandomStringArray(10, 10, false)));
7370
}

‎docs/reference/ml/anomaly-detection/apis/put-job.asciidoc

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -379,10 +379,6 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=results-index-name]
379379
(Optional, long)
380380
include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=results-retention-days]
381381

382-
`system_annotations_retention_days`::
383-
(Optional, long)
384-
include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=system-annotations-retention-days]
385-
386382
[[ml-put-job-example]]
387383
== {api-examples-title}
388384

‎docs/reference/ml/anomaly-detection/apis/update-job.asciidoc

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -235,10 +235,6 @@ close the job, then reopen the job and restart the {dfeed} for the changes to ta
235235
(long)
236236
include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=results-retention-days]
237237

238-
`system_annotations_retention_days`::
239-
(long)
240-
include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=system-annotations-retention-days]
241-
242238

243239
[[ml-update-job-example]]
244240
== {api-examples-title}
@@ -259,8 +255,7 @@ POST _ml/anomaly_detectors/low_request_rate/_update
259255
"renormalization_window_days": 30,
260256
"background_persist_interval": "2h",
261257
"model_snapshot_retention_days": 7,
262-
"results_retention_days": 60,
263-
"system_annotations_retention_days": 60
258+
"results_retention_days": 60
264259
}
265260
--------------------------------------------------
266261
// TEST[skip:setup:Kibana sample data]

‎docs/reference/ml/ml-shared.asciidoc

Lines changed: 3 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1430,19 +1430,11 @@ retained. Age is calculated relative to the timestamp of the latest bucket
14301430
result. If this property has a non-null value, once per day at 00:30 (server
14311431
time), results that are the specified number of days older than the latest
14321432
bucket result are deleted from {es}. The default value is null, which means all
1433-
results are retained.
1433+
results are retained. Annotations generated by the system also count as results
1434+
for retention purposes; they are deleted after the same number of days as
1435+
results. Annotations added by users are retained forever.
14341436
end::results-retention-days[]
14351437

1436-
tag::system-annotations-retention-days[]
1437-
Advanced configuration option. The period of time (in days) that automatically
1438-
created annotations are retained. Age is calculated relative to the timestamp of
1439-
the latest bucket result. If this property has a non-null value, once per day at
1440-
00:30 (server time), annotations that are the specified number of days older
1441-
than the latest bucket result are deleted from {es}. The default value is null,
1442-
which means all annotations are retained.
1443-
User created annotations are never deleted automatically.
1444-
end::system-annotations-retention-days[]
1445-
14461438
tag::retain[]
14471439
If `true`, this snapshot will not be deleted during automatic cleanup of
14481440
snapshots older than `model_snapshot_retention_days`. However, this snapshot

0 commit comments

Comments
 (0)