Skip to content

Commit 7ac5ea3

Browse files
droberts195droberts195
authored
[ML] Use results retention time for deleting system annotations (#76096)
In #75617 a new setting, system_annotations_retention_days, was added to control how long system annotations are retained for. We now feel that this setting is redundant and that system annotations should be retained for the same period as results. This is intuitive and defensible, as system annotations can be considered a type of result. Followup to #75617
1 parent 858cc32 commit 7ac5ea3

File tree

19 files changed

+54
-189
lines changed

19 files changed

+54
-189
lines changed

‎client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/config/Job.java

Lines changed: 3 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,6 @@ public class Job implements ToXContentObject {
5555
public static final ParseField DAILY_MODEL_SNAPSHOT_RETENTION_AFTER_DAYS =
5656
new ParseField("daily_model_snapshot_retention_after_days");
5757
public static final ParseField RESULTS_RETENTION_DAYS = new ParseField("results_retention_days");
58-
public static final ParseField SYSTEM_ANNOTATIONS_RETENTION_DAYS = new ParseField("system_annotations_retention_days");
5958
public static final ParseField MODEL_SNAPSHOT_ID = new ParseField("model_snapshot_id");
6059
public static final ParseField RESULTS_INDEX_NAME = new ParseField("results_index_name");
6160
public static final ParseField DELETING = new ParseField("deleting");
@@ -84,7 +83,6 @@ public class Job implements ToXContentObject {
8483
PARSER.declareString((builder, val) -> builder.setBackgroundPersistInterval(
8584
TimeValue.parseTimeValue(val, BACKGROUND_PERSIST_INTERVAL.getPreferredName())), BACKGROUND_PERSIST_INTERVAL);
8685
PARSER.declareLong(Builder::setResultsRetentionDays, RESULTS_RETENTION_DAYS);
87-
PARSER.declareLong(Builder::setSystemAnnotationsRetentionDays, SYSTEM_ANNOTATIONS_RETENTION_DAYS);
8886
PARSER.declareLong(Builder::setModelSnapshotRetentionDays, MODEL_SNAPSHOT_RETENTION_DAYS);
8987
PARSER.declareLong(Builder::setDailyModelSnapshotRetentionAfterDays, DAILY_MODEL_SNAPSHOT_RETENTION_AFTER_DAYS);
9088
PARSER.declareField(Builder::setCustomSettings, (p, c) -> p.mapOrdered(), CUSTOM_SETTINGS, ValueType.OBJECT);
@@ -110,7 +108,6 @@ public class Job implements ToXContentObject {
110108
private final Long modelSnapshotRetentionDays;
111109
private final Long dailyModelSnapshotRetentionAfterDays;
112110
private final Long resultsRetentionDays;
113-
private final Long systemAnnotationsRetentionDays;
114111
private final Map<String, Object> customSettings;
115112
private final String modelSnapshotId;
116113
private final String resultsIndexName;
@@ -122,7 +119,7 @@ private Job(String jobId, String jobType, List<String> groups, String descriptio
122119
AnalysisConfig analysisConfig, AnalysisLimits analysisLimits, DataDescription dataDescription,
123120
ModelPlotConfig modelPlotConfig, Long renormalizationWindowDays, TimeValue backgroundPersistInterval,
124121
Long modelSnapshotRetentionDays, Long dailyModelSnapshotRetentionAfterDays, Long resultsRetentionDays,
125-
Long systemAnnotationsRetentionDays, Map<String, Object> customSettings, String modelSnapshotId, String resultsIndexName,
122+
Map<String, Object> customSettings, String modelSnapshotId, String resultsIndexName,
126123
Boolean deleting, Boolean allowLazyOpen) {
127124

128125
this.jobId = jobId;
@@ -140,7 +137,6 @@ private Job(String jobId, String jobType, List<String> groups, String descriptio
140137
this.modelSnapshotRetentionDays = modelSnapshotRetentionDays;
141138
this.dailyModelSnapshotRetentionAfterDays = dailyModelSnapshotRetentionAfterDays;
142139
this.resultsRetentionDays = resultsRetentionDays;
143-
this.systemAnnotationsRetentionDays = systemAnnotationsRetentionDays;
144140
this.customSettings = customSettings == null ? null : Collections.unmodifiableMap(customSettings);
145141
this.modelSnapshotId = modelSnapshotId;
146142
this.resultsIndexName = resultsIndexName;
@@ -266,10 +262,6 @@ public Long getResultsRetentionDays() {
266262
return resultsRetentionDays;
267263
}
268264

269-
public Long getSystemAnnotationsRetentionDays() {
270-
return systemAnnotationsRetentionDays;
271-
}
272-
273265
public Map<String, Object> getCustomSettings() {
274266
return customSettings;
275267
}
@@ -332,9 +324,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
332324
if (resultsRetentionDays != null) {
333325
builder.field(RESULTS_RETENTION_DAYS.getPreferredName(), resultsRetentionDays);
334326
}
335-
if (systemAnnotationsRetentionDays != null) {
336-
builder.field(SYSTEM_ANNOTATIONS_RETENTION_DAYS.getPreferredName(), systemAnnotationsRetentionDays);
337-
}
338327
if (customSettings != null) {
339328
builder.field(CUSTOM_SETTINGS.getPreferredName(), customSettings);
340329
}
@@ -383,7 +372,6 @@ public boolean equals(Object other) {
383372
&& Objects.equals(this.customSettings, that.customSettings)
384373
&& Objects.equals(this.modelSnapshotId, that.modelSnapshotId)
385374
&& Objects.equals(this.resultsIndexName, that.resultsIndexName)
386-
&& Objects.equals(this.systemAnnotationsRetentionDays, that.systemAnnotationsRetentionDays)
387375
&& Objects.equals(this.deleting, that.deleting)
388376
&& Objects.equals(this.allowLazyOpen, that.allowLazyOpen);
389377
}
@@ -393,7 +381,7 @@ public int hashCode() {
393381
return Objects.hash(jobId, jobType, groups, description, createTime, finishedTime,
394382
analysisConfig, analysisLimits, dataDescription, modelPlotConfig, renormalizationWindowDays,
395383
backgroundPersistInterval, modelSnapshotRetentionDays, dailyModelSnapshotRetentionAfterDays, resultsRetentionDays,
396-
systemAnnotationsRetentionDays, customSettings, modelSnapshotId, resultsIndexName, deleting, allowLazyOpen);
384+
customSettings, modelSnapshotId, resultsIndexName, deleting, allowLazyOpen);
397385
}
398386

399387
@Override
@@ -422,7 +410,6 @@ public static class Builder {
422410
private Long modelSnapshotRetentionDays;
423411
private Long dailyModelSnapshotRetentionAfterDays;
424412
private Long resultsRetentionDays;
425-
private Long systemAnnotationsRetentionDays;
426413
private Map<String, Object> customSettings;
427414
private String modelSnapshotId;
428415
private String resultsIndexName;
@@ -452,7 +439,6 @@ public Builder(Job job) {
452439
this.modelSnapshotRetentionDays = job.getModelSnapshotRetentionDays();
453440
this.dailyModelSnapshotRetentionAfterDays = job.getDailyModelSnapshotRetentionAfterDays();
454441
this.resultsRetentionDays = job.getResultsRetentionDays();
455-
this.systemAnnotationsRetentionDays = job.getSystemAnnotationsRetentionDays();
456442
this.customSettings = job.getCustomSettings() == null ? null : new LinkedHashMap<>(job.getCustomSettings());
457443
this.modelSnapshotId = job.getModelSnapshotId();
458444
this.resultsIndexName = job.getResultsIndexNameNoPrefix();
@@ -544,11 +530,6 @@ public Builder setResultsRetentionDays(Long resultsRetentionDays) {
544530
return this;
545531
}
546532

547-
public Builder setSystemAnnotationsRetentionDays(Long systemAnnotationsRetentionDays) {
548-
this.systemAnnotationsRetentionDays = systemAnnotationsRetentionDays;
549-
return this;
550-
}
551-
552533
public Builder setModelSnapshotId(String modelSnapshotId) {
553534
this.modelSnapshotId = modelSnapshotId;
554535
return this;
@@ -581,7 +562,7 @@ public Job build() {
581562
id, jobType, groups, description, createTime, finishedTime,
582563
analysisConfig, analysisLimits, dataDescription, modelPlotConfig, renormalizationWindowDays,
583564
backgroundPersistInterval, modelSnapshotRetentionDays, dailyModelSnapshotRetentionAfterDays, resultsRetentionDays,
584-
systemAnnotationsRetentionDays, customSettings, modelSnapshotId, resultsIndexName, deleting, allowLazyOpen);
565+
customSettings, modelSnapshotId, resultsIndexName, deleting, allowLazyOpen);
585566
}
586567
}
587568
}

‎client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/config/JobUpdate.java

Lines changed: 3 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,6 @@ public class JobUpdate implements ToXContentObject {
4040
TimeValue.parseTimeValue(val, Job.BACKGROUND_PERSIST_INTERVAL.getPreferredName())), Job.BACKGROUND_PERSIST_INTERVAL);
4141
PARSER.declareLong(Builder::setRenormalizationWindowDays, Job.RENORMALIZATION_WINDOW_DAYS);
4242
PARSER.declareLong(Builder::setResultsRetentionDays, Job.RESULTS_RETENTION_DAYS);
43-
PARSER.declareLong(Builder::setSystemAnnotationsRetentionDays, Job.SYSTEM_ANNOTATIONS_RETENTION_DAYS);
4443
PARSER.declareLong(Builder::setModelSnapshotRetentionDays, Job.MODEL_SNAPSHOT_RETENTION_DAYS);
4544
PARSER.declareLong(Builder::setDailyModelSnapshotRetentionAfterDays, Job.DAILY_MODEL_SNAPSHOT_RETENTION_AFTER_DAYS);
4645
PARSER.declareStringArray(Builder::setCategorizationFilters, AnalysisConfig.CATEGORIZATION_FILTERS);
@@ -63,7 +62,6 @@ public class JobUpdate implements ToXContentObject {
6362
private final Long modelSnapshotRetentionDays;
6463
private final Long dailyModelSnapshotRetentionAfterDays;
6564
private final Long resultsRetentionDays;
66-
private final Long systemAnnotationsRetentionDays;
6765
private final List<String> categorizationFilters;
6866
private final PerPartitionCategorizationConfig perPartitionCategorizationConfig;
6967
private final Map<String, Object> customSettings;
@@ -74,7 +72,7 @@ private JobUpdate(String jobId, @Nullable List<String> groups, @Nullable String
7472
@Nullable List<DetectorUpdate> detectorUpdates, @Nullable ModelPlotConfig modelPlotConfig,
7573
@Nullable AnalysisLimits analysisLimits, @Nullable TimeValue backgroundPersistInterval,
7674
@Nullable Long renormalizationWindowDays, @Nullable Long resultsRetentionDays,
77-
@Nullable Long systemAnnotationsRetentionDays, @Nullable Long modelSnapshotRetentionDays,
75+
@Nullable Long modelSnapshotRetentionDays,
7876
@Nullable Long dailyModelSnapshotRetentionAfterDays, @Nullable List<String> categorizationFilters,
7977
@Nullable PerPartitionCategorizationConfig perPartitionCategorizationConfig,
8078
@Nullable Map<String, Object> customSettings, @Nullable Boolean allowLazyOpen, @Nullable TimeValue modelPruneWindow) {
@@ -89,7 +87,6 @@ private JobUpdate(String jobId, @Nullable List<String> groups, @Nullable String
8987
this.modelSnapshotRetentionDays = modelSnapshotRetentionDays;
9088
this.dailyModelSnapshotRetentionAfterDays = dailyModelSnapshotRetentionAfterDays;
9189
this.resultsRetentionDays = resultsRetentionDays;
92-
this.systemAnnotationsRetentionDays = systemAnnotationsRetentionDays;
9390
this.categorizationFilters = categorizationFilters;
9491
this.perPartitionCategorizationConfig = perPartitionCategorizationConfig;
9592
this.customSettings = customSettings;
@@ -137,10 +134,6 @@ public Long getResultsRetentionDays() {
137134
return resultsRetentionDays;
138135
}
139136

140-
public Long getSystemAnnotationsRetentionDays() {
141-
return systemAnnotationsRetentionDays;
142-
}
143-
144137
public List<String> getCategorizationFilters() {
145138
return categorizationFilters;
146139
}
@@ -195,9 +188,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
195188
if (resultsRetentionDays != null) {
196189
builder.field(Job.RESULTS_RETENTION_DAYS.getPreferredName(), resultsRetentionDays);
197190
}
198-
if (systemAnnotationsRetentionDays != null) {
199-
builder.field(Job.SYSTEM_ANNOTATIONS_RETENTION_DAYS.getPreferredName(), systemAnnotationsRetentionDays);
200-
}
201191
if (categorizationFilters != null) {
202192
builder.field(AnalysisConfig.CATEGORIZATION_FILTERS.getPreferredName(), categorizationFilters);
203193
}
@@ -240,7 +230,6 @@ public boolean equals(Object other) {
240230
&& Objects.equals(this.modelSnapshotRetentionDays, that.modelSnapshotRetentionDays)
241231
&& Objects.equals(this.dailyModelSnapshotRetentionAfterDays, that.dailyModelSnapshotRetentionAfterDays)
242232
&& Objects.equals(this.resultsRetentionDays, that.resultsRetentionDays)
243-
&& Objects.equals(this.systemAnnotationsRetentionDays, that.systemAnnotationsRetentionDays)
244233
&& Objects.equals(this.categorizationFilters, that.categorizationFilters)
245234
&& Objects.equals(this.perPartitionCategorizationConfig, that.perPartitionCategorizationConfig)
246235
&& Objects.equals(this.customSettings, that.customSettings)
@@ -252,7 +241,7 @@ public boolean equals(Object other) {
252241
public int hashCode() {
253242
return Objects.hash(jobId, groups, description, detectorUpdates, modelPlotConfig, analysisLimits, renormalizationWindowDays,
254243
backgroundPersistInterval, modelSnapshotRetentionDays, dailyModelSnapshotRetentionAfterDays, resultsRetentionDays,
255-
systemAnnotationsRetentionDays, categorizationFilters, perPartitionCategorizationConfig, customSettings, allowLazyOpen,
244+
categorizationFilters, perPartitionCategorizationConfig, customSettings, allowLazyOpen,
256245
modelPruneWindow);
257246
}
258247

@@ -348,7 +337,6 @@ public static class Builder {
348337
private Long modelSnapshotRetentionDays;
349338
private Long dailyModelSnapshotRetentionAfterDays;
350339
private Long resultsRetentionDays;
351-
private Long systemAnnotationsRetentionDays;
352340
private List<String> categorizationFilters;
353341
private PerPartitionCategorizationConfig perPartitionCategorizationConfig;
354342
private Map<String, Object> customSettings;
@@ -484,18 +472,6 @@ public Builder setResultsRetentionDays(Long resultsRetentionDays) {
484472
return this;
485473
}
486474

487-
/**
488-
* Advanced configuration option. The number of days for which job annotations are retained
489-
*
490-
* Updates the {@link Job#systemAnnotationsRetentionDays} setting
491-
*
492-
* @param systemAnnotationsRetentionDays number of days to keep results.
493-
*/
494-
public Builder setSystemAnnotationsRetentionDays(Long systemAnnotationsRetentionDays) {
495-
this.systemAnnotationsRetentionDays = systemAnnotationsRetentionDays;
496-
return this;
497-
}
498-
499475
/**
500476
* Sets the categorization filters on the {@link Job}
501477
*
@@ -546,7 +522,7 @@ public Builder setModelPruneWindow(TimeValue modelPruneWindow) {
546522

547523
public JobUpdate build() {
548524
return new JobUpdate(jobId, groups, description, detectorUpdates, modelPlotConfig, analysisLimits, backgroundPersistInterval,
549-
renormalizationWindowDays, resultsRetentionDays, systemAnnotationsRetentionDays, modelSnapshotRetentionDays,
525+
renormalizationWindowDays, resultsRetentionDays, modelSnapshotRetentionDays,
550526
dailyModelSnapshotRetentionAfterDays, categorizationFilters, perPartitionCategorizationConfig, customSettings,
551527
allowLazyOpen, modelPruneWindow);
552528
}

‎client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/config/JobTests.java

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -145,9 +145,6 @@ public static Job.Builder createRandomizedJobBuilder() {
145145
if (randomBoolean()) {
146146
builder.setResultsRetentionDays(randomNonNegativeLong());
147147
}
148-
if (randomBoolean()) {
149-
builder.setSystemAnnotationsRetentionDays(randomNonNegativeLong());
150-
}
151148
if (randomBoolean()) {
152149
builder.setCustomSettings(Collections.singletonMap(randomAlphaOfLength(10), randomAlphaOfLength(10)));
153150
}

‎client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/config/JobUpdateTests.java

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -65,9 +65,6 @@ public static JobUpdate createRandom(String jobId) {
6565
if (randomBoolean()) {
6666
update.setResultsRetentionDays(randomNonNegativeLong());
6767
}
68-
if (randomBoolean()) {
69-
update.setSystemAnnotationsRetentionDays(randomNonNegativeLong());
70-
}
7168
if (randomBoolean()) {
7269
update.setCategorizationFilters(Arrays.asList(generateRandomStringArray(10, 10, false)));
7370
}

‎docs/reference/ml/anomaly-detection/apis/put-job.asciidoc

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -379,10 +379,6 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=results-index-name]
379379
(Optional, long)
380380
include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=results-retention-days]
381381

382-
`system_annotations_retention_days`::
383-
(Optional, long)
384-
include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=system-annotations-retention-days]
385-
386382
[[ml-put-job-example]]
387383
== {api-examples-title}
388384

‎docs/reference/ml/anomaly-detection/apis/update-job.asciidoc

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -236,10 +236,6 @@ close the job, then reopen the job and restart the {dfeed} for the changes to ta
236236
(long)
237237
include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=results-retention-days]
238238

239-
`system_annotations_retention_days`::
240-
(long)
241-
include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=system-annotations-retention-days]
242-
243239

244240
[[ml-update-job-example]]
245241
== {api-examples-title}
@@ -260,8 +256,7 @@ POST _ml/anomaly_detectors/low_request_rate/_update
260256
"renormalization_window_days": 30,
261257
"background_persist_interval": "2h",
262258
"model_snapshot_retention_days": 7,
263-
"results_retention_days": 60,
264-
"system_annotations_retention_days": 60
259+
"results_retention_days": 60
265260
}
266261
--------------------------------------------------
267262
// TEST[skip:setup:Kibana sample data]

‎docs/reference/ml/ml-shared.asciidoc

Lines changed: 3 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1430,19 +1430,11 @@ retained. Age is calculated relative to the timestamp of the latest bucket
14301430
result. If this property has a non-null value, once per day at 00:30 (server
14311431
time), results that are the specified number of days older than the latest
14321432
bucket result are deleted from {es}. The default value is null, which means all
1433-
results are retained.
1433+
results are retained. Annotations generated by the system also count as results
1434+
for retention purposes; they are deleted after the same number of days as
1435+
results. Annotations added by users are retained forever.
14341436
end::results-retention-days[]
14351437

1436-
tag::system-annotations-retention-days[]
1437-
Advanced configuration option. The period of time (in days) that automatically
1438-
created annotations are retained. Age is calculated relative to the timestamp of
1439-
the latest bucket result. If this property has a non-null value, once per day at
1440-
00:30 (server time), annotations that are the specified number of days older
1441-
than the latest bucket result are deleted from {es}. The default value is null,
1442-
which means all annotations are retained.
1443-
User created annotations are never deleted automatically.
1444-
end::system-annotations-retention-days[]
1445-
14461438
tag::retain[]
14471439
If `true`, this snapshot will not be deleted during automatic cleanup of
14481440
snapshots older than `model_snapshot_retention_days`. However, this snapshot

‎x-pack/plugin/build.gradle

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,13 @@ def v7compatibilityNotSupportedTests = {
145145
'rollup/start_job/Test start job twice',
146146
'service_accounts/10_basic/Test service account tokens', // https://github.com/elastic/elasticsearch/pull/75200
147147

148+
// temporarily muted awaiting backport of https://github.com/elastic/elasticsearch/pull/76097
149+
'ml/delete_expired_data/Test delete expired data with body parameters',
150+
'ml/delete_expired_data/Test delete expired data with no body',
151+
'ml/delete_expired_data/Test delete expired data with path parameters',
152+
'ml/delete_expired_data/Test delete expired data with unknown job id',
153+
'ml/jobs_crud/Test update job',
154+
148155
// a type field was added to cat.ml_trained_models #73660, this is a backwards compatible change.
149156
// still this is a cat api, and we don't support them with rest api compatibility. (the test would be very hard to transform too)
150157
'ml/trained_model_cat_apis/Test cat trained models'

0 commit comments

Comments
 (0)