Skip to content

Commit ae6332d

Browse files
dakronedakrone
authored
Add min_count and max_count as SLM retention predicates (#44926)
This adds the configuration options for `min_count` and `max_count` as well as the logic for determining whether a snapshot meets this criteria to SLM's retention feature. These options are optional and one, two, or all three can be specified in an SLM policy. Relates to #43663
1 parent 337a51c commit ae6332d

File tree

7 files changed

+320
-32
lines changed

7 files changed

+320
-32
lines changed

‎client/rest-high-level/src/main/java/org/elasticsearch/client/slm/SnapshotRetentionConfiguration.java

+44-6
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
package org.elasticsearch.client.slm;
2121

22+
import org.elasticsearch.common.Nullable;
2223
import org.elasticsearch.common.ParseField;
2324
import org.elasticsearch.common.Strings;
2425
import org.elasticsearch.common.unit.TimeValue;
@@ -32,25 +33,46 @@
3233

3334
public class SnapshotRetentionConfiguration implements ToXContentObject {
3435

35-
public static final SnapshotRetentionConfiguration EMPTY = new SnapshotRetentionConfiguration((TimeValue) null);
36+
public static final SnapshotRetentionConfiguration EMPTY = new SnapshotRetentionConfiguration(null, null, null);
3637

3738
private static final ParseField EXPIRE_AFTER = new ParseField("expire_after");
39+
private static final ParseField MINIMUM_SNAPSHOT_COUNT = new ParseField("min_count");
40+
private static final ParseField MAXIMUM_SNAPSHOT_COUNT = new ParseField("max_count");
3841

3942
private static final ConstructingObjectParser<SnapshotRetentionConfiguration, Void> PARSER =
4043
new ConstructingObjectParser<>("snapshot_retention", true, a -> {
4144
TimeValue expireAfter = a[0] == null ? null : TimeValue.parseTimeValue((String) a[0], EXPIRE_AFTER.getPreferredName());
42-
return new SnapshotRetentionConfiguration(expireAfter);
45+
Integer minCount = (Integer) a[1];
46+
Integer maxCount = (Integer) a[2];
47+
return new SnapshotRetentionConfiguration(expireAfter, minCount, maxCount);
4348
});
4449

4550
static {
4651
PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), EXPIRE_AFTER);
52+
PARSER.declareInt(ConstructingObjectParser.optionalConstructorArg(), MINIMUM_SNAPSHOT_COUNT);
53+
PARSER.declareInt(ConstructingObjectParser.optionalConstructorArg(), MAXIMUM_SNAPSHOT_COUNT);
4754
}
4855

49-
// TODO: add the rest of the configuration values
5056
private final TimeValue expireAfter;
57+
private final Integer minimumSnapshotCount;
58+
private final Integer maximumSnapshotCount;
5159

52-
public SnapshotRetentionConfiguration(TimeValue expireAfter) {
60+
public SnapshotRetentionConfiguration(@Nullable TimeValue expireAfter,
61+
@Nullable Integer minimumSnapshotCount,
62+
@Nullable Integer maximumSnapshotCount) {
5363
this.expireAfter = expireAfter;
64+
this.minimumSnapshotCount = minimumSnapshotCount;
65+
this.maximumSnapshotCount = maximumSnapshotCount;
66+
if (this.minimumSnapshotCount != null && this.minimumSnapshotCount < 1) {
67+
throw new IllegalArgumentException("minimum snapshot count must be at least 1, but was: " + this.minimumSnapshotCount);
68+
}
69+
if (this.maximumSnapshotCount != null && this.maximumSnapshotCount < 1) {
70+
throw new IllegalArgumentException("maximum snapshot count must be at least 1, but was: " + this.maximumSnapshotCount);
71+
}
72+
if ((maximumSnapshotCount != null && minimumSnapshotCount != null) && this.minimumSnapshotCount > this.maximumSnapshotCount) {
73+
throw new IllegalArgumentException("minimum snapshot count " + this.minimumSnapshotCount +
74+
" cannot be larger than maximum snapshot count " + this.maximumSnapshotCount);
75+
}
5476
}
5577

5678
public static SnapshotRetentionConfiguration parse(XContentParser parser, String name) {
@@ -61,19 +83,33 @@ public TimeValue getExpireAfter() {
6183
return this.expireAfter;
6284
}
6385

86+
public Integer getMinimumSnapshotCount() {
87+
return this.minimumSnapshotCount;
88+
}
89+
90+
public Integer getMaximumSnapshotCount() {
91+
return this.maximumSnapshotCount;
92+
}
93+
6494
@Override
6595
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
6696
builder.startObject();
6797
if (expireAfter != null) {
6898
builder.field(EXPIRE_AFTER.getPreferredName(), expireAfter.getStringRep());
6999
}
100+
if (minimumSnapshotCount != null) {
101+
builder.field(MINIMUM_SNAPSHOT_COUNT.getPreferredName(), minimumSnapshotCount);
102+
}
103+
if (maximumSnapshotCount != null) {
104+
builder.field(MAXIMUM_SNAPSHOT_COUNT.getPreferredName(), maximumSnapshotCount);
105+
}
70106
builder.endObject();
71107
return builder;
72108
}
73109

74110
@Override
75111
public int hashCode() {
76-
return Objects.hash(expireAfter);
112+
return Objects.hash(expireAfter, minimumSnapshotCount, maximumSnapshotCount);
77113
}
78114

79115
@Override
@@ -85,7 +121,9 @@ public boolean equals(Object obj) {
85121
return false;
86122
}
87123
SnapshotRetentionConfiguration other = (SnapshotRetentionConfiguration) obj;
88-
return Objects.equals(this.expireAfter, other.expireAfter);
124+
return Objects.equals(this.expireAfter, other.expireAfter) &&
125+
Objects.equals(minimumSnapshotCount, other.minimumSnapshotCount) &&
126+
Objects.equals(maximumSnapshotCount, other.maximumSnapshotCount);
89127
}
90128

91129
@Override

‎client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/ILMDocumentationIT.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -775,7 +775,7 @@ public void testAddSnapshotLifecyclePolicy() throws Exception {
775775
Map<String, Object> config = new HashMap<>();
776776
config.put("indices", Collections.singletonList("idx"));
777777
SnapshotRetentionConfiguration retention =
778-
new SnapshotRetentionConfiguration(TimeValue.timeValueDays(30));
778+
new SnapshotRetentionConfiguration(TimeValue.timeValueDays(30), 2, 10);
779779
SnapshotLifecyclePolicy policy = new SnapshotLifecyclePolicy(
780780
"policy_id", "name", "1 2 3 * * ?",
781781
"my_repository", config, retention);

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotRetentionConfiguration.java

+142-15
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@
66

77
package org.elasticsearch.xpack.core.slm;
88

9+
import org.apache.logging.log4j.LogManager;
10+
import org.apache.logging.log4j.Logger;
911
import org.elasticsearch.common.Nullable;
1012
import org.elasticsearch.common.ParseField;
1113
import org.elasticsearch.common.Strings;
@@ -20,35 +22,73 @@
2022
import org.elasticsearch.snapshots.SnapshotInfo;
2123

2224
import java.io.IOException;
25+
import java.util.Comparator;
2326
import java.util.List;
2427
import java.util.Objects;
28+
import java.util.Set;
29+
import java.util.function.LongSupplier;
2530
import java.util.function.Predicate;
31+
import java.util.stream.Collectors;
2632

2733
public class SnapshotRetentionConfiguration implements ToXContentObject, Writeable {
2834

29-
public static final SnapshotRetentionConfiguration EMPTY = new SnapshotRetentionConfiguration((TimeValue) null);
35+
public static final SnapshotRetentionConfiguration EMPTY = new SnapshotRetentionConfiguration(null, null, null);
3036

3137
private static final ParseField EXPIRE_AFTER = new ParseField("expire_after");
38+
private static final ParseField MINIMUM_SNAPSHOT_COUNT = new ParseField("min_count");
39+
private static final ParseField MAXIMUM_SNAPSHOT_COUNT = new ParseField("max_count");
40+
private static final Logger logger = LogManager.getLogger(SnapshotRetentionConfiguration.class);
3241

3342
private static final ConstructingObjectParser<SnapshotRetentionConfiguration, Void> PARSER =
3443
new ConstructingObjectParser<>("snapshot_retention", true, a -> {
3544
TimeValue expireAfter = a[0] == null ? null : TimeValue.parseTimeValue((String) a[0], EXPIRE_AFTER.getPreferredName());
36-
return new SnapshotRetentionConfiguration(expireAfter);
45+
Integer minCount = (Integer) a[1];
46+
Integer maxCount = (Integer) a[2];
47+
return new SnapshotRetentionConfiguration(expireAfter, minCount, maxCount);
3748
});
3849

3950
static {
4051
PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), EXPIRE_AFTER);
52+
PARSER.declareInt(ConstructingObjectParser.optionalConstructorArg(), MINIMUM_SNAPSHOT_COUNT);
53+
PARSER.declareInt(ConstructingObjectParser.optionalConstructorArg(), MAXIMUM_SNAPSHOT_COUNT);
4154
}
4255

43-
// TODO: add the rest of the configuration values
56+
private final LongSupplier nowSupplier;
4457
private final TimeValue expireAfter;
45-
46-
public SnapshotRetentionConfiguration(@Nullable TimeValue expireAfter) {
47-
this.expireAfter = expireAfter;
48-
}
58+
private final Integer minimumSnapshotCount;
59+
private final Integer maximumSnapshotCount;
4960

5061
SnapshotRetentionConfiguration(StreamInput in) throws IOException {
62+
nowSupplier = System::currentTimeMillis;
5163
this.expireAfter = in.readOptionalTimeValue();
64+
this.minimumSnapshotCount = in.readOptionalVInt();
65+
this.maximumSnapshotCount = in.readOptionalVInt();
66+
}
67+
68+
public SnapshotRetentionConfiguration(@Nullable TimeValue expireAfter,
69+
@Nullable Integer minimumSnapshotCount,
70+
@Nullable Integer maximumSnapshotCount) {
71+
this(System::currentTimeMillis, expireAfter, minimumSnapshotCount, maximumSnapshotCount);
72+
}
73+
74+
public SnapshotRetentionConfiguration(LongSupplier nowSupplier,
75+
@Nullable TimeValue expireAfter,
76+
@Nullable Integer minimumSnapshotCount,
77+
@Nullable Integer maximumSnapshotCount) {
78+
this.nowSupplier = nowSupplier;
79+
this.expireAfter = expireAfter;
80+
this.minimumSnapshotCount = minimumSnapshotCount;
81+
this.maximumSnapshotCount = maximumSnapshotCount;
82+
if (this.minimumSnapshotCount != null && this.minimumSnapshotCount < 1) {
83+
throw new IllegalArgumentException("minimum snapshot count must be at least 1, but was: " + this.minimumSnapshotCount);
84+
}
85+
if (this.maximumSnapshotCount != null && this.maximumSnapshotCount < 1) {
86+
throw new IllegalArgumentException("maximum snapshot count must be at least 1, but was: " + this.maximumSnapshotCount);
87+
}
88+
if ((maximumSnapshotCount != null && minimumSnapshotCount != null) && this.minimumSnapshotCount > this.maximumSnapshotCount) {
89+
throw new IllegalArgumentException("minimum snapshot count " + this.minimumSnapshotCount +
90+
" cannot be larger than maximum snapshot count " + this.maximumSnapshotCount);
91+
}
5292
}
5393

5494
public static SnapshotRetentionConfiguration parse(XContentParser parser, String name) {
@@ -59,44 +99,129 @@ public TimeValue getExpireAfter() {
5999
return this.expireAfter;
60100
}
61101

102+
public Integer getMinimumSnapshotCount() {
103+
return this.minimumSnapshotCount;
104+
}
105+
106+
public Integer getMaximumSnapshotCount() {
107+
return this.maximumSnapshotCount;
108+
}
109+
62110
/**
63111
* Return a predicate by which a SnapshotInfo can be tested to see
64112
* whether it should be deleted according to this retention policy.
65113
* @param allSnapshots a list of all snapshot pertaining to this SLM policy and repository
66114
*/
67115
public Predicate<SnapshotInfo> getSnapshotDeletionPredicate(final List<SnapshotInfo> allSnapshots) {
116+
final int snapCount = allSnapshots.size();
117+
List<SnapshotInfo> sortedSnapshots = allSnapshots.stream()
118+
.sorted(Comparator.comparingLong(SnapshotInfo::startTime))
119+
.collect(Collectors.toList());
120+
68121
return si -> {
122+
final String snapName = si.snapshotId().getName();
123+
124+
// First, enforce the maximum count, if the size is over the maximum number of
125+
// snapshots, then allow the oldest N (where N is the number over the maximum snapshot
126+
// count) snapshots to be eligible for deletion
127+
if (this.maximumSnapshotCount != null) {
128+
if (allSnapshots.size() > this.maximumSnapshotCount) {
129+
int snapsToDelete = allSnapshots.size() - this.maximumSnapshotCount;
130+
boolean eligible = sortedSnapshots.stream()
131+
.limit(snapsToDelete)
132+
.anyMatch(s -> s.equals(si));
133+
134+
if (eligible) {
135+
logger.trace("[{}]: ELIGIBLE as it is one of the {} oldest snapshots with " +
136+
"{} total snapshots, over the limit of {} maximum snapshots",
137+
snapName, snapsToDelete, snapCount, this.maximumSnapshotCount);
138+
return true;
139+
} else {
140+
logger.trace("[{}]: INELIGIBLE as it is not one of the {} oldest snapshots with " +
141+
"{} total snapshots, over the limit of {} maximum snapshots",
142+
snapName, snapsToDelete, snapCount, this.maximumSnapshotCount);
143+
return false;
144+
}
145+
}
146+
}
147+
148+
// Next check the minimum count, since that is a blanket requirement regardless of time,
149+
// if we haven't hit the minimum then we need to keep the snapshot regardless of
150+
// expiration time
151+
if (this.minimumSnapshotCount != null) {
152+
if (allSnapshots.size() <= this.minimumSnapshotCount) {
153+
logger.trace("[{}]: INELIGIBLE as there are {} snapshots and {} minimum snapshots needed",
154+
snapName, snapCount, this.minimumSnapshotCount);
155+
return false;
156+
}
157+
}
158+
159+
// Finally, check the expiration time of the snapshot, if it is past, then it is
160+
// eligible for deletion
69161
if (this.expireAfter != null) {
70-
TimeValue snapshotAge = new TimeValue(System.currentTimeMillis() - si.startTime());
162+
TimeValue snapshotAge = new TimeValue(nowSupplier.getAsLong() - si.startTime());
163+
164+
if (this.minimumSnapshotCount != null) {
165+
int eligibleForExpiration = snapCount - minimumSnapshotCount;
166+
167+
// Only the oldest N snapshots are actually eligible, since if we went below this we
168+
// would fall below the configured minimum number of snapshots to keep
169+
Set<SnapshotInfo> snapsEligibleForExpiration = sortedSnapshots.stream()
170+
.limit(eligibleForExpiration)
171+
.collect(Collectors.toSet());
172+
173+
if (snapsEligibleForExpiration.contains(si) == false) {
174+
// This snapshot is *not* one of the N oldest snapshots, so even if it were
175+
// old enough, the other snapshots would be deleted before it
176+
logger.trace("[{}]: INELIGIBLE as snapshot expiration would pass the " +
177+
"minimum number of configured snapshots ({}) to keep, regardless of age",
178+
snapName, this.minimumSnapshotCount);
179+
return false;
180+
}
181+
}
182+
71183
if (snapshotAge.compareTo(this.expireAfter) > 0) {
184+
logger.trace("[{}]: ELIGIBLE as snapshot age of {} is older than {}",
185+
snapName, snapshotAge.toHumanReadableString(3), this.expireAfter.toHumanReadableString(3));
72186
return true;
73187
} else {
188+
logger.trace("[{}]: INELIGIBLE as snapshot age of {} is newer than {}",
189+
snapName, snapshotAge.toHumanReadableString(3), this.expireAfter.toHumanReadableString(3));
74190
return false;
75191
}
76192
}
77193
// If nothing matched, the snapshot is not eligible for deletion
194+
logger.trace("[{}]: INELIGIBLE as no retention predicates matched", snapName);
78195
return false;
79196
};
80197
}
81198

199+
@Override
200+
public void writeTo(StreamOutput out) throws IOException {
201+
out.writeOptionalTimeValue(this.expireAfter);
202+
out.writeOptionalVInt(this.minimumSnapshotCount);
203+
out.writeOptionalVInt(this.maximumSnapshotCount);
204+
}
205+
82206
@Override
83207
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
84208
builder.startObject();
85209
if (expireAfter != null) {
86210
builder.field(EXPIRE_AFTER.getPreferredName(), expireAfter.getStringRep());
87211
}
212+
if (minimumSnapshotCount != null) {
213+
builder.field(MINIMUM_SNAPSHOT_COUNT.getPreferredName(), minimumSnapshotCount);
214+
}
215+
if (maximumSnapshotCount != null) {
216+
builder.field(MAXIMUM_SNAPSHOT_COUNT.getPreferredName(), maximumSnapshotCount);
217+
}
88218
builder.endObject();
89219
return builder;
90220
}
91221

92-
@Override
93-
public void writeTo(StreamOutput out) throws IOException {
94-
out.writeOptionalTimeValue(this.expireAfter);
95-
}
96-
97222
@Override
98223
public int hashCode() {
99-
return Objects.hash(expireAfter);
224+
return Objects.hash(expireAfter, minimumSnapshotCount, maximumSnapshotCount);
100225
}
101226

102227
@Override
@@ -108,7 +233,9 @@ public boolean equals(Object obj) {
108233
return false;
109234
}
110235
SnapshotRetentionConfiguration other = (SnapshotRetentionConfiguration) obj;
111-
return Objects.equals(this.expireAfter, other.expireAfter);
236+
return Objects.equals(this.expireAfter, other.expireAfter) &&
237+
Objects.equals(minimumSnapshotCount, other.minimumSnapshotCount) &&
238+
Objects.equals(maximumSnapshotCount, other.maximumSnapshotCount);
112239
}
113240

114241
@Override

‎x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicyMetadataTests.java

+4-2
Original file line numberDiff line numberDiff line change
@@ -111,8 +111,10 @@ public static SnapshotLifecyclePolicy randomSnapshotLifecyclePolicy(String polic
111111
}
112112

113113
public static SnapshotRetentionConfiguration randomRetention() {
114-
return rarely() ? null : new SnapshotRetentionConfiguration(rarely() ? null :
115-
TimeValue.parseTimeValue(randomTimeValue(), "random retention generation"));
114+
return rarely() ? null : new SnapshotRetentionConfiguration(
115+
rarely() ? null : TimeValue.parseTimeValue(randomTimeValue(), "random retention generation"),
116+
rarely() ? null : randomIntBetween(1, 10),
117+
rarely() ? null : randomIntBetween(15, 30));
116118
}
117119

118120
public static String randomSchedule() {

0 commit comments

Comments
 (0)