Skip to content

Commit 21bb472

Browse files
talevytalevy
authored
add notion of version and modified_date to LifecyclePolicyMetadata (#33450)
It is useful to keep track of which version of a policy is currently being executed by a specific index. For management purposes, it would also be useful to know at which time the latest version was inserted so that an audit trail is left for reconciling changes happening in ILM.
1 parent b335487 commit 21bb472

File tree

15 files changed

+234
-68
lines changed

15 files changed

+234
-68
lines changed

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/LifecyclePolicyMetadata.java

Lines changed: 42 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -18,23 +18,33 @@
1818
import org.elasticsearch.common.xcontent.XContentParser;
1919

2020
import java.io.IOException;
21+
import java.time.Instant;
22+
import java.time.ZoneOffset;
23+
import java.time.ZonedDateTime;
2124
import java.util.Map;
2225
import java.util.Objects;
2326

2427
public class LifecyclePolicyMetadata extends AbstractDiffable<LifecyclePolicyMetadata>
2528
implements ToXContentObject, Diffable<LifecyclePolicyMetadata> {
2629

27-
public static final ParseField POLICY = new ParseField("policy");
28-
public static final ParseField HEADERS = new ParseField("headers");
30+
static final ParseField POLICY = new ParseField("policy");
31+
static final ParseField HEADERS = new ParseField("headers");
32+
static final ParseField VERSION = new ParseField("version");
33+
static final ParseField MODIFIED_DATE = new ParseField("modified_date");
34+
static final ParseField MODIFIED_DATE_STRING = new ParseField("modified_date_string");
35+
2936
@SuppressWarnings("unchecked")
3037
public static final ConstructingObjectParser<LifecyclePolicyMetadata, String> PARSER = new ConstructingObjectParser<>("policy_metadata",
3138
a -> {
3239
LifecyclePolicy policy = (LifecyclePolicy) a[0];
33-
return new LifecyclePolicyMetadata(policy, (Map<String, String>) a[1]);
40+
return new LifecyclePolicyMetadata(policy, (Map<String, String>) a[1], (long) a[2], (long) a[3]);
3441
});
3542
static {
3643
PARSER.declareObject(ConstructingObjectParser.constructorArg(), LifecyclePolicy::parse, POLICY);
3744
PARSER.declareField(ConstructingObjectParser.constructorArg(), XContentParser::mapStrings, HEADERS, ValueType.OBJECT);
45+
PARSER.declareLong(ConstructingObjectParser.constructorArg(), VERSION);
46+
PARSER.declareLong(ConstructingObjectParser.constructorArg(), MODIFIED_DATE);
47+
PARSER.declareString(ConstructingObjectParser.constructorArg(), MODIFIED_DATE_STRING);
3848
}
3949

4050
public static LifecyclePolicyMetadata parse(XContentParser parser, String name) {
@@ -43,16 +53,22 @@ public static LifecyclePolicyMetadata parse(XContentParser parser, String name)
4353

4454
private final LifecyclePolicy policy;
4555
private final Map<String, String> headers;
56+
private final long version;
57+
private final long modifiedDate;
4658

47-
public LifecyclePolicyMetadata(LifecyclePolicy policy, Map<String, String> headers) {
59+
public LifecyclePolicyMetadata(LifecyclePolicy policy, Map<String, String> headers, long version, long modifiedDate) {
4860
this.policy = policy;
4961
this.headers = headers;
62+
this.version = version;
63+
this.modifiedDate = modifiedDate;
5064
}
5165

5266
@SuppressWarnings("unchecked")
5367
public LifecyclePolicyMetadata(StreamInput in) throws IOException {
5468
this.policy = new LifecyclePolicy(in);
5569
this.headers = (Map<String, String>) in.readGenericValue();
70+
this.version = in.readVLong();
71+
this.modifiedDate = in.readVLong();
5672
}
5773

5874
public Map<String, String> getHeaders() {
@@ -67,11 +83,27 @@ public String getName() {
6783
return policy.getName();
6884
}
6985

86+
public long getVersion() {
87+
return version;
88+
}
89+
90+
public long getModifiedDate() {
91+
return modifiedDate;
92+
}
93+
94+
public String getModifiedDateString() {
95+
ZonedDateTime modifiedDateTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(modifiedDate), ZoneOffset.UTC);
96+
return modifiedDateTime.toString();
97+
}
98+
7099
@Override
71100
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
72101
builder.startObject();
73102
builder.field(POLICY.getPreferredName(), policy);
74103
builder.field(HEADERS.getPreferredName(), headers);
104+
builder.field(VERSION.getPreferredName(), version);
105+
builder.field(MODIFIED_DATE.getPreferredName(), modifiedDate);
106+
builder.field(MODIFIED_DATE_STRING.getPreferredName(), getModifiedDateString());
75107
builder.endObject();
76108
return builder;
77109
}
@@ -80,11 +112,13 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
80112
public void writeTo(StreamOutput out) throws IOException {
81113
policy.writeTo(out);
82114
out.writeGenericValue(headers);
115+
out.writeVLong(version);
116+
out.writeVLong(modifiedDate);
83117
}
84118

85119
@Override
86120
public int hashCode() {
87-
return Objects.hash(policy, headers);
121+
return Objects.hash(policy, headers, version, modifiedDate);
88122
}
89123

90124
@Override
@@ -97,7 +131,9 @@ public boolean equals(Object obj) {
97131
}
98132
LifecyclePolicyMetadata other = (LifecyclePolicyMetadata) obj;
99133
return Objects.equals(policy, other.policy) &&
100-
Objects.equals(headers, other.headers);
134+
Objects.equals(headers, other.headers) &&
135+
Objects.equals(version, other.version) &&
136+
Objects.equals(modifiedDate, other.modifiedDate);
101137
}
102138

103139
}

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/GetLifecycleAction.java

Lines changed: 68 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
import org.elasticsearch.common.Strings;
1414
import org.elasticsearch.common.io.stream.StreamInput;
1515
import org.elasticsearch.common.io.stream.StreamOutput;
16+
import org.elasticsearch.common.io.stream.Writeable;
1617
import org.elasticsearch.common.xcontent.ToXContentObject;
1718
import org.elasticsearch.common.xcontent.XContentBuilder;
1819
import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicy;
@@ -37,32 +38,36 @@ public Response newResponse() {
3738

3839
public static class Response extends ActionResponse implements ToXContentObject {
3940

40-
private List<LifecyclePolicy> policies;
41+
private List<LifecyclePolicyResponseItem> policies;
4142

4243
public Response() {
4344
}
4445

45-
public Response(List<LifecyclePolicy> policies) {
46+
public Response(List<LifecyclePolicyResponseItem> policies) {
4647
this.policies = policies;
4748
}
4849

49-
public List<LifecyclePolicy> getPolicies() {
50+
public List<LifecyclePolicyResponseItem> getPolicies() {
5051
return policies;
5152
}
5253

5354
@Override
5455
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
5556
builder.startObject();
56-
for (LifecyclePolicy policy : policies) {
57-
builder.field(policy.getName(), policy);
57+
for (LifecyclePolicyResponseItem item : policies) {
58+
builder.startObject(item.getLifecyclePolicy().getName());
59+
builder.field("version", item.getVersion());
60+
builder.field("modified_date", item.getModifiedDate());
61+
builder.field("policy", item.getLifecyclePolicy());
62+
builder.endObject();
5863
}
5964
builder.endObject();
6065
return builder;
6166
}
6267

6368
@Override
6469
public void readFrom(StreamInput in) throws IOException {
65-
policies = in.readList(LifecyclePolicy::new);
70+
this.policies = in.readList(LifecyclePolicyResponseItem::new);
6671
}
6772

6873
@Override
@@ -148,4 +153,61 @@ public boolean equals(Object obj) {
148153

149154
}
150155

156+
public static class LifecyclePolicyResponseItem implements Writeable {
157+
private final LifecyclePolicy lifecyclePolicy;
158+
private final long version;
159+
private final String modifiedDate;
160+
161+
public LifecyclePolicyResponseItem(LifecyclePolicy lifecyclePolicy, long version, String modifiedDate) {
162+
this.lifecyclePolicy = lifecyclePolicy;
163+
this.version = version;
164+
this.modifiedDate = modifiedDate;
165+
}
166+
167+
LifecyclePolicyResponseItem(StreamInput in) throws IOException {
168+
this.lifecyclePolicy = new LifecyclePolicy(in);
169+
this.version = in.readVLong();
170+
this.modifiedDate = in.readString();
171+
}
172+
173+
@Override
174+
public void writeTo(StreamOutput out) throws IOException {
175+
lifecyclePolicy.writeTo(out);
176+
out.writeVLong(version);
177+
out.writeString(modifiedDate);
178+
}
179+
180+
public LifecyclePolicy getLifecyclePolicy() {
181+
return lifecyclePolicy;
182+
}
183+
184+
public long getVersion() {
185+
return version;
186+
}
187+
188+
public String getModifiedDate() {
189+
return modifiedDate;
190+
}
191+
192+
@Override
193+
public int hashCode() {
194+
return Objects.hash(lifecyclePolicy, version, modifiedDate);
195+
}
196+
197+
@Override
198+
public boolean equals(Object obj) {
199+
if (obj == null) {
200+
return false;
201+
}
202+
if (obj.getClass() != getClass()) {
203+
return false;
204+
}
205+
LifecyclePolicyResponseItem other = (LifecyclePolicyResponseItem) obj;
206+
return Objects.equals(lifecyclePolicy, other.lifecyclePolicy) &&
207+
Objects.equals(version, other.version) &&
208+
Objects.equals(modifiedDate, other.modifiedDate);
209+
}
210+
211+
}
212+
151213
}

‎x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/LifecyclePolicyMetadataTests.java

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,8 @@ protected LifecyclePolicyMetadata createTestInstance() {
7575
for (int i = 0; i < numberHeaders; i++) {
7676
headers.put(randomAlphaOfLength(10), randomAlphaOfLength(10));
7777
}
78-
return new LifecyclePolicyMetadata(LifecyclePolicyTests.randomTimeseriesLifecyclePolicy(lifecycleName), headers);
78+
return new LifecyclePolicyMetadata(LifecyclePolicyTests.randomTimeseriesLifecyclePolicy(lifecycleName), headers,
79+
randomNonNegativeLong(), randomNonNegativeLong());
7980
}
8081

8182
@Override
@@ -87,7 +88,9 @@ protected Reader<LifecyclePolicyMetadata> instanceReader() {
8788
protected LifecyclePolicyMetadata mutateInstance(LifecyclePolicyMetadata instance) throws IOException {
8889
LifecyclePolicy policy = instance.getPolicy();
8990
Map<String, String> headers = instance.getHeaders();
90-
switch (between(0, 1)) {
91+
long version = instance.getVersion();
92+
long creationDate = instance.getModifiedDate();
93+
switch (between(0, 3)) {
9194
case 0:
9295
policy = new LifecyclePolicy(TimeseriesLifecycleType.INSTANCE, policy.getName() + randomAlphaOfLengthBetween(1, 5),
9396
policy.getPhases());
@@ -96,10 +99,16 @@ protected LifecyclePolicyMetadata mutateInstance(LifecyclePolicyMetadata instanc
9699
headers = new HashMap<>(headers);
97100
headers.put(randomAlphaOfLength(11), randomAlphaOfLength(11));
98101
break;
102+
case 2:
103+
version++;
104+
break;
105+
case 3:
106+
creationDate++;
107+
break;
99108
default:
100109
throw new AssertionError("Illegal randomisation branch");
101110
}
102-
return new LifecyclePolicyMetadata(policy, headers);
111+
return new LifecyclePolicyMetadata(policy, headers, version, creationDate);
103112
}
104113

105114
}

‎x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/action/GetLifecycleResponseTests.java

Lines changed: 13 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,10 @@
88
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
99
import org.elasticsearch.test.AbstractStreamableTestCase;
1010
import org.elasticsearch.xpack.core.indexlifecycle.LifecycleAction;
11-
import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicy;
1211
import org.elasticsearch.xpack.core.indexlifecycle.LifecycleType;
1312
import org.elasticsearch.xpack.core.indexlifecycle.MockAction;
1413
import org.elasticsearch.xpack.core.indexlifecycle.TestLifecycleType;
14+
import org.elasticsearch.xpack.core.indexlifecycle.action.GetLifecycleAction.LifecyclePolicyResponseItem;
1515
import org.elasticsearch.xpack.core.indexlifecycle.action.GetLifecycleAction.Response;
1616

1717
import java.util.ArrayList;
@@ -25,11 +25,12 @@ public class GetLifecycleResponseTests extends AbstractStreamableTestCase<GetLif
2525
@Override
2626
protected Response createTestInstance() {
2727
String randomPrefix = randomAlphaOfLength(5);
28-
List<LifecyclePolicy> policies = new ArrayList<>();
28+
List<LifecyclePolicyResponseItem> responseItems = new ArrayList<>();
2929
for (int i = 0; i < randomIntBetween(0, 2); i++) {
30-
policies.add(randomTestLifecyclePolicy(randomPrefix + i));
30+
responseItems.add(new LifecyclePolicyResponseItem(randomTestLifecyclePolicy(randomPrefix + i),
31+
randomNonNegativeLong(), randomAlphaOfLength(8)));
3132
}
32-
return new Response(policies);
33+
return new Response(responseItems);
3334
}
3435

3536
@Override
@@ -45,16 +46,18 @@ protected NamedWriteableRegistry getNamedWriteableRegistry() {
4546

4647
@Override
4748
protected Response mutateInstance(Response response) {
48-
List<LifecyclePolicy> policies = new ArrayList<>(response.getPolicies());
49-
if (policies.size() > 0) {
49+
List<LifecyclePolicyResponseItem> responseItems = new ArrayList<>(response.getPolicies());
50+
if (responseItems.size() > 0) {
5051
if (randomBoolean()) {
51-
policies.add(randomTestLifecyclePolicy(randomAlphaOfLength(5)));
52+
responseItems.add(new LifecyclePolicyResponseItem(randomTestLifecyclePolicy(randomAlphaOfLength(5)),
53+
randomNonNegativeLong(), randomAlphaOfLength(4)));
5254
} else {
53-
policies.remove(policies.size() - 1);
55+
responseItems.remove(0);
5456
}
5557
} else {
56-
policies.add(randomTestLifecyclePolicy(randomAlphaOfLength(2)));
58+
responseItems.add(new LifecyclePolicyResponseItem(randomTestLifecyclePolicy(randomAlphaOfLength(2)),
59+
randomNonNegativeLong(), randomAlphaOfLength(4)));
5760
}
58-
return new Response(policies);
61+
return new Response(responseItems);
5962
}
6063
}

‎x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportGetLifecycleAction.java

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,9 @@
2020
import org.elasticsearch.threadpool.ThreadPool;
2121
import org.elasticsearch.transport.TransportService;
2222
import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleMetadata;
23-
import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicy;
23+
import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicyMetadata;
2424
import org.elasticsearch.xpack.core.indexlifecycle.action.GetLifecycleAction;
25+
import org.elasticsearch.xpack.core.indexlifecycle.action.GetLifecycleAction.LifecyclePolicyResponseItem;
2526
import org.elasticsearch.xpack.core.indexlifecycle.action.GetLifecycleAction.Request;
2627
import org.elasticsearch.xpack.core.indexlifecycle.action.GetLifecycleAction.Response;
2728

@@ -49,24 +50,29 @@ protected Response newResponse() {
4950
}
5051

5152
@Override
52-
protected void masterOperation(Request request, ClusterState state, ActionListener<Response> listener) throws Exception {
53+
protected void masterOperation(Request request, ClusterState state, ActionListener<Response> listener) {
5354
IndexLifecycleMetadata metadata = clusterService.state().metaData().custom(IndexLifecycleMetadata.TYPE);
5455
if (metadata == null) {
5556
listener.onFailure(new ResourceNotFoundException("Lifecycle policy not found: {}", Arrays.toString(request.getPolicyNames())));
5657
} else {
57-
List<LifecyclePolicy> requestedPolicies;
58+
List<LifecyclePolicyResponseItem> requestedPolicies;
5859
// if no policies explicitly provided, behave as if `*` was specified
5960
if (request.getPolicyNames().length == 0) {
60-
requestedPolicies = new ArrayList<>(metadata.getPolicies().values());
61+
requestedPolicies = new ArrayList<>(metadata.getPolicyMetadatas().size());
62+
for (LifecyclePolicyMetadata policyMetadata : metadata.getPolicyMetadatas().values()) {
63+
requestedPolicies.add(new LifecyclePolicyResponseItem(policyMetadata.getPolicy(),
64+
policyMetadata.getVersion(), policyMetadata.getModifiedDateString()));
65+
}
6166
} else {
6267
requestedPolicies = new ArrayList<>(request.getPolicyNames().length);
6368
for (String name : request.getPolicyNames()) {
64-
LifecyclePolicy policy = metadata.getPolicies().get(name);
65-
if (policy == null) {
69+
LifecyclePolicyMetadata policyMetadata = metadata.getPolicyMetadatas().get(name);
70+
if (policyMetadata == null) {
6671
listener.onFailure(new ResourceNotFoundException("Lifecycle policy not found: {}", name));
6772
return;
6873
}
69-
requestedPolicies.add(policy);
74+
requestedPolicies.add(new LifecyclePolicyResponseItem(policyMetadata.getPolicy(),
75+
policyMetadata.getVersion(), policyMetadata.getModifiedDateString()));
7076
}
7177
}
7278
listener.onResponse(new Response(requestedPolicies));
@@ -77,4 +83,4 @@ protected void masterOperation(Request request, ClusterState state, ActionListen
7783
protected ClusterBlockException checkBlock(Request request, ClusterState state) {
7884
return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE);
7985
}
80-
}
86+
}

‎x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportPutLifecycleAction.java

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import org.elasticsearch.xpack.core.indexlifecycle.action.PutLifecycleAction.Request;
2929
import org.elasticsearch.xpack.core.indexlifecycle.action.PutLifecycleAction.Response;
3030

31+
import java.time.Instant;
3132
import java.util.Map;
3233
import java.util.SortedMap;
3334
import java.util.TreeMap;
@@ -79,9 +80,12 @@ public ClusterState execute(ClusterState currentState) throws Exception {
7980
if (currentMetadata == null) { // first time using index-lifecycle feature, bootstrap metadata
8081
currentMetadata = IndexLifecycleMetadata.EMPTY;
8182
}
82-
// NORELEASE Check if current step exists in new policy and if not move to next available step
83+
LifecyclePolicyMetadata existingPolicyMetadata = currentMetadata.getPolicyMetadatas()
84+
.get(request.getPolicy().getName());
85+
long nextVersion = (existingPolicyMetadata == null) ? 1L : existingPolicyMetadata.getVersion() + 1L;
8386
SortedMap<String, LifecyclePolicyMetadata> newPolicies = new TreeMap<>(currentMetadata.getPolicyMetadatas());
84-
LifecyclePolicyMetadata lifecyclePolicyMetadata = new LifecyclePolicyMetadata(request.getPolicy(), filteredHeaders);
87+
LifecyclePolicyMetadata lifecyclePolicyMetadata = new LifecyclePolicyMetadata(request.getPolicy(), filteredHeaders,
88+
nextVersion, Instant.now().toEpochMilli());
8589
newPolicies.put(lifecyclePolicyMetadata.getName(), lifecyclePolicyMetadata);
8690
IndexLifecycleMetadata newMetadata = new IndexLifecycleMetadata(newPolicies, OperationMode.RUNNING);
8791
newState.metaData(MetaData.builder(currentState.getMetaData())

0 commit comments

Comments
 (0)