Skip to content

Commit c363d27

Browse files
andreidanandreidan
authored
ILM: parse origination date from index name (#46755)
* ILM: parse origination date from index name Introduce the `index.lifecycle.parse_origination_date` setting that indicates if the origination date should be parsed from the index name. If set to true an index which doesn't match the expected format (namely `indexName-{dateFormat}-optional_digits` will fail before being created. The origination date will be parsed when initialising a lifecycle for an index and it will be set as the `index.lifecycle.origination_date` for that index. A user set value for `index.lifecycle.origination_date` will always override a possible parsable date from the index name.
1 parent 2c7fd82 commit c363d27

File tree

9 files changed

+314
-10
lines changed

9 files changed

+314
-10
lines changed

‎docs/reference/settings/ilm-settings.asciidoc

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,14 @@ information about rollover, see <<using-policies-rollover>>.
2727
(<<time-units, time units>>) How often {ilm} checks for indices that meet policy
2828
criteria. Defaults to `10m`.
2929

30+
`index.lifecycle.parse_origination_date`::
31+
When configured to `true` the origination date will be parsed from the index
32+
name. The index format must match the pattern `^.*-{date_format}-\\d+`, where
33+
the `date_format` is `yyyy.MM.dd` and the trailing digits are optional (an
34+
index that was rolled over would normally match the full format eg.
35+
`logs-2016.10.31-000002`). If the index name doesn't match the pattern
36+
the index creation will fail.
37+
3038
`index.lifecycle.origination_date`::
3139
The timestamp that will be used to calculate the index age for its phase
3240
transitions. This allows the users to create an index containing old data and

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleOriginationDateParser.java

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
package org.elasticsearch.xpack.core.ilm;
7+
8+
import org.elasticsearch.ElasticsearchParseException;
9+
import org.elasticsearch.common.settings.Settings;
10+
import org.elasticsearch.common.time.DateFormatter;
11+
12+
import java.util.regex.Matcher;
13+
import java.util.regex.Pattern;
14+
15+
import static org.elasticsearch.xpack.core.ilm.LifecycleSettings.LIFECYCLE_ORIGINATION_DATE;
16+
import static org.elasticsearch.xpack.core.ilm.LifecycleSettings.LIFECYCLE_PARSE_ORIGINATION_DATE;
17+
18+
public class IndexLifecycleOriginationDateParser {
19+
20+
private static final DateFormatter DATE_FORMATTER = DateFormatter.forPattern("yyyy.MM.dd");
21+
private static final String INDEX_NAME_REGEX = "^.*-(\\d{4}.\\d{2}.\\d{2})(-[\\d]+)?$";
22+
private static final Pattern INDEX_NAME_PATTERN = Pattern.compile(INDEX_NAME_REGEX);
23+
24+
/**
25+
* Determines if the origination date needs to be parsed from the index name.
26+
*/
27+
public static boolean shouldParseIndexName(Settings indexSettings) {
28+
return indexSettings.getAsLong(LIFECYCLE_ORIGINATION_DATE, -1L) == -1L &&
29+
indexSettings.getAsBoolean(LIFECYCLE_PARSE_ORIGINATION_DATE, false);
30+
}
31+
32+
/**
33+
* Parses the index according to the supported format and extracts the origination date. If the index does not match the expected
34+
* format or the date in the index name doesn't match the `yyyy.MM.dd` format it throws an {@link IllegalArgumentException}
35+
*/
36+
public static long parseIndexNameAndExtractDate(String indexName) {
37+
Matcher matcher = INDEX_NAME_PATTERN.matcher(indexName);
38+
if (matcher.matches()) {
39+
String dateAsString = matcher.group(1);
40+
try {
41+
return DATE_FORMATTER.parseMillis(dateAsString);
42+
} catch (ElasticsearchParseException | IllegalArgumentException e) {
43+
throw new IllegalArgumentException("index name [" + indexName + "] contains date [" + dateAsString + "] which " +
44+
"couldn't be parsed using the 'yyyy.MM.dd' format", e);
45+
}
46+
}
47+
48+
throw new IllegalArgumentException("index name [" + indexName + "] does not match pattern '" + INDEX_NAME_REGEX + "'");
49+
}
50+
}

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/InitializePolicyContextStep.java

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,11 @@
1010
import org.elasticsearch.cluster.ClusterState;
1111
import org.elasticsearch.cluster.metadata.IndexMetaData;
1212
import org.elasticsearch.cluster.metadata.MetaData;
13+
import org.elasticsearch.common.settings.Settings;
1314
import org.elasticsearch.index.Index;
1415

16+
import static org.elasticsearch.xpack.core.ilm.IndexLifecycleOriginationDateParser.parseIndexNameAndExtractDate;
17+
import static org.elasticsearch.xpack.core.ilm.IndexLifecycleOriginationDateParser.shouldParseIndexName;
1518
import static org.elasticsearch.xpack.core.ilm.LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY;
1619

1720
/**
@@ -34,19 +37,34 @@ public ClusterState performAction(Index index, ClusterState clusterState) {
3437
// Index must have been since deleted, ignore it
3538
return clusterState;
3639
}
40+
3741
LifecycleExecutionState lifecycleState = LifecycleExecutionState
3842
.fromIndexMetadata(indexMetaData);
43+
3944
if (lifecycleState.getLifecycleDate() != null) {
4045
return clusterState;
4146
}
4247

48+
IndexMetaData.Builder indexMetadataBuilder = IndexMetaData.builder(indexMetaData);
49+
if (shouldParseIndexName(indexMetaData.getSettings())) {
50+
long parsedOriginationDate = parseIndexNameAndExtractDate(index.getName());
51+
indexMetadataBuilder.settingsVersion(indexMetaData.getSettingsVersion() + 1)
52+
.settings(Settings.builder()
53+
.put(indexMetaData.getSettings())
54+
.put(LifecycleSettings.LIFECYCLE_ORIGINATION_DATE, parsedOriginationDate)
55+
.build()
56+
);
57+
}
58+
4359
ClusterState.Builder newClusterStateBuilder = ClusterState.builder(clusterState);
4460

4561
LifecycleExecutionState.Builder newCustomData = LifecycleExecutionState.builder(lifecycleState);
4662
newCustomData.setIndexCreationDate(indexMetaData.getCreationDate());
47-
newClusterStateBuilder.metaData(MetaData.builder(clusterState.getMetaData()).put(IndexMetaData
48-
.builder(indexMetaData)
49-
.putCustom(ILM_CUSTOM_METADATA_KEY, newCustomData.build().asMap())));
63+
indexMetadataBuilder.putCustom(ILM_CUSTOM_METADATA_KEY, newCustomData.build().asMap());
64+
65+
newClusterStateBuilder.metaData(
66+
MetaData.builder(clusterState.getMetaData()).put(indexMetadataBuilder)
67+
);
5068
return newClusterStateBuilder.build();
5169
}
5270
}

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecycleSettings.java

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ public class LifecycleSettings {
1818
public static final String LIFECYCLE_NAME = "index.lifecycle.name";
1919
public static final String LIFECYCLE_INDEXING_COMPLETE = "index.lifecycle.indexing_complete";
2020
public static final String LIFECYCLE_ORIGINATION_DATE = "index.lifecycle.origination_date";
21+
public static final String LIFECYCLE_PARSE_ORIGINATION_DATE = "index.lifecycle.parse_origination_date";
2122

2223
public static final String SLM_HISTORY_INDEX_ENABLED = "slm.history_index_enabled";
2324
public static final String SLM_RETENTION_SCHEDULE = "slm.retention_schedule";
@@ -32,6 +33,8 @@ public class LifecycleSettings {
3233
Setting.Property.Dynamic, Setting.Property.IndexScope);
3334
public static final Setting<Long> LIFECYCLE_ORIGINATION_DATE_SETTING =
3435
Setting.longSetting(LIFECYCLE_ORIGINATION_DATE, -1, -1, Setting.Property.Dynamic, Setting.Property.IndexScope);
36+
public static final Setting<Boolean> LIFECYCLE_PARSE_ORIGINATION_DATE_SETTING = Setting.boolSetting(LIFECYCLE_PARSE_ORIGINATION_DATE,
37+
false, Setting.Property.Dynamic, Setting.Property.IndexScope);
3538

3639
public static final Setting<Boolean> SLM_HISTORY_INDEX_ENABLED_SETTING = Setting.boolSetting(SLM_HISTORY_INDEX_ENABLED, true,
3740
Setting.Property.NodeScope);

‎x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleOriginationDateParserTests.java

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
package org.elasticsearch.xpack.core.ilm;
7+
8+
import org.elasticsearch.common.settings.Settings;
9+
import org.elasticsearch.test.ESTestCase;
10+
11+
import java.text.ParseException;
12+
import java.text.SimpleDateFormat;
13+
import java.util.Locale;
14+
import java.util.TimeZone;
15+
16+
import static org.elasticsearch.xpack.core.ilm.IndexLifecycleOriginationDateParser.parseIndexNameAndExtractDate;
17+
import static org.elasticsearch.xpack.core.ilm.IndexLifecycleOriginationDateParser.shouldParseIndexName;
18+
import static org.hamcrest.Matchers.is;
19+
20+
public class IndexLifecycleOriginationDateParserTests extends ESTestCase {
21+
22+
public void testShouldParseIndexNameReturnsFalseWhenOriginationDateIsSet() {
23+
Settings settings = Settings.builder()
24+
.put(LifecycleSettings.LIFECYCLE_ORIGINATION_DATE, 1L)
25+
.build();
26+
assertThat(shouldParseIndexName(settings), is(false));
27+
}
28+
29+
public void testShouldParseIndexNameReturnsFalseIfParseOriginationDateIsDisabled() {
30+
Settings settings = Settings.builder()
31+
.put(LifecycleSettings.LIFECYCLE_PARSE_ORIGINATION_DATE, false)
32+
.build();
33+
assertThat(shouldParseIndexName(settings), is(false));
34+
}
35+
36+
public void testShouldParseIndexNameReturnsTrueIfParseOriginationDateIsTrueAndOriginationDateIsNotSet() {
37+
Settings settings = Settings.builder()
38+
.put(LifecycleSettings.LIFECYCLE_PARSE_ORIGINATION_DATE, true)
39+
.build();
40+
assertThat(shouldParseIndexName(settings), is(true));
41+
}
42+
43+
public void testParseIndexNameThatMatchesExpectedFormat() throws ParseException {
44+
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy.MM.dd", Locale.getDefault());
45+
dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
46+
long expectedDate = dateFormat.parse("2019.09.04").getTime();
47+
48+
{
49+
long parsedDate = parseIndexNameAndExtractDate("indexName-2019.09.04");
50+
assertThat("indexName-yyyy.MM.dd is a valid index format", parsedDate, is(expectedDate));
51+
}
52+
53+
{
54+
long parsedDate = parseIndexNameAndExtractDate("indexName-2019.09.04-0000001");
55+
assertThat("indexName-yyyy.MM.dd-\\d+$ is a valid index format", parsedDate, is(expectedDate));
56+
}
57+
58+
{
59+
long parsedDate = parseIndexNameAndExtractDate("indexName-2019.09.04-2019.09.24");
60+
long secondDateInIndexName = dateFormat.parse("2019.09.24").getTime();
61+
assertThat("indexName-yyyy.MM.dd-yyyy.MM.dd is a valid index format and the second date should be parsed",
62+
parsedDate, is(secondDateInIndexName));
63+
}
64+
65+
{
66+
long parsedDate = parseIndexNameAndExtractDate("index-2019.09.04-2019.09.24-00002");
67+
long secondDateInIndexName = dateFormat.parse("2019.09.24").getTime();
68+
assertThat("indexName-yyyy.MM.dd-yyyy.MM.dd-digits is a valid index format and the second date should be parsed",
69+
parsedDate, is(secondDateInIndexName));
70+
}
71+
}
72+
73+
public void testParseIndexNameThrowsIllegalArgumentExceptionForInvalidIndexFormat() {
74+
expectThrows(
75+
IllegalArgumentException.class,
76+
"plainIndexName does not match the expected pattern",
77+
() -> parseIndexNameAndExtractDate("plainIndexName")
78+
);
79+
80+
expectThrows(
81+
IllegalArgumentException.class,
82+
"indexName--00001 does not match the expected pattern as the origination date is missing",
83+
() -> parseIndexNameAndExtractDate("indexName--00001")
84+
);
85+
86+
expectThrows(
87+
IllegalArgumentException.class,
88+
"indexName-00001 does not match the expected pattern as the origination date is missing",
89+
() -> parseIndexNameAndExtractDate("indexName-00001")
90+
);
91+
92+
expectThrows(
93+
IllegalArgumentException.class,
94+
"indexName_2019.09.04_00001 does not match the expected pattern as _ is not the expected delimiter",
95+
() -> parseIndexNameAndExtractDate("indexName_2019.09.04_00001")
96+
);
97+
}
98+
99+
public void testParseIndexNameThrowsIllegalArgumentExceptionForInvalidDateFormat() {
100+
expectThrows(
101+
IllegalArgumentException.class,
102+
"indexName-2019.04-00001 does not match the expected pattern as the date does not conform with the yyyy.MM.dd pattern",
103+
() -> parseIndexNameAndExtractDate("indexName-2019.04-00001")
104+
);
105+
106+
expectThrows(
107+
IllegalArgumentException.class,
108+
"java.lang.IllegalArgumentException: failed to parse date field [2019.09.44] with format [yyyy.MM.dd]",
109+
() -> parseIndexNameAndExtractDate("index-2019.09.44")
110+
);
111+
}
112+
}

‎x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import org.elasticsearch.core.internal.io.IOUtils;
2828
import org.elasticsearch.env.Environment;
2929
import org.elasticsearch.env.NodeEnvironment;
30+
import org.elasticsearch.index.IndexModule;
3031
import org.elasticsearch.plugins.ActionPlugin;
3132
import org.elasticsearch.plugins.Plugin;
3233
import org.elasticsearch.rest.RestController;
@@ -141,6 +142,7 @@ public List<Setting<?>> getSettings() {
141142
LifecycleSettings.LIFECYCLE_POLL_INTERVAL_SETTING,
142143
LifecycleSettings.LIFECYCLE_NAME_SETTING,
143144
LifecycleSettings.LIFECYCLE_ORIGINATION_DATE_SETTING,
145+
LifecycleSettings.LIFECYCLE_PARSE_ORIGINATION_DATE_SETTING,
144146
LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE_SETTING,
145147
RolloverAction.LIFECYCLE_ROLLOVER_ALIAS_SETTING,
146148
LifecycleSettings.SLM_HISTORY_INDEX_ENABLED_SETTING,
@@ -268,6 +270,14 @@ public List<RestHandler> getRestHandlers(Settings settings, RestController restC
268270
return actions;
269271
}
270272

273+
@Override
274+
public void onIndexModule(IndexModule indexModule) {
275+
if (ilmEnabled) {
276+
assert indexLifecycleInitialisationService.get() != null;
277+
indexModule.addIndexEventListener(indexLifecycleInitialisationService.get());
278+
}
279+
}
280+
271281
@Override
272282
public void close() {
273283
try {

‎x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycleService.java

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@
2222
import org.elasticsearch.common.settings.Settings;
2323
import org.elasticsearch.common.unit.TimeValue;
2424
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
25+
import org.elasticsearch.index.Index;
26+
import org.elasticsearch.index.shard.IndexEventListener;
2527
import org.elasticsearch.threadpool.ThreadPool;
2628
import org.elasticsearch.xpack.core.XPackField;
2729
import org.elasticsearch.xpack.core.ilm.IndexLifecycleMetadata;
@@ -39,11 +41,14 @@
3941
import java.util.Set;
4042
import java.util.function.LongSupplier;
4143

44+
import static org.elasticsearch.xpack.core.ilm.IndexLifecycleOriginationDateParser.parseIndexNameAndExtractDate;
45+
import static org.elasticsearch.xpack.core.ilm.IndexLifecycleOriginationDateParser.shouldParseIndexName;
46+
4247
/**
4348
* A service which runs the {@link LifecyclePolicy}s associated with indexes.
4449
*/
4550
public class IndexLifecycleService
46-
implements ClusterStateListener, ClusterStateApplier, SchedulerEngine.Listener, Closeable, LocalNodeMasterListener {
51+
implements ClusterStateListener, ClusterStateApplier, SchedulerEngine.Listener, Closeable, LocalNodeMasterListener, IndexEventListener {
4752
private static final Logger logger = LogManager.getLogger(IndexLifecycleService.class);
4853
private static final Set<String> IGNORE_STEPS_MAINTENANCE_REQUESTED = Collections.singleton(ShrinkStep.NAME);
4954
private volatile boolean isMaster = false;
@@ -148,6 +153,13 @@ public String executorName() {
148153
return ThreadPool.Names.MANAGEMENT;
149154
}
150155

156+
@Override
157+
public void beforeIndexAddedToCluster(Index index, Settings indexSettings) {
158+
if (shouldParseIndexName(indexSettings)) {
159+
parseIndexNameAndExtractDate(index.getName());
160+
}
161+
}
162+
151163
private void updatePollInterval(TimeValue newInterval) {
152164
this.pollInterval = newInterval;
153165
maybeScheduleJob();

0 commit comments

Comments
 (0)