Skip to content

Commit d23e256

Browse files
przemekwitekprzemekwitek
committed
Create .ml-state-000001 index instead of .ml-state
1 parent 0c309ef commit d23e256

File tree

15 files changed

+135
-76
lines changed

15 files changed

+135
-76
lines changed

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndex.java

Lines changed: 34 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,9 @@
2121
import org.elasticsearch.xpack.core.template.TemplateUtils;
2222

2323
import java.util.Arrays;
24-
import java.util.Collections;
24+
import java.util.Comparator;
25+
import java.util.function.Predicate;
26+
import java.util.regex.Pattern;
2527

2628
import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN;
2729
import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin;
@@ -36,6 +38,29 @@ public final class AnomalyDetectorsIndex {
3638
private static final String RESULTS_MAPPINGS_VERSION_VARIABLE = "xpack.ml.version";
3739
private static final String RESOURCE_PATH = "/org/elasticsearch/xpack/core/ml/anomalydetection/";
3840

41+
// Visible for testing
42+
static final Comparator<String> STATE_INDEX_NAME_COMPARATOR = new Comparator<>() {
43+
44+
private final Predicate<String> HAS_SIX_DIGIT_SUFFIX = Pattern.compile("\\d{6}").asMatchPredicate();
45+
46+
@Override
47+
public int compare(String index1, String index2) {
48+
String[] index1Parts = index1.split("-");
49+
String index1Suffix = index1Parts[index1Parts.length - 1];
50+
boolean index1HasSixDigitsSuffix = HAS_SIX_DIGIT_SUFFIX.test(index1Suffix);
51+
String[] index2Parts = index2.split("-");
52+
String index2Suffix = index2Parts[index2Parts.length - 1];
53+
boolean index2HasSixDigitsSuffix = HAS_SIX_DIGIT_SUFFIX.test(index2Suffix);
54+
if (index1HasSixDigitsSuffix && index2HasSixDigitsSuffix) {
55+
return index1Suffix.compareTo(index2Suffix);
56+
} else if (index1HasSixDigitsSuffix != index2HasSixDigitsSuffix) {
57+
return Boolean.compare(index1HasSixDigitsSuffix, index2HasSixDigitsSuffix);
58+
} else {
59+
return index1.compareTo(index2);
60+
}
61+
}
62+
};
63+
3964
private AnomalyDetectorsIndex() {
4065
}
4166

@@ -89,8 +114,8 @@ public static String configIndexName() {
89114
}
90115

91116
/**
92-
* Create the .ml-state index (if necessary)
93-
* Create the .ml-state-write alias for the .ml-state index (if necessary)
117+
* Creates the .ml-state-000001 index (if necessary)
118+
* Creates the .ml-state-write alias for the .ml-state-000001 index (if necessary)
94119
*/
95120
public static void createStateIndexAndAliasIfNecessary(Client client, ClusterState state, final ActionListener<Boolean> finalListener) {
96121

@@ -122,12 +147,14 @@ public static void createStateIndexAndAliasIfNecessary(Client client, ClusterSta
122147
IndicesOptions.lenientExpandOpen(),
123148
jobStateIndexPattern());
124149
if (stateIndices.length > 0) {
125-
Arrays.sort(stateIndices, Collections.reverseOrder());
126-
createAliasListener.onResponse(stateIndices[0]);
150+
String latestStateIndex = Arrays.stream(stateIndices).max(STATE_INDEX_NAME_COMPARATOR).get();
151+
createAliasListener.onResponse(latestStateIndex);
127152
} else {
153+
// The initial index name must be suitable for rollover functionality.
154+
String initialJobStateIndex = AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX + "-000001";
128155
CreateIndexRequest createIndexRequest = client.admin()
129156
.indices()
130-
.prepareCreate(AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX)
157+
.prepareCreate(initialJobStateIndex)
131158
.addAlias(new Alias(jobStateIndexWriteAlias()))
132159
.request();
133160
executeAsyncWithOrigin(client.threadPool().getThreadContext(),
@@ -140,7 +167,7 @@ public static void createStateIndexAndAliasIfNecessary(Client client, ClusterSta
140167
// Adding an alias that already exists is idempotent. So, no need to double check if the alias exists
141168
// as well.
142169
if (ExceptionsHelper.unwrapCause(createIndexFailure) instanceof ResourceAlreadyExistsException) {
143-
createAliasListener.onResponse(AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX);
170+
createAliasListener.onResponse(initialJobStateIndex);
144171
} else {
145172
finalListener.onFailure(createIndexFailure);
146173
}

‎x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndexTests.java

Lines changed: 46 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -38,9 +38,11 @@
3838

3939
import java.util.Arrays;
4040
import java.util.Collections;
41+
import java.util.Comparator;
4142
import java.util.List;
4243
import java.util.Map;
4344
import java.util.function.Function;
45+
import java.util.stream.Stream;
4446

4547
import static java.util.stream.Collectors.toMap;
4648
import static org.hamcrest.Matchers.contains;
@@ -55,7 +57,8 @@
5557

5658
public class AnomalyDetectorsIndexTests extends ESTestCase {
5759

58-
private static final String ML_STATE = ".ml-state";
60+
private static final String LEGACY_ML_STATE = ".ml-state";
61+
private static final String INITIAL_ML_STATE = ".ml-state-000001";
5962
private static final String ML_STATE_WRITE_ALIAS = ".ml-state-write";
6063

6164
private ThreadPool threadPool;
@@ -73,9 +76,9 @@ public void setUpMocks() {
7376
when(threadPool.getThreadContext()).thenReturn(new ThreadContext(Settings.EMPTY));
7477

7578
indicesAdminClient = mock(IndicesAdminClient.class);
76-
when(indicesAdminClient.prepareCreate(ML_STATE))
77-
.thenReturn(new CreateIndexRequestBuilder(client, CreateIndexAction.INSTANCE, ML_STATE));
78-
doAnswer(withResponse(new CreateIndexResponse(true, true, ML_STATE))).when(indicesAdminClient).create(any(), any());
79+
when(indicesAdminClient.prepareCreate(INITIAL_ML_STATE))
80+
.thenReturn(new CreateIndexRequestBuilder(client, CreateIndexAction.INSTANCE, INITIAL_ML_STATE));
81+
doAnswer(withResponse(new CreateIndexResponse(true, true, INITIAL_ML_STATE))).when(indicesAdminClient).create(any(), any());
7982
when(indicesAdminClient.prepareAliases()).thenReturn(new IndicesAliasesRequestBuilder(client, IndicesAliasesAction.INSTANCE));
8083
doAnswer(withResponse(new AcknowledgedResponse(true))).when(indicesAdminClient).aliases(any(), any());
8184

@@ -102,12 +105,12 @@ public void testCreateStateIndexAndAliasIfNecessary_CleanState() {
102105
AnomalyDetectorsIndex.createStateIndexAndAliasIfNecessary(client, clusterState, finalListener);
103106

104107
InOrder inOrder = inOrder(indicesAdminClient, finalListener);
105-
inOrder.verify(indicesAdminClient).prepareCreate(ML_STATE);
108+
inOrder.verify(indicesAdminClient).prepareCreate(INITIAL_ML_STATE);
106109
inOrder.verify(indicesAdminClient).create(createRequestCaptor.capture(), any());
107110
inOrder.verify(finalListener).onResponse(true);
108111

109112
CreateIndexRequest createRequest = createRequestCaptor.getValue();
110-
assertThat(createRequest.index(), equalTo(ML_STATE));
113+
assertThat(createRequest.index(), equalTo(INITIAL_ML_STATE));
111114
assertThat(createRequest.aliases(), equalTo(Collections.singleton(new Alias(ML_STATE_WRITE_ALIAS))));
112115
}
113116

@@ -118,8 +121,12 @@ private void assertNoClientInteractionsWhenWriteAliasAlreadyExists(String indexN
118121
verify(finalListener).onResponse(false);
119122
}
120123

124+
public void testCreateStateIndexAndAliasIfNecessary_WriteAliasAlreadyExistsAndPointsAtLegacyStateIndex() {
125+
assertNoClientInteractionsWhenWriteAliasAlreadyExists(LEGACY_ML_STATE);
126+
}
127+
121128
public void testCreateStateIndexAndAliasIfNecessary_WriteAliasAlreadyExistsAndPointsAtInitialStateIndex() {
122-
assertNoClientInteractionsWhenWriteAliasAlreadyExists(".ml-state-000001");
129+
assertNoClientInteractionsWhenWriteAliasAlreadyExists(INITIAL_ML_STATE);
123130
}
124131

125132
public void testCreateStateIndexAndAliasIfNecessary_WriteAliasAlreadyExistsAndPointsAtSubsequentStateIndex() {
@@ -147,9 +154,14 @@ private void assertMlStateWriteAliasAddedToMostRecentMlStateIndex(List<String> e
147154
contains(AliasActions.add().alias(ML_STATE_WRITE_ALIAS).index(expectedWriteIndexName)));
148155
}
149156

157+
public void testCreateStateIndexAndAliasIfNecessary_WriteAliasDoesNotExistButLegacyStateIndexExists() {
158+
assertMlStateWriteAliasAddedToMostRecentMlStateIndex(
159+
Arrays.asList(LEGACY_ML_STATE), LEGACY_ML_STATE);
160+
}
161+
150162
public void testCreateStateIndexAndAliasIfNecessary_WriteAliasDoesNotExistButInitialStateIndexExists() {
151163
assertMlStateWriteAliasAddedToMostRecentMlStateIndex(
152-
Arrays.asList(".ml-state-000001"), ".ml-state-000001");
164+
Arrays.asList(INITIAL_ML_STATE), INITIAL_ML_STATE);
153165
}
154166

155167
public void testCreateStateIndexAndAliasIfNecessary_WriteAliasDoesNotExistButSubsequentStateIndicesExist() {
@@ -159,7 +171,32 @@ public void testCreateStateIndexAndAliasIfNecessary_WriteAliasDoesNotExistButSub
159171

160172
public void testCreateStateIndexAndAliasIfNecessary_WriteAliasDoesNotExistButBothLegacyAndNewStateIndicesDoExist() {
161173
assertMlStateWriteAliasAddedToMostRecentMlStateIndex(
162-
Arrays.asList(ML_STATE, ".ml-state-000003", ".ml-state-000040", ".ml-state-000500"), ".ml-state-000500");
174+
Arrays.asList(LEGACY_ML_STATE, ".ml-state-000003", ".ml-state-000040", ".ml-state-000500"), ".ml-state-000500");
175+
}
176+
177+
public void testStateIndexNameComparator() {
178+
Comparator<String> comparator = AnomalyDetectorsIndex.STATE_INDEX_NAME_COMPARATOR;
179+
assertThat(
180+
Stream.of(".ml-state-000001").max(comparator).get(),
181+
equalTo(".ml-state-000001"));
182+
assertThat(
183+
Stream.of(".ml-state-000002", ".ml-state-000001").max(comparator).get(),
184+
equalTo(".ml-state-000002"));
185+
assertThat(
186+
Stream.of(".ml-state-000003", ".ml-state-000040", ".ml-state-000500").max(comparator).get(),
187+
equalTo(".ml-state-000500"));
188+
assertThat(
189+
Stream.of(".ml-state-000042", ".ml-state-000049", ".ml-state-000038").max(comparator).get(),
190+
equalTo(".ml-state-000049"));
191+
assertThat(
192+
Stream.of(".ml-state", ".ml-state-000003", ".ml-state-000040", ".ml-state-000500").max(comparator).get(),
193+
equalTo(".ml-state-000500"));
194+
assertThat(
195+
Stream.of(".reindexed-6-ml-state", ".ml-state-000042").max(comparator).get(),
196+
equalTo(".ml-state-000042"));
197+
assertThat(
198+
Stream.of(".a-000002", ".b-000001").max(comparator).get(),
199+
equalTo(".a-000002"));
163200
}
164201

165202
@SuppressWarnings("unchecked")

‎x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ClassificationIT.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -480,7 +480,7 @@ public void testDeleteExpiredData_RemovesUnusedState() throws Exception {
480480
// Now calling the _delete_expired_data API should remove unused state
481481
assertThat(deleteExpiredData().isDeleted(), is(true));
482482

483-
SearchResponse stateIndexSearchResponse = client().prepareSearch(".ml-state").execute().actionGet();
483+
SearchResponse stateIndexSearchResponse = client().prepareSearch(".ml-state*").execute().actionGet();
484484
assertThat(stateIndexSearchResponse.getHits().getTotalHits().value, equalTo(0L));
485485
}
486486

‎x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RegressionIT.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -329,7 +329,7 @@ public void testDeleteExpiredData_RemovesUnusedState() throws Exception {
329329
// Now calling the _delete_expired_data API should remove unused state
330330
assertThat(deleteExpiredData().isDeleted(), is(true));
331331

332-
SearchResponse stateIndexSearchResponse = client().prepareSearch(".ml-state").execute().actionGet();
332+
SearchResponse stateIndexSearchResponse = client().prepareSearch(".ml-state*").execute().actionGet();
333333
assertThat(stateIndexSearchResponse.getHits().getTotalHits().value, equalTo(0L));
334334
}
335335

‎x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RevertModelSnapshotIT.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -156,7 +156,7 @@ record = new HashMap<>();
156156
}
157157

158158
private Quantiles getQuantiles(String jobId) {
159-
SearchResponse response = client().prepareSearch(".ml-state")
159+
SearchResponse response = client().prepareSearch(".ml-state*")
160160
.setQuery(QueryBuilders.idsQuery().addIds(Quantiles.documentId(jobId)))
161161
.setSize(1)
162162
.get();

‎x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsActionTests.java

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@
88
import org.elasticsearch.Version;
99
import org.elasticsearch.cluster.ClusterName;
1010
import org.elasticsearch.cluster.ClusterState;
11-
import org.elasticsearch.cluster.metadata.AliasMetaData;
1211
import org.elasticsearch.cluster.metadata.IndexMetaData;
1312
import org.elasticsearch.cluster.metadata.MetaData;
1413
import org.elasticsearch.cluster.routing.IndexRoutingTable;
@@ -22,7 +21,6 @@
2221
import org.elasticsearch.index.shard.ShardId;
2322
import org.elasticsearch.test.ESTestCase;
2423
import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex;
25-
import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndexFields;
2624

2725
import java.util.List;
2826

@@ -45,9 +43,6 @@ public void testVerifyIndicesPrimaryShardsAreActive() {
4543
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
4644
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
4745
);
48-
if (indexName.equals(AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX)) {
49-
indexMetaData.putAlias(new AliasMetaData.Builder(AnomalyDetectorsIndex.jobStateIndexWriteAlias()));
50-
}
5146
metaData.put(indexMetaData);
5247
Index index = new Index(indexName, "_uuid");
5348
ShardId shardId = new ShardId(index, 0);

‎x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -960,7 +960,7 @@ setup:
960960
headers:
961961
Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
962962
index:
963-
index: .ml-state
963+
index: .ml-state-000001
964964
id: "delete_foo_regression_state#1"
965965
body: >
966966
{
@@ -970,7 +970,7 @@ setup:
970970
headers:
971971
Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
972972
index:
973-
index: .ml-state
973+
index: .ml-state-000001
974974
id: "data_frame_analytics-delete_foo-progress"
975975
body: >
976976
{
@@ -980,11 +980,11 @@ setup:
980980
headers:
981981
Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
982982
indices.refresh:
983-
index: .ml-state
983+
index: .ml-state*
984984

985985
- do:
986986
search:
987-
index: .ml-state
987+
index: .ml-state*
988988
body:
989989
size: 0
990990
query:

‎x-pack/plugin/src/test/resources/rest-api-spec/test/ml/delete_model_snapshot.yml

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ setup:
5555
headers:
5656
Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
5757
index:
58-
index: .ml-state
58+
index: .ml-state-000001
5959
id: "delete-model-snapshot_model_state_inactive-snapshot#1"
6060
body: >
6161
{
@@ -66,7 +66,7 @@ setup:
6666
headers:
6767
Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
6868
index:
69-
index: .ml-state
69+
index: .ml-state-000001
7070
id: "delete-model-snapshot_model_state_inactive-snapshot#2"
7171
body: >
7272
{
@@ -118,7 +118,7 @@ setup:
118118
headers:
119119
Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
120120
indices.refresh:
121-
index: .ml-state
121+
index: .ml-state*
122122

123123
- do:
124124
headers:
@@ -159,7 +159,7 @@ setup:
159159
headers:
160160
Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
161161
count:
162-
index: .ml-state
162+
index: .ml-state*
163163

164164
- match: { count: 3 }
165165

@@ -179,7 +179,7 @@ setup:
179179
headers:
180180
Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
181181
indices.refresh:
182-
index: .ml-state
182+
index: .ml-state*
183183

184184
- do:
185185
ml.get_model_snapshots:
@@ -191,7 +191,7 @@ setup:
191191
headers:
192192
Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
193193
count:
194-
index: .ml-state
194+
index: .ml-state*
195195

196196
- match: { count: 1 }
197197

‎x-pack/plugin/src/test/resources/rest-api-spec/test/ml/get_model_snapshots.yml

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ setup:
3737
Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
3838
Content-Type: application/json
3939
index:
40-
index: .ml-state
40+
index: .ml-state-000001
4141
id: "get-model-snapshots_model_state_1#1"
4242
body: >
4343
{
@@ -62,7 +62,7 @@ setup:
6262
headers:
6363
Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
6464
index:
65-
index: .ml-state
65+
index: .ml-state-000001
6666
id: "get-model-snapshots_model_state_2#1"
6767
body: >
6868
{
@@ -72,7 +72,7 @@ setup:
7272
headers:
7373
Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
7474
index:
75-
index: .ml-state
75+
index: .ml-state-000001
7676
id: "get-model-snapshots_model_state_2#2"
7777
body: >
7878
{
@@ -82,7 +82,7 @@ setup:
8282
headers:
8383
Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
8484
indices.refresh:
85-
index: [.ml-anomalies-get-model-snapshots,.ml-state]
85+
index: [.ml-anomalies-get-model-snapshots,.ml-state*]
8686

8787
---
8888
"Test get model snapshots API with no params":

0 commit comments

Comments
 (0)