Skip to content

Commit ec13c09

Browse files
przemekwitekprzemekwitek
authored
Make ML index aliases hidden (#53160) (#53710)
1 parent 85cc7a0 commit ec13c09

File tree

12 files changed

+87
-37
lines changed

12 files changed

+87
-37
lines changed

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java

+5-3
Original file line numberDiff line numberDiff line change
@@ -46,9 +46,11 @@ public static void createAnnotationsIndexIfNecessary(Settings settings, Client c
4646
final ActionListener<Boolean> finalListener) {
4747

4848
final ActionListener<Boolean> createAliasListener = ActionListener.wrap(success -> {
49-
final IndicesAliasesRequest request = client.admin().indices().prepareAliases()
50-
.addAlias(INDEX_NAME, READ_ALIAS_NAME)
51-
.addAlias(INDEX_NAME, WRITE_ALIAS_NAME).request();
49+
final IndicesAliasesRequest request =
50+
client.admin().indices().prepareAliases()
51+
.addAliasAction(IndicesAliasesRequest.AliasActions.add().index(INDEX_NAME).alias(READ_ALIAS_NAME).isHidden(true))
52+
.addAliasAction(IndicesAliasesRequest.AliasActions.add().index(INDEX_NAME).alias(WRITE_ALIAS_NAME).isHidden(true))
53+
.request();
5254
executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, request,
5355
ActionListener.<AcknowledgedResponse>wrap(r -> finalListener.onResponse(r.isAcknowledged()), finalListener::onFailure),
5456
client.admin().indices()::aliases);

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,7 @@ private static void createFirstConcreteIndex(Client client,
135135
.indices()
136136
.prepareCreate(index);
137137
if (addAlias) {
138-
requestBuilder.addAlias(new Alias(alias));
138+
requestBuilder.addAlias(new Alias(alias).isHidden(true));
139139
}
140140
CreateIndexRequest request = requestBuilder.request();
141141

@@ -165,7 +165,7 @@ private static void updateWriteAlias(Client client,
165165
IndicesAliasesRequestBuilder requestBuilder = client.admin()
166166
.indices()
167167
.prepareAliases()
168-
.addAlias(newIndex, alias);
168+
.addAliasAction(IndicesAliasesRequest.AliasActions.add().index(newIndex).alias(alias).isHidden(true));
169169
if (currentIndex != null) {
170170
requestBuilder.removeAlias(currentIndex, alias);
171171
}

‎x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAliasTests.java

+4-4
Original file line numberDiff line numberDiff line change
@@ -114,7 +114,7 @@ public void testCreateStateIndexAndAliasIfNecessary_CleanState() {
114114

115115
CreateIndexRequest createRequest = createRequestCaptor.getValue();
116116
assertThat(createRequest.index(), equalTo(FIRST_CONCRETE_INDEX));
117-
assertThat(createRequest.aliases(), equalTo(Collections.singleton(new Alias(TEST_INDEX_ALIAS))));
117+
assertThat(createRequest.aliases(), equalTo(Collections.singleton(new Alias(TEST_INDEX_ALIAS).isHidden(true))));
118118
}
119119

120120
private void assertNoClientInteractionsWhenWriteAliasAlreadyExists(String indexName) {
@@ -157,7 +157,7 @@ public void testCreateStateIndexAndAliasIfNecessary_WriteAliasAlreadyExistsAndPo
157157
assertThat(
158158
indicesAliasesRequest.getAliasActions(),
159159
contains(
160-
AliasActions.add().alias(TEST_INDEX_ALIAS).index(FIRST_CONCRETE_INDEX),
160+
AliasActions.add().alias(TEST_INDEX_ALIAS).index(FIRST_CONCRETE_INDEX).isHidden(true),
161161
AliasActions.remove().alias(TEST_INDEX_ALIAS).index(LEGACY_INDEX_WITHOUT_SUFFIX)));
162162
}
163163

@@ -175,7 +175,7 @@ private void assertMlStateWriteAliasAddedToMostRecentMlStateIndex(List<String> e
175175
IndicesAliasesRequest indicesAliasesRequest = aliasesRequestCaptor.getValue();
176176
assertThat(
177177
indicesAliasesRequest.getAliasActions(),
178-
contains(AliasActions.add().alias(TEST_INDEX_ALIAS).index(expectedWriteIndexName)));
178+
contains(AliasActions.add().alias(TEST_INDEX_ALIAS).index(expectedWriteIndexName).isHidden(true)));
179179
}
180180

181181
public void testCreateStateIndexAndAliasIfNecessary_WriteAliasDoesNotExistButInitialStateIndexExists() {
@@ -205,7 +205,7 @@ public void testCreateStateIndexAndAliasIfNecessary_WriteAliasDoesNotExistButLeg
205205

206206
CreateIndexRequest createRequest = createRequestCaptor.getValue();
207207
assertThat(createRequest.index(), equalTo(FIRST_CONCRETE_INDEX));
208-
assertThat(createRequest.aliases(), equalTo(Collections.singleton(new Alias(TEST_INDEX_ALIAS))));
208+
assertThat(createRequest.aliases(), equalTo(Collections.singleton(new Alias(TEST_INDEX_ALIAS).isHidden(true))));
209209
}
210210

211211
public void testIndexNameComparator() {

‎x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java

+3
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
import org.elasticsearch.action.bulk.BulkResponse;
1111
import org.elasticsearch.action.index.IndexRequest;
1212
import org.elasticsearch.action.search.SearchResponse;
13+
import org.elasticsearch.action.support.IndicesOptions;
1314
import org.elasticsearch.action.support.WriteRequest;
1415
import org.elasticsearch.action.update.UpdateAction;
1516
import org.elasticsearch.action.update.UpdateRequest;
@@ -185,6 +186,7 @@ public void testDeleteExpiredData() throws Exception {
185186
retainAllSnapshots("snapshots-retention-with-retain");
186187

187188
long totalModelSizeStatsBeforeDelete = client().prepareSearch("*")
189+
.setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN)
188190
.setQuery(QueryBuilders.termQuery("result_type", "model_size_stats"))
189191
.get().getHits().getTotalHits().value;
190192
long totalNotificationsCountBeforeDelete =
@@ -233,6 +235,7 @@ public void testDeleteExpiredData() throws Exception {
233235
assertThat(getModelSnapshots("results-and-snapshots-retention").size(), equalTo(1));
234236

235237
long totalModelSizeStatsAfterDelete = client().prepareSearch("*")
238+
.setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN)
236239
.setQuery(QueryBuilders.termQuery("result_type", "model_size_stats"))
237240
.get().getHits().getTotalHits().value;
238241
long totalNotificationsCountAfterDelete =

‎x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java

+20-15
Original file line numberDiff line numberDiff line change
@@ -194,15 +194,17 @@ public void testCreateJobsWithIndexNameOption() throws Exception {
194194
// appear immediately so wait here.
195195
assertBusy(() -> {
196196
try {
197-
String aliasesResponse = EntityUtils.toString(client().performRequest(new Request("GET", "/_aliases")).getEntity());
198-
assertThat(aliasesResponse,
199-
containsString("\"" + AnomalyDetectorsIndex.jobResultsAliasedName("custom-" + indexName) + "\":{\"aliases\":{"));
197+
String aliasesResponse = getAliases();
198+
assertThat(aliasesResponse, containsString("\"" + AnomalyDetectorsIndex.jobResultsAliasedName("custom-" + indexName)
199+
+ "\":{\"aliases\":{"));
200200
assertThat(aliasesResponse, containsString("\"" + AnomalyDetectorsIndex.jobResultsAliasedName(jobId1)
201-
+ "\":{\"filter\":{\"term\":{\"job_id\":{\"value\":\"" + jobId1 + "\",\"boost\":1.0}}}}"));
202-
assertThat(aliasesResponse, containsString("\"" + AnomalyDetectorsIndex.resultsWriteAlias(jobId1) + "\":{}"));
201+
+ "\":{\"filter\":{\"term\":{\"job_id\":{\"value\":\"" + jobId1 + "\",\"boost\":1.0}}},\"is_hidden\":true}"));
202+
assertThat(aliasesResponse, containsString("\"" + AnomalyDetectorsIndex.resultsWriteAlias(jobId1)
203+
+ "\":{\"is_hidden\":true}"));
203204
assertThat(aliasesResponse, containsString("\"" + AnomalyDetectorsIndex.jobResultsAliasedName(jobId2)
204-
+ "\":{\"filter\":{\"term\":{\"job_id\":{\"value\":\"" + jobId2 + "\",\"boost\":1.0}}}}"));
205-
assertThat(aliasesResponse, containsString("\"" + AnomalyDetectorsIndex.resultsWriteAlias(jobId2) + "\":{}"));
205+
+ "\":{\"filter\":{\"term\":{\"job_id\":{\"value\":\"" + jobId2 + "\",\"boost\":1.0}}},\"is_hidden\":true}"));
206+
assertThat(aliasesResponse, containsString("\"" + AnomalyDetectorsIndex.resultsWriteAlias(jobId2)
207+
+ "\":{\"is_hidden\":true}"));
206208
} catch (ResponseException e) {
207209
throw new AssertionError(e);
208210
}
@@ -270,7 +272,7 @@ public void testCreateJobsWithIndexNameOption() throws Exception {
270272
client().performRequest(new Request("DELETE", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId1));
271273

272274
// check that indices still exist, but no longer have job1 entries and aliases are gone
273-
responseAsString = EntityUtils.toString(client().performRequest(new Request("GET", "/_aliases")).getEntity());
275+
responseAsString = getAliases();
274276
assertThat(responseAsString, not(containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId1))));
275277
assertThat(responseAsString, containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId2))); //job2 still exists
276278

@@ -286,7 +288,7 @@ public void testCreateJobsWithIndexNameOption() throws Exception {
286288

287289
// Delete the second job and verify aliases are gone, and original concrete/custom index is gone
288290
client().performRequest(new Request("DELETE", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId2));
289-
responseAsString = EntityUtils.toString(client().performRequest(new Request("GET", "/_aliases")).getEntity());
291+
responseAsString = getAliases();
290292
assertThat(responseAsString, not(containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId2))));
291293

292294
refreshAllIndices();
@@ -626,6 +628,7 @@ public void testMultiIndexDelete() throws Exception {
626628
extraIndex1.setJsonEntity("{\n" +
627629
" \"aliases\" : {\n" +
628630
" \"" + AnomalyDetectorsIndex.jobResultsAliasedName(jobId)+ "\" : {\n" +
631+
" \"is_hidden\" : true,\n" +
629632
" \"filter\" : {\n" +
630633
" \"term\" : {\"" + Job.ID + "\" : \"" + jobId + "\" }\n" +
631634
" }\n" +
@@ -637,6 +640,7 @@ public void testMultiIndexDelete() throws Exception {
637640
extraIndex2.setJsonEntity("{\n" +
638641
" \"aliases\" : {\n" +
639642
" \"" + AnomalyDetectorsIndex.jobResultsAliasedName(jobId)+ "\" : {\n" +
643+
" \"is_hidden\" : true,\n" +
640644
" \"filter\" : {\n" +
641645
" \"term\" : {\"" + Job.ID + "\" : \"" + jobId + "\" }\n" +
642646
" }\n" +
@@ -784,6 +788,9 @@ public void testDelete_multipleRequest() throws Exception {
784788
assertNull(recreationException.get().getMessage(), recreationException.get());
785789
}
786790

791+
String expectedReadAliasString = "\"" + AnomalyDetectorsIndex.jobResultsAliasedName(jobId)
792+
+ "\":{\"filter\":{\"term\":{\"job_id\":{\"value\":\"" + jobId + "\",\"boost\":1.0}}},\"is_hidden\":true}";
793+
String expectedWriteAliasString = "\"" + AnomalyDetectorsIndex.resultsWriteAlias(jobId) + "\":{\"is_hidden\":true}";
787794
try {
788795
// The idea of the code above is that the deletion is sufficiently time-consuming that
789796
// all threads enter the deletion call before the first one exits it. Usually this happens,
@@ -796,9 +803,8 @@ public void testDelete_multipleRequest() throws Exception {
796803
// if there's been a race between deletion and recreation these are what will be missing.
797804
String aliases = getAliases();
798805

799-
assertThat(aliases, containsString("\"" + AnomalyDetectorsIndex.jobResultsAliasedName(jobId)
800-
+ "\":{\"filter\":{\"term\":{\"job_id\":{\"value\":\"" + jobId + "\",\"boost\":1.0}}}}"));
801-
assertThat(aliases, containsString("\"" + AnomalyDetectorsIndex.resultsWriteAlias(jobId) + "\":{}"));
806+
assertThat(aliases, containsString(expectedReadAliasString));
807+
assertThat(aliases, containsString(expectedWriteAliasString));
802808

803809

804810
} catch (ResponseException missingJobException) {
@@ -807,9 +813,8 @@ public void testDelete_multipleRequest() throws Exception {
807813

808814
// The job aliases should be deleted
809815
String aliases = getAliases();
810-
assertThat(aliases, not(containsString("\"" + AnomalyDetectorsIndex.jobResultsAliasedName(jobId)
811-
+ "\":{\"filter\":{\"term\":{\"job_id\":{\"value\":\"" + jobId + "\",\"boost\":1.0}}}}")));
812-
assertThat(aliases, not(containsString("\"" + AnomalyDetectorsIndex.resultsWriteAlias(jobId) + "\":{}")));
816+
assertThat(aliases, not(containsString(expectedReadAliasString)));
817+
assertThat(aliases, not(containsString(expectedWriteAliasString)));
813818
}
814819

815820
assertEquals(numThreads, recreationGuard.get());

‎x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteJobAction.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -494,7 +494,7 @@ private void deleteAliases(ParentTaskAssigningClient parentTaskClient, String jo
494494

495495
// first find the concrete indices associated with the aliases
496496
GetAliasesRequest aliasesRequest = new GetAliasesRequest().aliases(readAliasName, writeAliasName)
497-
.indicesOptions(IndicesOptions.lenientExpandOpen());
497+
.indicesOptions(IndicesOptions.lenientExpandOpenHidden());
498498
executeAsyncWithOrigin(parentTaskClient.threadPool().getThreadContext(), ML_ORIGIN, aliasesRequest,
499499
ActionListener.<GetAliasesResponse>wrap(
500500
getAliasesResponse -> {

‎x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java

+10-3
Original file line numberDiff line numberDiff line change
@@ -283,9 +283,16 @@ public void createJobResultIndex(Job job, ClusterState state, final ActionListen
283283
final String indexName = tempIndexName;
284284

285285
ActionListener<Boolean> indexAndMappingsListener = ActionListener.wrap(success -> {
286-
final IndicesAliasesRequest request = client.admin().indices().prepareAliases()
287-
.addAlias(indexName, readAliasName, QueryBuilders.termQuery(Job.ID.getPreferredName(), job.getId()))
288-
.addAlias(indexName, writeAliasName).request();
286+
final IndicesAliasesRequest request =
287+
client.admin().indices().prepareAliases()
288+
.addAliasAction(
289+
IndicesAliasesRequest.AliasActions.add()
290+
.index(indexName)
291+
.alias(readAliasName)
292+
.isHidden(true)
293+
.filter(QueryBuilders.termQuery(Job.ID.getPreferredName(), job.getId())))
294+
.addAliasAction(IndicesAliasesRequest.AliasActions.add().index(indexName).alias(writeAliasName).isHidden(true))
295+
.request();
289296
executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, request,
290297
ActionListener.<AcknowledgedResponse>wrap(r -> finalListener.onResponse(true), finalListener::onFailure),
291298
client.admin().indices()::aliases);

‎x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/AnnotationIndexIT.java

+5
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@
1818

1919
import java.util.List;
2020

21+
import static org.hamcrest.Matchers.is;
22+
2123
public class AnnotationIndexIT extends MlSingleNodeTestCase {
2224

2325
@Override
@@ -71,6 +73,9 @@ private int numberOfAnnotationsAliases() {
7173
.getAliases();
7274
if (aliases != null) {
7375
for (ObjectObjectCursor<String, List<AliasMetaData>> entry : aliases) {
76+
for (AliasMetaData aliasMetaData : entry.value) {
77+
assertThat("Annotations aliases should be hidden but are not: " + aliases, aliasMetaData.isHidden(), is(true));
78+
}
7479
count += entry.value.size();
7580
}
7681
}

‎x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobResultsProviderIT.java

+14-8
Original file line numberDiff line numberDiff line change
@@ -144,8 +144,10 @@ public void testPutJob_CreatesResultsIndex() {
144144
assertThat(mappingProperties.keySet(), hasItem("by_field_1"));
145145

146146
// Check aliases have been created
147-
assertThat(getAliases(sharedResultsIndex), containsInAnyOrder(AnomalyDetectorsIndex.jobResultsAliasedName(job1.getId()),
148-
AnomalyDetectorsIndex.resultsWriteAlias(job1.getId())));
147+
assertThat(getAliases(sharedResultsIndex), containsInAnyOrder(
148+
AnomalyDetectorsIndex.jobResultsAliasedName(job1.getId()),
149+
AnomalyDetectorsIndex.resultsWriteAlias(job1.getId())
150+
));
149151

150152
// Now let's create a second job to test things work when the index exists already
151153
assertThat(mappingProperties.keySet(), not(hasItem("by_field_2")));
@@ -189,8 +191,10 @@ public void testPutJob_WithCustomResultsIndex() {
189191
assertThat(mappingProperties.keySet(), hasItem("by_field"));
190192

191193
// Check aliases have been created
192-
assertThat(getAliases(customIndex), containsInAnyOrder(AnomalyDetectorsIndex.jobResultsAliasedName(job.getId()),
193-
AnomalyDetectorsIndex.resultsWriteAlias(job.getId())));
194+
assertThat(getAliases(customIndex), containsInAnyOrder(
195+
AnomalyDetectorsIndex.jobResultsAliasedName(job.getId()),
196+
AnomalyDetectorsIndex.resultsWriteAlias(job.getId())
197+
));
194198
}
195199

196200
@AwaitsFix(bugUrl ="https://github.com/elastic/elasticsearch/issues/40134")
@@ -370,12 +374,14 @@ private Map<String, Object> getIndexMappingProperties(String index) {
370374
}
371375

372376
private Set<String> getAliases(String index) {
373-
GetAliasesResponse getAliasesResponse = client().admin().indices().getAliases(
374-
new GetAliasesRequest().indices(index)).actionGet();
377+
GetAliasesResponse getAliasesResponse = client().admin().indices().getAliases(new GetAliasesRequest().indices(index)).actionGet();
375378
ImmutableOpenMap<String, List<AliasMetaData>> aliases = getAliasesResponse.getAliases();
376379
assertThat(aliases.containsKey(index), is(true));
377-
List<AliasMetaData> aliasMetaData = aliases.get(index);
378-
return aliasMetaData.stream().map(AliasMetaData::alias).collect(Collectors.toSet());
380+
List<AliasMetaData> aliasMetaDataList = aliases.get(index);
381+
for (AliasMetaData aliasMetaData : aliasMetaDataList) {
382+
assertThat("Anomalies aliases should be hidden but are not: " + aliases, aliasMetaData.isHidden(), is(true));
383+
}
384+
return aliasMetaDataList.stream().map(AliasMetaData::alias).collect(Collectors.toSet());
379385
}
380386

381387
private List<Calendar> getCalendars(String jobId) throws Exception {

‎x-pack/plugin/src/test/resources/rest-api-spec/test/ml/custom_all_field.yml

+1
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,7 @@ setup:
148148

149149
- do:
150150
search:
151+
expand_wildcards: all
151152
rest_total_hits_as_int: true
152153
body: { query: { bool: { must: [
153154
{ query_string: { query: "result_type:record"} },

‎x-pack/plugin/src/test/resources/rest-api-spec/test/ml/index_layout.yml

+21
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,13 @@ setup:
100100
index: ".ml-state-000001"
101101
- is_true: ''
102102

103+
- do:
104+
headers:
105+
Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
106+
indices.exists_alias:
107+
name: ".ml-state-write"
108+
- is_true: ''
109+
103110
- do:
104111
headers:
105112
Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
@@ -439,6 +446,13 @@ setup:
439446
index: ".ml-state-000001"
440447
- is_true: ''
441448

449+
- do:
450+
headers:
451+
Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
452+
indices.exists_alias:
453+
name: ".ml-state-write"
454+
- is_true: ''
455+
442456
- do:
443457
headers:
444458
Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
@@ -681,6 +695,13 @@ setup:
681695
index: ".ml-state-000001"
682696
- is_true: ''
683697

698+
- do:
699+
headers:
700+
Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
701+
indices.exists_alias:
702+
name: ".ml-state-write"
703+
- is_true: ''
704+
684705
- do:
685706
headers:
686707
Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser

‎x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@
7070
- do:
7171
indices.get_alias:
7272
name: ".ml-anomalies-.write-job-crud-test-apis"
73-
- match: { \.ml-anomalies-shared.aliases.\.ml-anomalies-\.write-job-crud-test-apis: {} }
73+
- match: { \.ml-anomalies-shared.aliases.\.ml-anomalies-\.write-job-crud-test-apis: { is_hidden: true } }
7474

7575
- do:
7676
ml.delete_job:

0 commit comments

Comments
 (0)