Skip to content

Commit 833e0f8

Browse files
s1monws1monw
authored
Prevent throttled indices to be searched through wildcards by default (#34354)
Today if a wildcard, date-math expression or alias expands/resolves to an index that is search-throttled we still search it. This is likely not the desired behavior since it can unexpectedly slow down searches significantly. This change adds a new indices option that allows `search`, `count` and `msearch` to ignore throttled indices by default. Users can force expansion to throttled indices by using `ignore_throttled=true` on the rest request to expand also to throttled indices. Relates to #34352
1 parent 46c238d commit 833e0f8

File tree

28 files changed

+268
-92
lines changed

28 files changed

+268
-92
lines changed

‎client/rest-high-level/src/test/java/org/elasticsearch/client/RankEvalIT.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,7 @@ public void testRankEvalRequest() throws IOException {
108108

109109
// now try this when test2 is closed
110110
client().performRequest(new Request("POST", "index2/_close"));
111-
rankEvalRequest.indicesOptions(IndicesOptions.fromParameters(null, "true", null, SearchRequest.DEFAULT_INDICES_OPTIONS));
111+
rankEvalRequest.indicesOptions(IndicesOptions.fromParameters(null, "true", null, "false", SearchRequest.DEFAULT_INDICES_OPTIONS));
112112
response = execute(rankEvalRequest, highLevelClient()::rankEval, highLevelClient()::rankEvalAsync);
113113
}
114114

‎client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1055,7 +1055,8 @@ public void testMultiSearch() throws IOException {
10551055
IndicesOptions msearchDefault = new MultiSearchRequest().indicesOptions();
10561056
searchRequest.indicesOptions(IndicesOptions.fromOptions(randomlyGenerated.ignoreUnavailable(),
10571057
randomlyGenerated.allowNoIndices(), randomlyGenerated.expandWildcardsOpen(), randomlyGenerated.expandWildcardsClosed(),
1058-
msearchDefault.allowAliasesToMultipleIndices(), msearchDefault.forbidClosedIndices(), msearchDefault.ignoreAliases()));
1058+
msearchDefault.allowAliasesToMultipleIndices(), msearchDefault.forbidClosedIndices(), msearchDefault.ignoreAliases(),
1059+
msearchDefault.ignoreThrottled()));
10591060
multiSearchRequest.add(searchRequest);
10601061
}
10611062

‎client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/ExplainLifecycleRequestTests.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ private ExplainLifecycleRequest createTestInstance() {
3838
}
3939
if (randomBoolean()) {
4040
IndicesOptions indicesOptions = IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(),
41-
randomBoolean(), randomBoolean(), randomBoolean());
41+
randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean());
4242
request.indicesOptions(indicesOptions);
4343
}
4444
return request;
@@ -54,7 +54,7 @@ private ExplainLifecycleRequest mutateInstance(ExplainLifecycleRequest instance)
5454
break;
5555
case 1:
5656
indicesOptions = randomValueOtherThan(indicesOptions, () -> IndicesOptions.fromOptions(randomBoolean(), randomBoolean(),
57-
randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean()));
57+
randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean()));
5858
break;
5959
default:
6060
throw new AssertionError("Illegal randomisation branch");

‎client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/RemoveIndexLifecyclePolicyRequestTests.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ protected RemoveIndexLifecyclePolicyRequest createInstance() {
4646
if (randomBoolean()) {
4747
return new RemoveIndexLifecyclePolicyRequest(Arrays.asList(generateRandomStringArray(20, 20, false)),
4848
IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(),
49-
randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean()));
49+
randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean()));
5050
} else {
5151
return new RemoveIndexLifecyclePolicyRequest(Arrays.asList(generateRandomStringArray(20, 20, false)));
5252
}
@@ -57,14 +57,14 @@ private RemoveIndexLifecyclePolicyRequest copyInstance(RemoveIndexLifecyclePolic
5757
req.indicesOptions().ignoreUnavailable(), req.indicesOptions().allowNoIndices(),
5858
req.indicesOptions().expandWildcardsOpen(), req.indicesOptions().expandWildcardsClosed(),
5959
req.indicesOptions().allowAliasesToMultipleIndices(), req.indicesOptions().forbidClosedIndices(),
60-
req.indicesOptions().ignoreAliases()));
60+
req.indicesOptions().ignoreAliases(), req.indicesOptions().ignoreThrottled()));
6161
}
6262

6363
private RemoveIndexLifecyclePolicyRequest mutateInstance(RemoveIndexLifecyclePolicyRequest req) {
6464
if (randomBoolean()) {
6565
return new RemoveIndexLifecyclePolicyRequest(req.indices(),
6666
randomValueOtherThan(req.indicesOptions(), () -> IndicesOptions.fromOptions(randomBoolean(), randomBoolean(),
67-
randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean())));
67+
randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean())));
6868
} else {
6969
return new RemoveIndexLifecyclePolicyRequest(
7070
randomValueOtherThan(req.indices(), () -> Arrays.asList(generateRandomStringArray(20, 20, false))),

‎modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequest.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ public class MultiSearchTemplateRequest extends ActionRequest implements Composi
4545
private int maxConcurrentSearchRequests = 0;
4646
private List<SearchTemplateRequest> requests = new ArrayList<>();
4747

48-
private IndicesOptions indicesOptions = IndicesOptions.strictExpandOpenAndForbidClosed();
48+
private IndicesOptions indicesOptions = IndicesOptions.strictExpandOpenAndForbidClosedIgnoreThrottled();
4949

5050
/**
5151
* Add a search template request to execute. Note, the order is important, the search response will be returned in the

‎modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -286,45 +286,45 @@ public void testIndicesOptions() {
286286
// test that ignore_unavailable=true works but returns one result less
287287
assertTrue(client().admin().indices().prepareClose("test2").get().isAcknowledged());
288288

289-
request.indicesOptions(IndicesOptions.fromParameters(null, "true", null, SearchRequest.DEFAULT_INDICES_OPTIONS));
289+
request.indicesOptions(IndicesOptions.fromParameters(null, "true", null, "false", SearchRequest.DEFAULT_INDICES_OPTIONS));
290290
response = client().execute(RankEvalAction.INSTANCE, request).actionGet();
291291
details = (PrecisionAtK.Detail) response.getPartialResults().get("amsterdam_query").getMetricDetails();
292292
assertEquals(6, details.getRetrieved());
293293
assertEquals(5, details.getRelevantRetrieved());
294294

295295
// test that ignore_unavailable=false or default settings throw an IndexClosedException
296296
assertTrue(client().admin().indices().prepareClose("test2").get().isAcknowledged());
297-
request.indicesOptions(IndicesOptions.fromParameters(null, "false", null, SearchRequest.DEFAULT_INDICES_OPTIONS));
297+
request.indicesOptions(IndicesOptions.fromParameters(null, "false", null, "false", SearchRequest.DEFAULT_INDICES_OPTIONS));
298298
response = client().execute(RankEvalAction.INSTANCE, request).actionGet();
299299
assertEquals(1, response.getFailures().size());
300300
assertThat(response.getFailures().get("amsterdam_query"), instanceOf(IndexClosedException.class));
301301

302302
// test expand_wildcards
303303
request = new RankEvalRequest(task, new String[] { "tes*" });
304-
request.indicesOptions(IndicesOptions.fromParameters("none", null, null, SearchRequest.DEFAULT_INDICES_OPTIONS));
304+
request.indicesOptions(IndicesOptions.fromParameters("none", null, null, "false", SearchRequest.DEFAULT_INDICES_OPTIONS));
305305
response = client().execute(RankEvalAction.INSTANCE, request).actionGet();
306306
details = (PrecisionAtK.Detail) response.getPartialResults().get("amsterdam_query").getMetricDetails();
307307
assertEquals(0, details.getRetrieved());
308308

309-
request.indicesOptions(IndicesOptions.fromParameters("open", null, null, SearchRequest.DEFAULT_INDICES_OPTIONS));
309+
request.indicesOptions(IndicesOptions.fromParameters("open", null, null, "false", SearchRequest.DEFAULT_INDICES_OPTIONS));
310310
response = client().execute(RankEvalAction.INSTANCE, request).actionGet();
311311
details = (PrecisionAtK.Detail) response.getPartialResults().get("amsterdam_query").getMetricDetails();
312312
assertEquals(6, details.getRetrieved());
313313
assertEquals(5, details.getRelevantRetrieved());
314314

315-
request.indicesOptions(IndicesOptions.fromParameters("closed", null, null, SearchRequest.DEFAULT_INDICES_OPTIONS));
315+
request.indicesOptions(IndicesOptions.fromParameters("closed", null, null, "false", SearchRequest.DEFAULT_INDICES_OPTIONS));
316316
response = client().execute(RankEvalAction.INSTANCE, request).actionGet();
317317
assertEquals(1, response.getFailures().size());
318318
assertThat(response.getFailures().get("amsterdam_query"), instanceOf(IndexClosedException.class));
319319

320320
// test allow_no_indices
321321
request = new RankEvalRequest(task, new String[] { "bad*" });
322-
request.indicesOptions(IndicesOptions.fromParameters(null, null, "true", SearchRequest.DEFAULT_INDICES_OPTIONS));
322+
request.indicesOptions(IndicesOptions.fromParameters(null, null, "true", "false", SearchRequest.DEFAULT_INDICES_OPTIONS));
323323
response = client().execute(RankEvalAction.INSTANCE, request).actionGet();
324324
details = (PrecisionAtK.Detail) response.getPartialResults().get("amsterdam_query").getMetricDetails();
325325
assertEquals(0, details.getRetrieved());
326326

327-
request.indicesOptions(IndicesOptions.fromParameters(null, null, "false", SearchRequest.DEFAULT_INDICES_OPTIONS));
327+
request.indicesOptions(IndicesOptions.fromParameters(null, null, "false", "false", SearchRequest.DEFAULT_INDICES_OPTIONS));
328328
response = client().execute(RankEvalAction.INSTANCE, request).actionGet();
329329
assertEquals(1, response.getFailures().size());
330330
assertThat(response.getFailures().get("amsterdam_query"), instanceOf(IndexNotFoundException.class));

‎modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,8 @@ protected RankEvalRequest createTestInstance() {
5959
}
6060
RankEvalRequest rankEvalRequest = new RankEvalRequest(RankEvalSpecTests.createTestItem(), indices);
6161
IndicesOptions indicesOptions = IndicesOptions.fromOptions(
62-
randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean());
62+
randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(),
63+
randomBoolean());
6364
rankEvalRequest.indicesOptions(indicesOptions);
6465
return rankEvalRequest;
6566
}

‎rest-api-spec/src/main/resources/rest-api-spec/api/count.json

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,10 @@
2020
"type" : "boolean",
2121
"description" : "Whether specified concrete indices should be ignored when unavailable (missing or closed)"
2222
},
23+
"ignore_throttled": {
24+
"type" : "boolean",
25+
"description" : "Whether specified concrete, expanded or aliased indices should be ignored when throttled"
26+
},
2327
"allow_no_indices": {
2428
"type" : "boolean",
2529
"description" : "Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes `_all` string or when no indices have been specified)"

‎rest-api-spec/src/main/resources/rest-api-spec/api/search.json

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,10 @@
5858
"type" : "boolean",
5959
"description" : "Whether specified concrete indices should be ignored when unavailable (missing or closed)"
6060
},
61+
"ignore_throttled": {
62+
"type" : "boolean",
63+
"description" : "Whether specified concrete, expanded or aliased indices should be ignored when throttled"
64+
},
6165
"allow_no_indices": {
6266
"type" : "boolean",
6367
"description" : "Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes `_all` string or when no indices have been specified)"

‎rest-api-spec/src/main/resources/rest-api-spec/api/search_template.json

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,10 @@
2020
"type" : "boolean",
2121
"description" : "Whether specified concrete indices should be ignored when unavailable (missing or closed)"
2222
},
23+
"ignore_throttled": {
24+
"type" : "boolean",
25+
"description" : "Whether specified concrete, expanded or aliased indices should be ignored when throttled"
26+
},
2327
"allow_no_indices": {
2428
"type" : "boolean",
2529
"description" : "Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes `_all` string or when no indices have been specified)"

‎server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ public class IndicesAliasesRequest extends AcknowledgedRequest<IndicesAliasesReq
6767
// indices options that require every specified index to exist, expand wildcards only to open
6868
// indices, don't allow that no indices are resolved from wildcard expressions and resolve the
6969
// expressions only against indices
70-
private static final IndicesOptions INDICES_OPTIONS = IndicesOptions.fromOptions(false, false, true, false, true, false, true);
70+
private static final IndicesOptions INDICES_OPTIONS = IndicesOptions.fromOptions(false, false, true, false, true, false, true, false);
7171

7272
public IndicesAliasesRequest() {
7373
}

‎server/src/main/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexRequest.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ public class DeleteIndexRequest extends AcknowledgedRequest<DeleteIndexRequest>
3838

3939
private String[] indices;
4040
// Delete index should work by default on both open and closed indices.
41-
private IndicesOptions indicesOptions = IndicesOptions.fromOptions(false, true, true, true, false, false, true);
41+
private IndicesOptions indicesOptions = IndicesOptions.fromOptions(false, true, true, true, false, false, true, false);
4242

4343
public DeleteIndexRequest() {
4444
}

‎server/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ public class MultiSearchRequest extends ActionRequest implements CompositeIndice
5858
private int maxConcurrentSearchRequests = 0;
5959
private List<SearchRequest> requests = new ArrayList<>();
6060

61-
private IndicesOptions indicesOptions = IndicesOptions.strictExpandOpenAndForbidClosed();
61+
private IndicesOptions indicesOptions = IndicesOptions.strictExpandOpenAndForbidClosedIgnoreThrottled();
6262

6363
/**
6464
* Add a search request to execute. Note, the order is important, the search response will be returned in the
@@ -287,7 +287,7 @@ public static byte[] writeMultiLineFormat(MultiSearchRequest multiSearchRequest,
287287
}
288288
return output.toByteArray();
289289
}
290-
290+
291291
public static void writeSearchRequestParams(SearchRequest request, XContentBuilder xContentBuilder) throws IOException {
292292
xContentBuilder.startObject();
293293
if (request.indices() != null) {

‎server/src/main/java/org/elasticsearch/action/search/MultiSearchRequestBuilder.java

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
*/
2929
public class MultiSearchRequestBuilder extends ActionRequestBuilder<MultiSearchRequest, MultiSearchResponse> {
3030

31+
3132
public MultiSearchRequestBuilder(ElasticsearchClient client, MultiSearchAction action) {
3233
super(client, action, new MultiSearchRequest());
3334
}
@@ -40,7 +41,8 @@ public MultiSearchRequestBuilder(ElasticsearchClient client, MultiSearchAction a
4041
* will not be used (if set).
4142
*/
4243
public MultiSearchRequestBuilder add(SearchRequest request) {
43-
if (request.indicesOptions() == IndicesOptions.strictExpandOpenAndForbidClosed() && request().indicesOptions() != IndicesOptions.strictExpandOpenAndForbidClosed()) {
44+
if (request.indicesOptions() == IndicesOptions.strictExpandOpenAndForbidClosed()
45+
&& request().indicesOptions() != IndicesOptions.strictExpandOpenAndForbidClosed()) {
4446
request.indicesOptions(request().indicesOptions());
4547
}
4648

@@ -53,7 +55,8 @@ public MultiSearchRequestBuilder add(SearchRequest request) {
5355
* same order as the search requests.
5456
*/
5557
public MultiSearchRequestBuilder add(SearchRequestBuilder request) {
56-
if (request.request().indicesOptions() == IndicesOptions.strictExpandOpenAndForbidClosed() && request().indicesOptions() != IndicesOptions.strictExpandOpenAndForbidClosed()) {
58+
if (request.request().indicesOptions() == SearchRequest.DEFAULT_INDICES_OPTIONS
59+
&& request().indicesOptions() != SearchRequest.DEFAULT_INDICES_OPTIONS) {
5760
request.request().indicesOptions(request().indicesOptions());
5861
}
5962

‎server/src/main/java/org/elasticsearch/action/search/SearchRequest.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ public final class SearchRequest extends ActionRequest implements IndicesRequest
8888

8989
private String[] types = Strings.EMPTY_ARRAY;
9090

91-
public static final IndicesOptions DEFAULT_INDICES_OPTIONS = IndicesOptions.strictExpandOpenAndForbidClosed();
91+
public static final IndicesOptions DEFAULT_INDICES_OPTIONS = IndicesOptions.strictExpandOpenAndForbidClosedIgnoreThrottled();
9292

9393
private IndicesOptions indicesOptions = DEFAULT_INDICES_OPTIONS;
9494

0 commit comments

Comments
 (0)