Skip to content

Commit 82c00f0

Browse files
markharwoodmarkharwood
authored
Search optimisation - add canMatch early aborts for queries on "_index" field (#48681)
Make queries on the “_index” field fast-fail if the target shard is an index that doesn’t match the query expression. Part of the “canMatch” phase optimisations. Closes #48473
1 parent 18c2aab commit 82c00f0

File tree

11 files changed

+333
-4
lines changed

11 files changed

+333
-4
lines changed

‎qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/70_skip_shards.yml

+163
Original file line numberDiff line numberDiff line change
@@ -58,3 +58,166 @@
5858
- match: { _shards.failed: 0 }
5959
- match: { hits.total: 1 }
6060

61+
---
62+
"Test that queries on _index field that don't match alias are skipped":
63+
64+
- do:
65+
indices.create:
66+
index: skip_shards_local_index
67+
body:
68+
settings:
69+
index:
70+
number_of_shards: 2
71+
number_of_replicas: 0
72+
mappings:
73+
properties:
74+
created_at:
75+
type: date
76+
format: "yyyy-MM-dd"
77+
78+
- do:
79+
bulk:
80+
refresh: true
81+
body:
82+
- '{"index": {"_index": "skip_shards_local_index"}}'
83+
- '{"f1": "local_cluster", "sort_field": 0, "created_at" : "2017-01-01"}'
84+
- '{"index": {"_index": "skip_shards_local_index"}}'
85+
- '{"f1": "local_cluster", "sort_field": 1, "created_at" : "2017-01-02"}'
86+
- do:
87+
indices.put_alias:
88+
index: skip_shards_local_index
89+
name: test_skip_alias
90+
91+
# check that we match the alias with term query
92+
- do:
93+
search:
94+
track_total_hits: true
95+
index: "skip_shards_local_index"
96+
pre_filter_shard_size: 1
97+
ccs_minimize_roundtrips: false
98+
body: { "size" : 10, "query" : { "term" : { "_index" : "test_skip_alias" } } }
99+
100+
- match: { hits.total.value: 2 }
101+
- match: { hits.hits.0._index: "skip_shards_local_index"}
102+
- match: { _shards.total: 2 }
103+
- match: { _shards.successful: 2 }
104+
- match: { _shards.skipped : 0}
105+
- match: { _shards.failed: 0 }
106+
107+
# check that we match the alias with terms query
108+
- do:
109+
search:
110+
track_total_hits: true
111+
index: "skip_shards_local_index"
112+
pre_filter_shard_size: 1
113+
ccs_minimize_roundtrips: false
114+
body: { "size" : 10, "query" : { "terms" : { "_index" : ["test_skip_alias", "does_not_match"] } } }
115+
116+
- match: { hits.total.value: 2 }
117+
- match: { hits.hits.0._index: "skip_shards_local_index"}
118+
- match: { _shards.total: 2 }
119+
- match: { _shards.successful: 2 }
120+
- match: { _shards.skipped : 0}
121+
- match: { _shards.failed: 0 }
122+
123+
# check that we match the alias with prefix query
124+
- do:
125+
search:
126+
track_total_hits: true
127+
index: "skip_shards_local_index"
128+
pre_filter_shard_size: 1
129+
ccs_minimize_roundtrips: false
130+
body: { "size" : 10, "query" : { "prefix" : { "_index" : "test_skip_ali" } } }
131+
132+
- match: { hits.total.value: 2 }
133+
- match: { hits.hits.0._index: "skip_shards_local_index"}
134+
- match: { _shards.total: 2 }
135+
- match: { _shards.successful: 2 }
136+
- match: { _shards.skipped : 0}
137+
- match: { _shards.failed: 0 }
138+
139+
# check that we match the alias with wildcard query
140+
- do:
141+
search:
142+
track_total_hits: true
143+
index: "skip_shards_local_index"
144+
pre_filter_shard_size: 1
145+
ccs_minimize_roundtrips: false
146+
body: { "size" : 10, "query" : { "wildcard" : { "_index" : "test_skip_ali*" } } }
147+
148+
- match: { hits.total.value: 2 }
149+
- match: { hits.hits.0._index: "skip_shards_local_index"}
150+
- match: { _shards.total: 2 }
151+
- match: { _shards.successful: 2 }
152+
- match: { _shards.skipped : 0}
153+
- match: { _shards.failed: 0 }
154+
155+
156+
# check that skipped when we don't match the alias with a term query
157+
- do:
158+
search:
159+
track_total_hits: true
160+
index: "skip_shards_local_index"
161+
pre_filter_shard_size: 1
162+
ccs_minimize_roundtrips: false
163+
body: { "size" : 10, "query" : { "term" : { "_index" : "does_not_match" } } }
164+
165+
166+
- match: { hits.total.value: 0 }
167+
- match: { _shards.total: 2 }
168+
- match: { _shards.successful: 2 }
169+
# When all shards are skipped current logic returns 1 to produce a valid search result
170+
- match: { _shards.skipped : 1}
171+
- match: { _shards.failed: 0 }
172+
173+
# check that skipped when we don't match the alias with a terms query
174+
- do:
175+
search:
176+
track_total_hits: true
177+
index: "skip_shards_local_index"
178+
pre_filter_shard_size: 1
179+
ccs_minimize_roundtrips: false
180+
body: { "size" : 10, "query" : { "terms" : { "_index" : ["does_not_match", "also_does_not_match"] } } }
181+
182+
183+
- match: { hits.total.value: 0 }
184+
- match: { _shards.total: 2 }
185+
- match: { _shards.successful: 2 }
186+
# When all shards are skipped current logic returns 1 to produce a valid search result
187+
- match: { _shards.skipped : 1}
188+
- match: { _shards.failed: 0 }
189+
190+
# check that skipped when we don't match the alias with a prefix query
191+
- do:
192+
search:
193+
track_total_hits: true
194+
index: "skip_shards_local_index"
195+
pre_filter_shard_size: 1
196+
ccs_minimize_roundtrips: false
197+
body: { "size" : 10, "query" : { "prefix" : { "_index" : "does_not_matc" } } }
198+
199+
200+
- match: { hits.total.value: 0 }
201+
- match: { _shards.total: 2 }
202+
- match: { _shards.successful: 2 }
203+
# When all shards are skipped current logic returns 1 to produce a valid search result
204+
- match: { _shards.skipped : 1}
205+
- match: { _shards.failed: 0 }
206+
207+
# check that skipped when we don't match the alias with a wildcard query
208+
- do:
209+
search:
210+
track_total_hits: true
211+
index: "skip_shards_local_index"
212+
pre_filter_shard_size: 1
213+
ccs_minimize_roundtrips: false
214+
body: { "size" : 10, "query" : { "wildcard" : { "_index" : "does_not_matc*" } } }
215+
216+
217+
- match: { hits.total.value: 0 }
218+
- match: { _shards.total: 2 }
219+
- match: { _shards.successful: 2 }
220+
# When all shards are skipped current logic returns 1 to produce a valid search result
221+
- match: { _shards.skipped : 1}
222+
- match: { _shards.failed: 0 }
223+

‎qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/90_index_name_query.yml

+44
Original file line numberDiff line numberDiff line change
@@ -56,3 +56,47 @@ teardown:
5656
- match: { _shards.successful: 2 }
5757
- match: { _shards.skipped : 0}
5858
- match: { _shards.failed: 0 }
59+
60+
---
61+
"Test that queries on _index that don't match are skipped":
62+
63+
- do:
64+
bulk:
65+
refresh: true
66+
body:
67+
- '{"index": {"_index": "single_doc_index"}}'
68+
- '{"f1": "local_cluster", "sort_field": 0}'
69+
70+
- do:
71+
search:
72+
ccs_minimize_roundtrips: false
73+
track_total_hits: true
74+
index: "single_doc_index,my_remote_cluster:single_doc_index"
75+
pre_filter_shard_size: 1
76+
body:
77+
query:
78+
term:
79+
"_index": "does_not_match"
80+
81+
- match: { hits.total.value: 0 }
82+
- match: { _shards.total: 2 }
83+
- match: { _shards.successful: 2 }
84+
- match: { _shards.skipped : 1}
85+
- match: { _shards.failed: 0 }
86+
87+
- do:
88+
search:
89+
ccs_minimize_roundtrips: false
90+
track_total_hits: true
91+
index: "single_doc_index,my_remote_cluster:single_doc_index"
92+
pre_filter_shard_size: 1
93+
body:
94+
query:
95+
term:
96+
"_index": "my_remote_cluster:does_not_match"
97+
98+
- match: { hits.total.value: 0 }
99+
- match: { _shards.total: 2 }
100+
- match: { _shards.successful: 2 }
101+
- match: { _shards.skipped : 1}
102+
- match: { _shards.failed: 0 }

‎server/src/main/java/org/elasticsearch/index/query/PrefixQueryBuilder.java

+13
Original file line numberDiff line numberDiff line change
@@ -168,6 +168,19 @@ public static PrefixQueryBuilder fromXContent(XContentParser parser) throws IOEx
168168
public String getWriteableName() {
169169
return NAME;
170170
}
171+
172+
@Override
173+
protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws IOException {
174+
if ("_index".equals(fieldName)) {
175+
// Special-case optimisation for canMatch phase:
176+
// We can skip querying this shard if the index name doesn't match the value of this query on the "_index" field.
177+
QueryShardContext shardContext = queryRewriteContext.convertToShardContext();
178+
if (shardContext != null && shardContext.indexMatches(value + "*") == false) {
179+
return new MatchNoneQueryBuilder();
180+
}
181+
}
182+
return super.doRewrite(queryRewriteContext);
183+
}
171184

172185
@Override
173186
protected Query doToQuery(QueryShardContext context) throws IOException {

‎server/src/main/java/org/elasticsearch/index/query/TermQueryBuilder.java

+13
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,19 @@ public static TermQueryBuilder fromXContent(XContentParser parser) throws IOExce
129129
}
130130
return termQuery;
131131
}
132+
133+
@Override
134+
protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws IOException {
135+
if ("_index".equals(fieldName)) {
136+
// Special-case optimisation for canMatch phase:
137+
// We can skip querying this shard if the index name doesn't match the value of this query on the "_index" field.
138+
QueryShardContext shardContext = queryRewriteContext.convertToShardContext();
139+
if (shardContext != null && shardContext.indexMatches(BytesRefs.toString(value)) == false) {
140+
return new MatchNoneQueryBuilder();
141+
}
142+
}
143+
return super.doRewrite(queryRewriteContext);
144+
}
132145

133146
@Override
134147
protected Query doToQuery(QueryShardContext context) throws IOException {

‎server/src/main/java/org/elasticsearch/index/query/TermsQueryBuilder.java

+15
Original file line numberDiff line numberDiff line change
@@ -482,6 +482,21 @@ protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) {
482482
})));
483483
return new TermsQueryBuilder(this.fieldName, supplier::get);
484484
}
485+
if ("_index".equals(this.fieldName) && values != null) {
486+
// Special-case optimisation for canMatch phase:
487+
// We can skip querying this shard if the index name doesn't match any of the search terms.
488+
QueryShardContext shardContext = queryRewriteContext.convertToShardContext();
489+
if (shardContext != null) {
490+
for (Object localValue : values) {
491+
if (shardContext.indexMatches(BytesRefs.toString(localValue))) {
492+
// We can match - at least one index name matches
493+
return this;
494+
}
495+
}
496+
// all index names are invalid - no possibility of a match on this shard.
497+
return new MatchNoneQueryBuilder();
498+
}
499+
}
485500
return this;
486501
}
487502
}

‎server/src/main/java/org/elasticsearch/index/query/WildcardQueryBuilder.java

+15-1
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import org.elasticsearch.common.Strings;
2828
import org.elasticsearch.common.io.stream.StreamInput;
2929
import org.elasticsearch.common.io.stream.StreamOutput;
30+
import org.elasticsearch.common.lucene.BytesRefs;
3031
import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
3132
import org.elasticsearch.common.xcontent.XContentBuilder;
3233
import org.elasticsearch.common.xcontent.XContentParser;
@@ -177,7 +178,20 @@ public static WildcardQueryBuilder fromXContent(XContentParser parser) throws IO
177178
.rewrite(rewrite)
178179
.boost(boost)
179180
.queryName(queryName);
180-
}
181+
}
182+
183+
@Override
184+
protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws IOException {
185+
if ("_index".equals(fieldName)) {
186+
// Special-case optimisation for canMatch phase:
187+
// We can skip querying this shard if the index name doesn't match the value of this query on the "_index" field.
188+
QueryShardContext shardContext = queryRewriteContext.convertToShardContext();
189+
if (shardContext != null && shardContext.indexMatches(BytesRefs.toString(value)) == false) {
190+
return new MatchNoneQueryBuilder();
191+
}
192+
}
193+
return super.doRewrite(queryRewriteContext);
194+
}
181195

182196
@Override
183197
protected Query doToQuery(QueryShardContext context) throws IOException {

‎server/src/test/java/org/elasticsearch/index/query/PrefixQueryBuilderTests.java

+15
Original file line numberDiff line numberDiff line change
@@ -141,4 +141,19 @@ public void testParseFailsWithMultipleFields() throws IOException {
141141
e = expectThrows(ParsingException.class, () -> parseQuery(shortJson));
142142
assertEquals("[prefix] query doesn't support multiple fields, found [user1] and [user2]", e.getMessage());
143143
}
144+
145+
public void testRewriteIndexQueryToMatchNone() throws Exception {
146+
PrefixQueryBuilder query = prefixQuery("_index", "does_not_exist");
147+
QueryShardContext queryShardContext = createShardContext();
148+
QueryBuilder rewritten = query.rewrite(queryShardContext);
149+
assertThat(rewritten, instanceOf(MatchNoneQueryBuilder.class));
150+
}
151+
152+
public void testRewriteIndexQueryToNotMatchNone() throws Exception {
153+
PrefixQueryBuilder query = prefixQuery("_index", getIndex().getName());
154+
QueryShardContext queryShardContext = createShardContext();
155+
QueryBuilder rewritten = query.rewrite(queryShardContext);
156+
assertThat(rewritten, instanceOf(PrefixQueryBuilder.class));
157+
}
158+
144159
}

‎server/src/test/java/org/elasticsearch/index/query/TermQueryBuilderTests.java

+15-1
Original file line numberDiff line numberDiff line change
@@ -172,5 +172,19 @@ public void testTypeField() throws IOException {
172172
TermQueryBuilder builder = QueryBuilders.termQuery("_type", "value1");
173173
builder.doToQuery(createShardContext());
174174
assertWarnings(QueryShardContext.TYPES_DEPRECATION_MESSAGE);
175-
}
175+
}
176+
177+
public void testRewriteIndexQueryToMatchNone() throws IOException {
178+
TermQueryBuilder query = QueryBuilders.termQuery("_index", "does_not_exist");
179+
QueryShardContext queryShardContext = createShardContext();
180+
QueryBuilder rewritten = query.rewrite(queryShardContext);
181+
assertThat(rewritten, instanceOf(MatchNoneQueryBuilder.class));
182+
}
183+
184+
public void testRewriteIndexQueryToNotMatchNone() throws IOException {
185+
TermQueryBuilder query = QueryBuilders.termQuery("_index", getIndex().getName());
186+
QueryShardContext queryShardContext = createShardContext();
187+
QueryBuilder rewritten = query.rewrite(queryShardContext);
188+
assertThat(rewritten, instanceOf(TermQueryBuilder.class));
189+
}
176190
}

‎server/src/test/java/org/elasticsearch/index/query/TermsQueryBuilderTests.java

+16-1
Original file line numberDiff line numberDiff line change
@@ -312,7 +312,22 @@ public void testTypeField() throws IOException {
312312
builder.doToQuery(createShardContext());
313313
assertWarnings(QueryShardContext.TYPES_DEPRECATION_MESSAGE);
314314
}
315-
315+
316+
public void testRewriteIndexQueryToMatchNone() throws IOException {
317+
TermsQueryBuilder query = new TermsQueryBuilder("_index", "does_not_exist", "also_does_not_exist");
318+
QueryShardContext queryShardContext = createShardContext();
319+
QueryBuilder rewritten = query.rewrite(queryShardContext);
320+
assertThat(rewritten, instanceOf(MatchNoneQueryBuilder.class));
321+
}
322+
323+
public void testRewriteIndexQueryToNotMatchNone() throws IOException {
324+
// At least one name is good
325+
TermsQueryBuilder query = new TermsQueryBuilder("_index", "does_not_exist", getIndex().getName());
326+
QueryShardContext queryShardContext = createShardContext();
327+
QueryBuilder rewritten = query.rewrite(queryShardContext);
328+
assertThat(rewritten, instanceOf(TermsQueryBuilder.class));
329+
}
330+
316331
@Override
317332
protected QueryBuilder parseQuery(XContentParser parser) throws IOException {
318333
QueryBuilder query = super.parseQuery(parser);

‎server/src/test/java/org/elasticsearch/index/query/WildcardQueryBuilderTests.java

+16
Original file line numberDiff line numberDiff line change
@@ -138,4 +138,20 @@ public void testTypeField() throws IOException {
138138
builder.doToQuery(createShardContext());
139139
assertWarnings(QueryShardContext.TYPES_DEPRECATION_MESSAGE);
140140
}
141+
142+
public void testRewriteIndexQueryToMatchNone() throws IOException {
143+
WildcardQueryBuilder query = new WildcardQueryBuilder("_index", "does_not_exist");
144+
QueryShardContext queryShardContext = createShardContext();
145+
QueryBuilder rewritten = query.rewrite(queryShardContext);
146+
assertThat(rewritten, instanceOf(MatchNoneQueryBuilder.class));
147+
}
148+
149+
public void testRewriteIndexQueryNotMatchNone() throws IOException {
150+
String fullIndexName = getIndex().getName();
151+
String firstHalfOfIndexName = fullIndexName.substring(0,fullIndexName.length()/2);
152+
WildcardQueryBuilder query = new WildcardQueryBuilder("_index", firstHalfOfIndexName +"*");
153+
QueryShardContext queryShardContext = createShardContext();
154+
QueryBuilder rewritten = query.rewrite(queryShardContext);
155+
assertThat(rewritten, instanceOf(WildcardQueryBuilder.class));
156+
}
141157
}

0 commit comments

Comments
 (0)