Add support for aliases in queries on _index. (elastic#46944)

Previously, queries on the _index field were not able to specify index aliases.
This was a regression in functionality compared to the 'indices' query that was
deprecated and removed in 6.0.

Now queries on _index can specify an alias, which is resolved to the concrete
index names when we check whether an index matches. To match a remote shard
target, the pattern needs to be of the form 'cluster:index' to match the
fully-qualified index name. Index aliases can be specified in the following query
types: term, terms, prefix, and wildcard.

Author: jtibshirani

docs/reference/migration/migrate_7_5.asciidoc

@@ -0,0 +1,30 @@
+[[breaking-changes-7.5]]
+== Breaking changes in 7.5
+++++
+<titleabbrev>7.5</titleabbrev>
+++++
+
+This section discusses the changes that you need to be aware of when migrating
+your application to Elasticsearch 7.5.
+
+See also <<release-highlights>> and <<es-release-notes>>.
+
+coming[7.5.0]
+
+//NOTE: The notable-breaking-changes tagged regions are re-used in the
+//Installation and Upgrade Guide
+
+//tag::notable-breaking-changes[]
+
+//end::notable-breaking-changes[]
+
+[discrete]
+[[breaking_75_search_changes]]
+=== Search Changes
+
+[discrete]
+==== Stricter checking for wildcard queries on _index
+Previously, a wildcard query on the `_index` field matched directly against the
+fully-qualified index name. Now, in order to match against remote indices like
+i`cluster:index`, the query must contain a colon, as in `cl*ster:inde*`. This
+behavior aligns with the way indices are matched in the search endpoint.

qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/90_index_name_query.yml

@@ -0,0 +1,58 @@
+---
+setup:
+  - do:
+      indices.create:
+        index: single_doc_index
+        body:
+          settings:
+            index:
+              number_of_shards: 1
+              number_of_replicas: 0
+---
+teardown:
+  - do:
+      indices.delete:
+        index: single_doc_index
+        ignore_unavailable: true
+
+---
+"Test that queries on _index match against the correct indices.":
+
+  - do:
+      bulk:
+        refresh: true
+        body:
+            - '{"index": {"_index": "single_doc_index"}}'
+            - '{"f1": "local_cluster", "sort_field": 0}'
+
+  - do:
+     search:
+       rest_total_hits_as_int: true
+       index: "single_doc_index,my_remote_cluster:single_doc_index"
+       body:
+         query:
+            term:
+              "_index": "single_doc_index"
+
+  - match: { hits.total: 1 }
+  - match: { hits.hits.0._index: "single_doc_index"}
+  - match: { _shards.total: 2 }
+  - match: { _shards.successful: 2 }
+  - match: { _shards.skipped : 0}
+  - match: { _shards.failed: 0 }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: "single_doc_index,my_remote_cluster:single_doc_index"
+        body:
+          query:
+            term:
+              "_index": "my_remote_cluster:single_doc_index"
+
+  - match: { hits.total: 1 }
+  - match: { hits.hits.0._index: "my_remote_cluster:single_doc_index"}
+  - match: { _shards.total: 2 }
+  - match: { _shards.successful: 2 }
+  - match: { _shards.skipped : 0}
+  - match: { _shards.failed: 0 }

server/src/main/java/org/elasticsearch/index/IndexModule.java

@@ -30,6 +30,7 @@
 import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.Version;
 import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.TriFunction;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
@@ -386,6 +387,7 @@ public IndexService newIndexService(
             BigArrays bigArrays,
             ThreadPool threadPool,
             ScriptService scriptService,
+            ClusterService clusterService,
             Client client,
             IndicesQueryCache indicesQueryCache,
             MapperRegistry mapperRegistry,
@@ -411,7 +413,7 @@ public IndexService newIndexService(
         return new IndexService(indexSettings, indexCreationContext, environment, xContentRegistry,
                 new SimilarityService(indexSettings, scriptService, similarities),
                 shardStoreDeleter, analysisRegistry, engineFactory, circuitBreakerService, bigArrays, threadPool, scriptService,
-                client, queryCache, directoryFactory, eventListener, readerWrapperFactory, mapperRegistry,
+                clusterService, client, queryCache, directoryFactory, eventListener, readerWrapperFactory, mapperRegistry,
                 indicesFieldDataCache, searchOperationListeners, indexOperationListeners, namedWriteableRegistry);
     }
 

server/src/main/java/org/elasticsearch/index/IndexService.java

@@ -32,6 +32,7 @@
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
@@ -57,6 +58,7 @@
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
 import org.elasticsearch.index.fielddata.IndexFieldDataService;
 import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.query.SearchIndexNameMatcher;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.seqno.RetentionLeaseSyncer;
 import org.elasticsearch.index.shard.IndexEventListener;
@@ -134,6 +136,7 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust
     private final ThreadPool threadPool;
     private final BigArrays bigArrays;
     private final ScriptService scriptService;
+    private final ClusterService clusterService;
     private final Client client;
     private final CircuitBreakerService circuitBreakerService;
     private Supplier<Sort> indexSortSupplier;
@@ -151,6 +154,7 @@ public IndexService(
             BigArrays bigArrays,
             ThreadPool threadPool,
             ScriptService scriptService,
+            ClusterService clusterService,
             Client client,
             QueryCache queryCache,
             IndexStorePlugin.DirectoryFactory directoryFactory,
@@ -201,6 +205,7 @@ public IndexService(
         this.bigArrays = bigArrays;
         this.threadPool = threadPool;
         this.scriptService = scriptService;
+        this.clusterService = clusterService;
         this.client = client;
         this.eventListener = eventListener;
         this.nodeEnv = nodeEnv;
@@ -530,9 +535,11 @@ public IndexSettings getIndexSettings() {
      * {@link IndexReader}-specific optimizations, such as rewriting containing range queries.
      */
     public QueryShardContext newQueryShardContext(int shardId, IndexSearcher searcher, LongSupplier nowInMillis, String clusterAlias) {
+        SearchIndexNameMatcher indexNameMatcher = new SearchIndexNameMatcher(index().getName(), clusterAlias, clusterService);
         return new QueryShardContext(
             shardId, indexSettings, bigArrays, indexCache.bitsetFilterCache(), indexFieldData::getForField, mapperService(),
-            similarityService(), scriptService, xContentRegistry, namedWriteableRegistry, client, searcher, nowInMillis, clusterAlias);
+            similarityService(), scriptService, xContentRegistry, namedWriteableRegistry, client, searcher, nowInMillis, clusterAlias,
+            indexNameMatcher);
     }
 
     /**

server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java

@@ -129,11 +129,16 @@ public Query existsQuery(QueryShardContext context) {
          */
         @Override
         public Query termQuery(Object value, @Nullable QueryShardContext context) {
-            if (isSameIndex(value, context.getFullyQualifiedIndex().getName())) {
+            String pattern = value instanceof BytesRef
+                ? ((BytesRef) value).utf8ToString()
+                : value.toString();
+            if (context.indexMatches(pattern)) {
+                // No need to OR these clauses - we can only logically be
+                // running in the context of just one of these index names.
                 return Queries.newMatchAllQuery();
             } else {
-                return Queries.newMatchNoDocsQuery("Index didn't match. Index queried: " + context.index().getName()
-                    + " vs. " + value);
+                return Queries.newMatchNoDocsQuery("The index [" + context.getFullyQualifiedIndex().getName() +
+                    "] doesn't match the provided value [" + value + "].");
             }
         }
 
@@ -143,26 +148,29 @@ public Query termsQuery(List values, QueryShardContext context) {
                 return super.termsQuery(values, context);
             }
             for (Object value : values) {
-                if (isSameIndex(value, context.getFullyQualifiedIndex().getName())) {
+                String pattern = value instanceof BytesRef
+                    ? ((BytesRef) value).utf8ToString()
+                    : value.toString();
+                if (context.indexMatches(pattern)) {
                     // No need to OR these clauses - we can only logically be
                     // running in the context of just one of these index names.
                     return Queries.newMatchAllQuery();
                 }
             }
             // None of the listed index names are this one
-            return Queries.newMatchNoDocsQuery("Index didn't match. Index queried: " + context.getFullyQualifiedIndex().getName()
-                + " vs. " + values);
+            return Queries.newMatchNoDocsQuery("The index [" + context.getFullyQualifiedIndex().getName() +
+                "] doesn't match the provided values [" + values + "].");
         }
 
         @Override
         public Query prefixQuery(String value,
                                  @Nullable MultiTermQuery.RewriteMethod method,
                                  QueryShardContext context) {
-            String indexName = context.getFullyQualifiedIndex().getName();
-            if (indexName.startsWith(value)) {
+            String pattern = value + "*";
+            if (context.indexMatches(pattern)) {
                 return Queries.newMatchAllQuery();
             } else {
-                return Queries.newMatchNoDocsQuery("The index [" + indexName +
+                return Queries.newMatchNoDocsQuery("The index [" + context.getFullyQualifiedIndex().getName() +
                     "] doesn't match the provided prefix [" + value + "].");
             }
         }
@@ -176,29 +184,23 @@ public Query regexpQuery(String value, int flags, int maxDeterminizedStates,
             if (pattern.matcher(indexName).matches()) {
                 return Queries.newMatchAllQuery();
             } else {
-                return Queries.newMatchNoDocsQuery("The index [" + indexName +
-                    "] doesn't match the provided pattern [" + value + "].");
+                return Queries.newMatchNoDocsQuery("The index [" + context.getFullyQualifiedIndex().getName()
+                    + "] doesn't match the provided pattern [" + value + "].");
             }
         }
 
         @Override
         public Query wildcardQuery(String value,
                                    @Nullable MultiTermQuery.RewriteMethod method,
                                    QueryShardContext context) {
-            String indexName = context.getFullyQualifiedIndex().getName();
-            if (isSameIndex(value, indexName)) {
+            if (context.indexMatches(value)) {
                 return Queries.newMatchAllQuery();
             } else {
-                return Queries.newMatchNoDocsQuery("The index [" + indexName +
-                    "] doesn't match the provided pattern [" + value + "].");
+                return Queries.newMatchNoDocsQuery("The index [" + context.getFullyQualifiedIndex().getName()
+                    + "] doesn't match the provided pattern [" + value + "].");
             }
         }
 
-        private boolean isSameIndex(Object value, String indexName) {
-            String pattern = value instanceof BytesRef ? ((BytesRef) value).utf8ToString() : value.toString();
-            return Regex.simpleMatch(pattern, indexName);
-        }
-
         @Override
         public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) {
             return new ConstantIndexFieldData.Builder(mapperService -> fullyQualifiedIndexName);

server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java

@@ -69,6 +69,7 @@
 import java.util.function.BiConsumer;
 import java.util.function.BiFunction;
 import java.util.function.LongSupplier;
+import java.util.function.Predicate;
 
 import static java.util.Collections.unmodifiableMap;
 
@@ -93,7 +94,9 @@ public class QueryShardContext extends QueryRewriteContext {
     private String[] types = Strings.EMPTY_ARRAY;
     private boolean cacheable = true;
     private final SetOnce<Boolean> frozen = new SetOnce<>();
+
     private final Index fullyQualifiedIndex;
+    private final Predicate<String> indexNameMatcher;
 
     public void setTypes(String... types) {
         this.types = types;
@@ -109,45 +112,48 @@ public String[] getTypes() {
     private NestedScope nestedScope;
 
     public QueryShardContext(int shardId,
-                                IndexSettings indexSettings,
-                                BigArrays bigArrays,
-                                BitsetFilterCache bitsetFilterCache,
-                                BiFunction<MappedFieldType, String, IndexFieldData<?>> indexFieldDataLookup,
-                                MapperService mapperService,
-                                SimilarityService similarityService,
-                                ScriptService scriptService,
-                                NamedXContentRegistry xContentRegistry,
-                                NamedWriteableRegistry namedWriteableRegistry,
-                                Client client,
-                                IndexSearcher searcher,
-                                LongSupplier nowInMillis,
-                                String clusterAlias) {
+                             IndexSettings indexSettings,
+                             BigArrays bigArrays,
+                             BitsetFilterCache bitsetFilterCache,
+                             BiFunction<MappedFieldType, String, IndexFieldData<?>> indexFieldDataLookup,
+                             MapperService mapperService,
+                             SimilarityService similarityService,
+                             ScriptService scriptService,
+                             NamedXContentRegistry xContentRegistry,
+                             NamedWriteableRegistry namedWriteableRegistry,
+                             Client client,
+                             IndexSearcher searcher,
+                             LongSupplier nowInMillis,
+                             String clusterAlias,
+                             Predicate<String> indexNameMatcher) {
         this(shardId, indexSettings, bigArrays, bitsetFilterCache, indexFieldDataLookup, mapperService, similarityService,
-            scriptService, xContentRegistry, namedWriteableRegistry, client, searcher, nowInMillis,
+            scriptService, xContentRegistry, namedWriteableRegistry, client, searcher, nowInMillis, indexNameMatcher,
             new Index(RemoteClusterAware.buildRemoteIndexName(clusterAlias, indexSettings.getIndex().getName()),
                 indexSettings.getIndex().getUUID()));
     }
 
     public QueryShardContext(QueryShardContext source) {
         this(source.shardId, source.indexSettings, source.bigArrays, source.bitsetFilterCache, source.indexFieldDataService,
-                source.mapperService, source.similarityService, source.scriptService, source.getXContentRegistry(),
-                source.getWriteableRegistry(), source.client, source.searcher, source.nowInMillis, source.fullyQualifiedIndex);
+            source.mapperService, source.similarityService, source.scriptService, source.getXContentRegistry(),
+            source.getWriteableRegistry(), source.client, source.searcher, source.nowInMillis, source.indexNameMatcher,
+            source.fullyQualifiedIndex);
     }
 
     private QueryShardContext(int shardId,
-                                IndexSettings indexSettings,
-                                BigArrays bigArrays,
-                                BitsetFilterCache bitsetFilterCache,
-                                BiFunction<MappedFieldType, String, IndexFieldData<?>> indexFieldDataLookup,
-                                MapperService mapperService,
-                                SimilarityService similarityService,
-                                ScriptService scriptService,
-                                NamedXContentRegistry xContentRegistry,
-                                NamedWriteableRegistry namedWriteableRegistry,
-                                Client client,
-                                IndexSearcher searcher,
-                                LongSupplier nowInMillis,
-                                Index fullyQualifiedIndex) {
+                              IndexSettings indexSettings,
+                              BigArrays bigArrays,
+                              BitsetFilterCache bitsetFilterCache,
+                              BiFunction<MappedFieldType, String, IndexFieldData<?>> indexFieldDataLookup,
+                              MapperService mapperService,
+                              SimilarityService similarityService,
+                              ScriptService scriptService,
+                              NamedXContentRegistry xContentRegistry,
+                              NamedWriteableRegistry namedWriteableRegistry,
+                              Client client,
+                              IndexSearcher searcher,
+                              LongSupplier nowInMillis,
+                              Predicate<String> indexNameMatcher,
+                              Index fullyQualifiedIndex) {
         super(xContentRegistry, namedWriteableRegistry, client, nowInMillis);
         this.shardId = shardId;
         this.similarityService = similarityService;
@@ -160,6 +166,7 @@ private QueryShardContext(int shardId,
         this.scriptService = scriptService;
         this.indexSettings = indexSettings;
         this.searcher = searcher;
+        this.indexNameMatcher = indexNameMatcher;
         this.fullyQualifiedIndex = fullyQualifiedIndex;
     }
 
@@ -311,6 +318,14 @@ public Version indexVersionCreated() {
         return indexSettings.getIndexVersionCreated();
     }
 
+    /**
+     *  Given an index pattern, checks whether it matches against the current shard. The pattern
+     *  may represent a fully qualified index name if the search targets remote shards.
+     */
+    public boolean indexMatches(String pattern) {
+        return indexNameMatcher.test(pattern);
+    }
+
     public ParsedQuery toQuery(QueryBuilder queryBuilder) {
         return toQuery(queryBuilder, q -> {
             Query query = q.toQuery(this);