Add runtime_mappings to search request (backport of #64374) (#64889)

* Add `runtime_mappings` to search request (backport of #64374)

This adds a way to specify the `runtime_mappings` on a search request
which are always "runtime" fields. It looks like:
```
curl -XDELETE -uelastic:password -HContent-Type:application/json localhost:9200/test
curl -XPOST -uelastic:password -HContent-Type:application/json 'localhost:9200/test/_bulk?pretty&refresh' -d'
{"index": {}}
{"animal": "cat", "sound": "meow"}
{"index": {}}
{"animal": "dog", "sound": "woof"}
{"index": {}}
{"animal": "snake", "sound": "hisssssssssssssssss"}
'

curl -XPOST -uelastic:password -HContent-Type:application/json localhost:9200/test/_search?pretty -d'
{
  "runtime_mappings": {
    "animal.upper": {
      "type": "keyword",
      "script": "for (String s : doc[\"animal.keyword\"]) {emit(s.toUpperCase())}"
    }
  },
  "query": {
    "match": {
      "animal.upper": "DOG"
    }
  }
}'
```

NOTE:
If we have to send a search request with runtime mappings to a node that
doesn't support runtime mappings at all then we'll fail the search
request entirely. The alternative would be to not send those runtime
mappings and let the node fail the search request with an "unknown field"
error. I believe this is would be hard to surprising because you defined
the field in the search request.

NOTE:
It isn't obvious but you can also use `runtime_mappings` to override fields
inside objects by naming the runtime fields with `.` in them. Like this:
```
curl -XDELETE -uelastic:password -HContent-Type:application/json localhost:9200/test
curl -uelastic:password -XPOST -HContent-Type:application/json localhost:9200/test/_bulk?refresh -d'
{"index":{}}
{"name": {"first": "Andrew", "last": "Wiggin"}}
{"index":{}}
{"name": {"first": "Julian", "last": "Delphiki", "suffix": "II"}}
'

curl -uelastic:password -XPOST -HContent-Type:application/json localhost:9200/test/_search?pretty -d'{
  "runtime_mappings": {
    "name.first": {
      "type": "keyword",
      "script": "if (\"Wiggin\".equals(doc[\"name.last.keyword\"].value)) {emit(\"Ender\");} else if (\"Delphiki\".equals(doc[\"name.last.keyword\"].value)) {emit(\"Bean\");}"
    }
  },
  "query": {
    "match": {
      "name.first": "Bean"
    }
  }
}'
```

Relates to #59332

Author: nik9000

modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/DisableGraphQueryTests.java

@@ -46,6 +46,7 @@
 import java.util.Collection;
 import java.util.Collections;
 
+import static java.util.Collections.emptyMap;
 import static org.hamcrest.Matchers.equalTo;
 
 /**
@@ -84,7 +85,7 @@ public void setup() {
         indexService = createIndex("test", settings, "t",
             "text_shingle", "type=text,analyzer=text_shingle",
             "text_shingle_unigram", "type=text,analyzer=text_shingle_unigram");
-        shardContext = indexService.newQueryShardContext(0, null, () -> 0L, null);
+        shardContext = indexService.newQueryShardContext(0, null, () -> 0L, null, emptyMap());
 
         // parsed queries for "text_shingle_unigram:(foo bar baz)" with query parsers
         // that ignores position length attribute

modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java

@@ -29,9 +29,9 @@
 import org.apache.lucene.search.ScoreMode;
 import org.apache.lucene.search.Scorer;
 import org.apache.lucene.search.Weight;
-import org.elasticsearch.Version;
 import org.apache.lucene.store.ByteBuffersDirectory;
 import org.apache.lucene.store.Directory;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.ActionType;
@@ -89,6 +89,7 @@
 import java.util.Objects;
 
 import static java.util.Arrays.asList;
+import static java.util.Collections.emptyMap;
 import static java.util.Collections.unmodifiableList;
 import static org.elasticsearch.action.ValidateActions.addValidationError;
 import static org.elasticsearch.rest.RestRequest.Method.GET;
@@ -577,7 +578,7 @@ private static Response prepareRamIndex(Request request,
                         searcher.setQueryCache(null);
                         final long absoluteStartMillis = System.currentTimeMillis();
                         QueryShardContext context =
-                                indexService.newQueryShardContext(0, searcher, () -> absoluteStartMillis, null);
+                                indexService.newQueryShardContext(0, searcher, () -> absoluteStartMillis, null, emptyMap());
                         return handler.apply(context, indexReader.leaves().get(0));
                     }
                 }

modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java

@@ -32,6 +32,8 @@
 import java.util.List;
 import java.util.Map;
 
+import static java.util.Collections.emptyMap;
+
 /**
  * Test that needsScores() is reported correctly depending on whether _score is used
  */
@@ -45,7 +47,7 @@ public void testNeedsScores() {
         contexts.put(NumberSortScript.CONTEXT, Whitelist.BASE_WHITELISTS);
         PainlessScriptEngine service = new PainlessScriptEngine(Settings.EMPTY, contexts);
 
-        QueryShardContext shardContext = index.newQueryShardContext(0, null, () -> 0, null);
+        QueryShardContext shardContext = index.newQueryShardContext(0, null, () -> 0, null, emptyMap());
 
         NumberSortScript.Factory factory = service.compile(null, "1.2", NumberSortScript.CONTEXT, Collections.emptyMap());
         NumberSortScript.LeafFactory ss = factory.newFactory(Collections.emptyMap(), shardContext.lookup());

modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java

@@ -103,6 +103,7 @@
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
+import static java.util.Collections.emptyMap;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
@@ -535,7 +536,7 @@ public void testQueryWithRewrite() throws Exception {
         QueryShardContext shardContext = indexService.newQueryShardContext(
             randomInt(20), null, () -> {
                 throw new UnsupportedOperationException();
-            }, null);
+            }, null, emptyMap());
         PlainActionFuture<QueryBuilder> future = new PlainActionFuture<>();
         Rewriteable.rewriteAndFetch(queryBuilder, shardContext, future);
         assertQueryBuilder(qbSource, future.get());

modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchTests.java

@@ -51,6 +51,7 @@
 import java.util.Map;
 import java.util.function.Function;
 
+import static java.util.Collections.emptyMap;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
 import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
@@ -258,7 +259,7 @@ public void testRangeQueriesWithNow() throws Exception {
         try (Engine.Searcher searcher = indexService.getShard(0).acquireSearcher("test")) {
             long[] currentTime = new long[] {System.currentTimeMillis()};
             QueryShardContext queryShardContext =
-                indexService.newQueryShardContext(0, searcher, () -> currentTime[0], null);
+                indexService.newQueryShardContext(0, searcher, () -> currentTime[0], null, emptyMap());
 
             BytesReference source = BytesReference.bytes(jsonBuilder().startObject()
                 .field("field1", "value")

modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelperTests.java

@@ -38,6 +38,7 @@ public void testSliceIntoSubRequests() throws IOException {
                 () -> null,
                 () -> null,
                 () -> emptyList(),
+                () -> null,
                 () -> null));
         if (searchRequest.source() != null) {
             // Clear the slice builder if there is one set. We can't call sliceIntoSubRequests if it is.

rest-api-spec/src/main/resources/rest-api-spec/test/index/80_date_nanos.yml

@@ -0,0 +1,36 @@
+---
+"date_nanos requires dates after 1970 and before 2262":
+  - skip:
+      version: " - 6.99.99"
+      reason:  "Implemented in 7.0"
+
+  - do:
+      indices.create:
+          index: date_ns
+          body:
+            settings:
+              number_of_shards: 3
+              number_of_replicas: 0
+            mappings:
+              properties:
+                date:
+                  type: date_nanos
+                field:
+                  type: long
+
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - '{ "index" : { "_index" : "date_ns", "_id" : "date_ns_1" } }'
+          - '{"date" : "1969-10-28T12:12:12.123456789Z" }'
+          - '{ "index" : { "_index" : "date_ns", "_id" : "date_ns_2" } }'
+          - '{"date" : "2263-10-29T12:12:12.123456789Z" }'
+
+  - match: { errors: true }
+  - match: { items.0.index.status: 400 }
+  - match: { items.0.index.error.type: mapper_parsing_exception }
+  - match: { items.0.index.error.caused_by.reason: "date[1969-10-28T12:12:12.123456789Z] is before the epoch in 1970 and cannot be stored in nanosecond resolution" }
+  - match: { items.1.index.status: 400 }
+  - match: { items.1.index.error.type: mapper_parsing_exception }
+  - match: { items.1.index.error.caused_by.reason: "date[2263-10-29T12:12:12.123456789Z] is after 2262-04-11T23:47:16.854775807 and cannot be stored in nanosecond resolution" }

rest-api-spec/src/main/resources/rest-api-spec/test/search/240_date_nanos.yml

@@ -73,28 +73,6 @@ setup:
   - match: { hits.hits.1._id: "second" }
   - match: { hits.hits.1.sort: [1540815132987654321] }
 
-
----
-"date_nanos requires dates after 1970 and before 2262":
-
-  - do:
-      bulk:
-        refresh: true
-        body:
-          - '{ "index" : { "_index" : "date_ns", "_id" : "date_ns_1" } }'
-          - '{"date" : "1969-10-28T12:12:12.123456789Z" }'
-          - '{ "index" : { "_index" : "date_ns", "_id" : "date_ns_2" } }'
-          - '{"date" : "2263-10-29T12:12:12.123456789Z" }'
-
-  - match: { errors: true }
-  - match: { items.0.index.status: 400 }
-  - match: { items.0.index.error.type: mapper_parsing_exception }
-  - match: { items.0.index.error.caused_by.reason: "date[1969-10-28T12:12:12.123456789Z] is before the epoch in 1970 and cannot be stored in nanosecond resolution" }
-  - match: { items.1.index.status: 400 }
-  - match: { items.1.index.error.type: mapper_parsing_exception }
-  - match: { items.1.index.error.caused_by.reason: "date[2263-10-29T12:12:12.123456789Z] is after 2262-04-11T23:47:16.854775807 and cannot be stored in nanosecond resolution" }
-
-
 ---
 "doc value fields are working as expected across date and date_nanos fields":
 

server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java

@@ -55,6 +55,7 @@
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
+import static java.util.Collections.emptyMap;
 import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findConflictingV1Templates;
 import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findConflictingV2Templates;
 import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findV2Template;
@@ -181,7 +182,7 @@ public static Template resolveTemplate(final String matchingTemplate, final Stri
                 resolvedAliases, tempClusterState.metadata(), aliasValidator, xContentRegistry,
                 // the context is only used for validation so it's fine to pass fake values for the
                 // shard id and the current timestamp
-                tempIndexService.newQueryShardContext(0, null, () -> 0L, null)));
+                tempIndexService.newQueryShardContext(0, null, () -> 0L, null, emptyMap())));
         Map<String, AliasMetadata> aliasesByName = aliases.stream().collect(
             Collectors.toMap(AliasMetadata::getAlias, Function.identity()));
 

server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java

@@ -89,7 +89,8 @@ public void run() {
                     CollapseBuilder innerCollapseBuilder = innerHitBuilder.getInnerCollapseBuilder();
                     SearchSourceBuilder sourceBuilder = buildExpandSearchSourceBuilder(innerHitBuilder, innerCollapseBuilder)
                         .query(groupQuery)
-                        .postFilter(searchRequest.source().postFilter());
+                        .postFilter(searchRequest.source().postFilter())
+                        .runtimeMappings(searchRequest.source().runtimeMappings());
                     SearchRequest groupRequest = new SearchRequest(searchRequest);
                     groupRequest.source(sourceBuilder);
                     multiRequest.add(groupRequest);

server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java

@@ -20,6 +20,7 @@
 package org.elasticsearch.cluster.metadata;
 
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
+
 import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -100,6 +101,7 @@
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
+import static java.util.Collections.emptyMap;
 import static java.util.Collections.singletonMap;
 import static java.util.stream.Collectors.toList;
 import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING;
@@ -494,7 +496,7 @@ private ClusterState applyCreateIndexRequestWithV1Templates(final ClusterState c
                 MetadataIndexTemplateService.resolveAliases(templates), currentState.metadata(), aliasValidator,
                 // the context is only used for validation so it's fine to pass fake values for the
                 // shard id and the current timestamp
-                xContentRegistry, indexService.newQueryShardContext(0, null, () -> 0L, null)),
+                xContentRegistry, indexService.newQueryShardContext(0, null, () -> 0L, null, emptyMap())),
             templates.stream().map(IndexTemplateMetadata::getName).collect(toList()), metadataTransformer);
     }
 
@@ -527,7 +529,7 @@ private ClusterState applyCreateIndexRequestWithV2Template(final ClusterState cu
                 MetadataIndexTemplateService.resolveAliases(currentState.metadata(), templateName), currentState.metadata(), aliasValidator,
                 // the context is only used for validation so it's fine to pass fake values for the
                 // shard id and the current timestamp
-                xContentRegistry, indexService.newQueryShardContext(0, null, () -> 0L, null)),
+                xContentRegistry, indexService.newQueryShardContext(0, null, () -> 0L, null, emptyMap())),
             Collections.singletonList(templateName), metadataTransformer);
     }
 
@@ -578,7 +580,7 @@ private ClusterState applyCreateIndexRequestWithExistingMetadata(final ClusterSt
                 currentState.metadata(), aliasValidator, xContentRegistry,
                 // the context is only used for validation so it's fine to pass fake values for the
                 // shard id and the current timestamp
-                indexService.newQueryShardContext(0, null, () -> 0L, null)),
+                indexService.newQueryShardContext(0, null, () -> 0L, null, emptyMap())),
             org.elasticsearch.common.collect.List.of(), metadataTransformer);
     }
 

server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesService.java

@@ -47,6 +47,7 @@
 import java.util.function.Function;
 
 import static java.util.Collections.emptyList;
+import static java.util.Collections.emptyMap;
 import static org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason.NO_LONGER_ASSIGNED;
 
 /**
@@ -149,7 +150,7 @@ public ClusterState applyAliasActions(ClusterState currentState, Iterable<AliasA
                         // the context is only used for validation so it's fine to pass fake values for the shard id,
                         // but the current timestamp should be set to real value as we may use `now` in a filtered alias
                         aliasValidator.validateAliasFilter(alias, filter, indexService.newQueryShardContext(0, null,
-                            () -> System.currentTimeMillis(), null), xContentRegistry);
+                            () -> System.currentTimeMillis(), null, emptyMap()), xContentRegistry);
                     }
                 };
                 if (action.apply(newAliasValidator, metadata, index)) {

server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java

@@ -80,6 +80,7 @@
 import java.util.function.Predicate;
 import java.util.stream.Collectors;
 
+import static java.util.Collections.emptyMap;
 import static org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService.validateTimestampFieldMapping;
 import static org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason.NO_LONGER_ASSIGNED;
 
@@ -1092,7 +1093,7 @@ private static void validateCompositeTemplate(final ClusterState state,
                     new AliasValidator(),
                     // the context is only used for validation so it's fine to pass fake values for the
                     // shard id and the current timestamp
-                    xContentRegistry, tempIndexService.newQueryShardContext(0, null, () -> 0L, null));
+                    xContentRegistry, tempIndexService.newQueryShardContext(0, null, () -> 0L, null, emptyMap()));
 
                 // triggers inclusion of _timestamp field and its validation:
                 String indexName = DataStream.BACKING_INDEX_PREFIX + temporaryIndexName;

server/src/main/java/org/elasticsearch/index/IndexService.java

@@ -193,7 +193,7 @@ public IndexService(
             assert indexAnalyzers != null;
             this.mapperService = new MapperService(indexSettings, indexAnalyzers, xContentRegistry, similarityService, mapperRegistry,
                 // we parse all percolator queries as they would be parsed on shard 0
-                () -> newQueryShardContext(0, null, System::currentTimeMillis, null), idFieldDataEnabled, scriptService);
+                () -> newQueryShardContext(0, null, System::currentTimeMillis, null, emptyMap()), idFieldDataEnabled, scriptService);
             this.indexFieldData = new IndexFieldDataService(indexSettings, indicesFieldDataCache, circuitBreakerService, mapperService);
             if (indexSettings.getIndexSortConfig().hasIndexSort()) {
                 // we delay the actual creation of the sort order for this index because the mapping has not been merged yet.
@@ -588,13 +588,19 @@ public IndexSettings getIndexSettings() {
      * Passing a {@code null} {@link IndexSearcher} will return a valid context, however it won't be able to make
      * {@link IndexReader}-specific optimizations, such as rewriting containing range queries.
      */
-    public QueryShardContext newQueryShardContext(int shardId, IndexSearcher searcher, LongSupplier nowInMillis, String clusterAlias) {
+    public QueryShardContext newQueryShardContext(
+        int shardId,
+        IndexSearcher searcher,
+        LongSupplier nowInMillis,
+        String clusterAlias,
+        Map<String, Object> runtimeMappings
+    ) {
         final SearchIndexNameMatcher indexNameMatcher =
             new SearchIndexNameMatcher(index().getName(), clusterAlias, clusterService, expressionResolver);
         return new QueryShardContext(
             shardId, indexSettings, bigArrays, indexCache.bitsetFilterCache(), indexFieldData::getForField, mapperService(),
             similarityService(), scriptService, xContentRegistry, namedWriteableRegistry, client, searcher, nowInMillis, clusterAlias,
-            indexNameMatcher, allowExpensiveQueries, valuesSourceRegistry);
+            indexNameMatcher, allowExpensiveQueries, valuesSourceRegistry, runtimeMappings);
     }
 
     /**