only apply _timestamp meta field if index is created as part of a data stream or data stream rollover,

this fixes a docs test, where a regular index creation matches (logs-*) with a template with a data stream definition.

Author: martijnvg

docs/reference/indices/rollover-index.asciidoc

@@ -233,14 +233,14 @@ PUT _index_template/template
   "template": {
     "mappings": {
       "properties": {
-        "date": {
+        "@timestamp": {
           "type": "date"
         }
       }
     }
   },
   "data_stream": {
-    "timestamp_field": "date"
+    "timestamp_field": "@timestamp"
   }
 }
 -----------------------------------
@@ -255,15 +255,15 @@ PUT /_data_stream/my-data-stream <1>
 POST /my-data-stream/_rollover <2>
 {
   "conditions" : {
-  "max_age": "7d",
-  "max_docs": 1000,
-  "max_size": "5gb"
+    "max_age": "7d",
+    "max_docs": 1000,
+    "max_size": "5gb"
   }
 }
 --------------------------------------------------
 // TEST[continued]
 // TEST[setup:huge_twitter]
-// TEST[s/# Add > 1000 documents to my-data-stream/POST _reindex?refresh\n{"source":{"index":"twitter"},"dest":{"index":"my-data-stream","op_type":"create"}}/]
+// TEST[s/# Add > 1000 documents to my-data-stream/POST _reindex?refresh\n{"source":{"index":"twitter"},"dest":{"index":"my-data-stream","op_type":"create"},"script":{"source":"ctx._source.put('@timestamp',ctx._source.remove('date'))"}}/]
 <1> Creates a data stream called `my-data-stream` with one initial backing index
 named `my-data-stream-000001`.
 <2> This request creates a new backing index, `my-data-stream-000002`, and adds
@@ -305,7 +305,7 @@ The API returns the following response:
 [source,console]
 -----------------------------------
 DELETE /_data_stream/my-data-stream
-DELETE /_index_template/*
+DELETE /_index_template/template
 -----------------------------------
 // TEST[continued]
 ////

server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java

@@ -200,7 +200,7 @@ public static Template resolveTemplate(final String matchingTemplate, final Stri
 
         // empty request mapping as the user can't specify any explicit mappings via the simulate api
         List<Map<String, Object>> mappings = MetadataCreateIndexService.collectV2Mappings(
-            "{}", simulatedState, matchingTemplate, xContentRegistry);
+            "{}", simulatedState, matchingTemplate, xContentRegistry, true);
 
         CompressedXContent mergedMapping = indicesService.<CompressedXContent, Exception>withTempIndexService(indexMetadata,
             tempIndexService -> {

server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java

@@ -489,7 +489,8 @@ private ClusterState applyCreateIndexRequestWithV2Template(final ClusterState cu
                                                                                     throws Exception {
         logger.debug("applying create index request using composable template [{}]", templateName);
 
-        final List<Map<String, Object>> mappings = collectV2Mappings(request.mappings(), currentState, templateName, xContentRegistry);
+        final List<Map<String, Object>> mappings =
+            collectV2Mappings(request.mappings(), currentState, templateName, xContentRegistry, request.dataStreamName() != null);
         final Settings aggregatedIndexSettings =
             aggregateIndexSettings(currentState, request,
                 MetadataIndexTemplateService.resolveSettings(currentState.metadata(), templateName),
@@ -509,10 +510,11 @@ private ClusterState applyCreateIndexRequestWithV2Template(final ClusterState cu
     public static List<Map<String, Object>> collectV2Mappings(final String requestMappings,
                                                               final ClusterState currentState,
                                                               final String templateName,
-                                                              final NamedXContentRegistry xContentRegistry) throws Exception {
+                                                              final NamedXContentRegistry xContentRegistry,
+                                                              final boolean createDataStream) throws Exception {
         List<Map<String, Object>> result = new ArrayList<>();
 
-        List<CompressedXContent> templateMappings = MetadataIndexTemplateService.collectMappings(currentState, templateName);
+        List<CompressedXContent> templateMappings = MetadataIndexTemplateService.collectMappings(currentState, templateName, createDataStream);
         for (CompressedXContent templateMapping : templateMappings) {
             Map<String, Object> parsedTemplateMapping = MapperService.parseMapping(xContentRegistry, templateMapping.string());
             result.add(parsedTemplateMapping);

server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java

@@ -887,7 +887,9 @@ public static String findV2Template(Metadata metadata, String indexName, boolean
     /**
      * Collect the given v2 template into an ordered list of mappings.
      */
-    public static List<CompressedXContent> collectMappings(final ClusterState state, final String templateName) {
+    public static List<CompressedXContent> collectMappings(final ClusterState state,
+                                                           final String templateName,
+                                                           final boolean createDataStream) {
         final ComposableIndexTemplate template = state.metadata().templatesV2().get(templateName);
         assert template != null : "attempted to resolve mappings for a template [" + templateName +
             "] that did not exist in the cluster state";
@@ -909,17 +911,19 @@ public static List<CompressedXContent> collectMappings(final ClusterState state,
             .ifPresent(mappings::add);
 
         // Add the mapping of a data stream's timestamp field if available, since it has the highest precedence:
-        Optional.ofNullable(template.getDataStreamTemplate())
-            .map(ComposableIndexTemplate.DataStreamTemplate::getDataSteamMappingSnippet)
-            .map(mapping -> {
-                try (XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent())) {
-                    builder.value(mapping);
-                    return new CompressedXContent(BytesReference.bytes(builder));
-                } catch (IOException e) {
-                    throw new UncheckedIOException(e);
-                }
-            })
-            .ifPresent(mappings::add);
+        if (createDataStream) {
+            Optional.ofNullable(template.getDataStreamTemplate())
+                .map(ComposableIndexTemplate.DataStreamTemplate::getDataSteamMappingSnippet)
+                .map(mapping -> {
+                    try (XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent())) {
+                        builder.value(mapping);
+                        return new CompressedXContent(BytesReference.bytes(builder));
+                    } catch (IOException e) {
+                        throw new UncheckedIOException(e);
+                    }
+                })
+                .ifPresent(mappings::add);
+        }
         return Collections.unmodifiableList(mappings);
     }
 
@@ -1066,7 +1070,7 @@ private static void validateCompositeTemplate(final ClusterState state,
                     xContentRegistry, tempIndexService.newQueryShardContext(0, null, () -> 0L, null));
 
                 // Parse mappings to ensure they are valid after being composed
-                List<CompressedXContent> mappings = collectMappings(stateWithIndex, templateName);
+                List<CompressedXContent> mappings = collectMappings(stateWithIndex, templateName, true);
                 try {
                     MapperService mapperService = tempIndexService.mapperService();
                     for (CompressedXContent mapping : mappings) {

server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java

@@ -706,7 +706,7 @@ public void testResolveConflictingMappings() throws Exception {
             List.of("ct_low", "ct_high"), 0L, 1L, null, null);
         state = service.addIndexTemplateV2(state, true, "my-template", it);
 
-        List<CompressedXContent> mappings = MetadataIndexTemplateService.collectMappings(state, "my-template");
+        List<CompressedXContent> mappings = MetadataIndexTemplateService.collectMappings(state, "my-template", false);
 
         assertNotNull(mappings);
         assertThat(mappings.size(), equalTo(3));
@@ -769,7 +769,7 @@ public void testResolveMappings() throws Exception {
             List.of("ct_low", "ct_high"), 0L, 1L, null, null);
         state = service.addIndexTemplateV2(state, true, "my-template", it);
 
-        List<CompressedXContent> mappings = MetadataIndexTemplateService.collectMappings(state, "my-template");
+        List<CompressedXContent> mappings = MetadataIndexTemplateService.collectMappings(state, "my-template", false);
 
         assertNotNull(mappings);
         assertThat(mappings.size(), equalTo(3));