Fix the timestamp field of a data stream to @timestamp (#59210)

Backport of #59076 to 7.x branch.

The commit makes the following changes:
* The timestamp field of a data stream definition in a composable
  index template can only be set to '@timestamp'.
* Removed custom data stream timestamp field validation and reuse the validation from `TimestampFieldMapper` and
  instead only check that the _timestamp field mapping has been defined on a backing index of a data stream.
* Moved code that injects _timestamp meta field mapping from `MetadataCreateIndexService#applyCreateIndexRequestWithV2Template58956(...)` method
  to `MetadataIndexTemplateService#collectMappings(...)` method.
* Fixed a bug (#58956) that cases timestamp field validation to be performed
  for each template and instead of the final mappings that is created.
* only apply _timestamp meta field if index is created as part of a data stream or data stream rollover,
this fixes a docs test, where a regular index creation matches (logs-*) with a template with a data stream definition.

Relates to #58642
Relates to #53100
Closes #58956
Closes #58583

Author: martijnvg

client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetComposableIndexTemplatesResponseTests.java

@@ -86,7 +86,7 @@ private static ComposableIndexTemplate randomIndexTemplate() {
         Long priority = randomBoolean() ? null : randomNonNegativeLong();
         Long version = randomBoolean() ? null : randomNonNegativeLong();
         if (randomBoolean()) {
-            dataStreamTemplate = new ComposableIndexTemplate.DataStreamTemplate(randomAlphaOfLength(8));
+            dataStreamTemplate = new ComposableIndexTemplate.DataStreamTemplate("@timestamp");
         }
         return new ComposableIndexTemplate(patterns, randomTemplate(), composedOf, priority, version, meta, dataStreamTemplate);
     }

client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetDataStreamResponseTests.java

@@ -55,7 +55,7 @@ private static DataStreamInfo randomInstance() {
         long generation = indices.size() + randomLongBetween(1, 128);
         String dataStreamName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
         indices.add(new Index(getDefaultBackingIndexName(dataStreamName, generation), UUIDs.randomBase64UUID(random())));
-        DataStream dataStream = new DataStream(dataStreamName, createTimestampField(randomAlphaOfLength(10)), indices, generation);
+        DataStream dataStream = new DataStream(dataStreamName, createTimestampField("@timestamp"), indices, generation);
         return new DataStreamInfo(dataStream, ClusterHealthStatus.YELLOW, randomAlphaOfLengthBetween(2, 10),
             randomAlphaOfLengthBetween(2, 10));
     }

docs/reference/indices/rollover-index.asciidoc

@@ -235,14 +235,14 @@ PUT _index_template/template
   "template": {
     "mappings": {
       "properties": {
-        "date": {
+        "@timestamp": {
           "type": "date"
         }
       }
     }
   },
   "data_stream": {
-    "timestamp_field": "date"
+    "timestamp_field": "@timestamp"
   }
 }
 -----------------------------------
@@ -257,15 +257,15 @@ PUT /_data_stream/my-data-stream <1>
 POST /my-data-stream/_rollover <2>
 {
   "conditions" : {
-  "max_age": "7d",
-  "max_docs": 1000,
-  "max_size": "5gb"
+    "max_age": "7d",
+    "max_docs": 1000,
+    "max_size": "5gb"
   }
 }
 --------------------------------------------------
 // TEST[continued]
 // TEST[setup:huge_twitter]
-// TEST[s/# Add > 1000 documents to my-data-stream/POST _reindex?refresh\n{"source":{"index":"twitter"},"dest":{"index":"my-data-stream","op_type":"create"}}/]
+// TEST[s/# Add > 1000 documents to my-data-stream/POST _reindex?refresh\n{"source":{"index":"twitter"},"dest":{"index":"my-data-stream","op_type":"create"},"script":{"source":"ctx._source.put('@timestamp',ctx._source.remove('date'))"}}/]
 <1> Creates a data stream called `my-data-stream` with one initial backing index
 named `my-data-stream-000001`.
 <2> This request creates a new backing index, `my-data-stream-000002`, and adds
@@ -307,7 +307,7 @@ The API returns the following response:
 [source,console]
 -----------------------------------
 DELETE /_data_stream/my-data-stream
-DELETE /_index_template/*
+DELETE /_index_template/template
 -----------------------------------
 // TEST[continued]
 ////

modules/reindex/src/test/resources/rest-api-spec/test/reindex/96_data_streams.yml

@@ -10,11 +10,11 @@ setup:
         body:
           index_patterns: logs-*
           data_stream:
-            timestamp_field: timestamp
+            timestamp_field: '@timestamp'
           template:
             mappings:
               properties:
-                timestamp:
+                '@timestamp':
                   type: date
 
 ---
@@ -36,7 +36,7 @@ teardown:
         refresh: true
         body:
           foo: bar
-          timestamp: '2020-12-12'
+          '@timestamp': '2020-12-12'
 
   - do:
       reindex:
@@ -69,7 +69,7 @@ teardown:
         refresh: true
         body:
           foo: bar
-          timestamp: '2020-12-12'
+          '@timestamp': '2020-12-12'
 
   - do:
       reindex:
@@ -102,7 +102,7 @@ teardown:
         refresh: true
         body:
           foo: bar
-          timestamp: '2020-12-12'
+          '@timestamp': '2020-12-12'
 
   - do:
       reindex:

qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/100_resolve_index.yml

@@ -51,4 +51,4 @@
   - match: {data_streams.1.name: my_remote_cluster:simple-data-stream2}
   - match: {data_streams.1.backing_indices.0: .ds-simple-data-stream2-000001}
   - match: {data_streams.1.backing_indices.1: .ds-simple-data-stream2-000002}
-  - match: {data_streams.1.timestamp_field: "@timestamp2"}
+  - match: {data_streams.1.timestamp_field: "@timestamp"}

qa/multi-cluster-search/src/test/resources/rest-api-spec/test/remote_cluster/10_basic.yml

@@ -27,10 +27,10 @@
           template:
             mappings:
               properties:
-                '@timestamp2':
+                '@timestamp':
                   type: date
           data_stream:
-            timestamp_field: '@timestamp2'
+            timestamp_field: '@timestamp'
 
   - do:
       indices.create_data_stream:

rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/10_basic.yml

@@ -27,10 +27,10 @@ setup:
           template:
             mappings:
               properties:
-                '@timestamp2':
+                '@timestamp':
                   type: date
           data_stream:
-            timestamp_field: '@timestamp2'
+            timestamp_field: '@timestamp'
 
 ---
 "Create data stream":
@@ -59,7 +59,7 @@ setup:
   - match: { data_streams.0.status: 'GREEN' }
   - match: { data_streams.0.template: 'my-template1' }
   - match: { data_streams.1.name: simple-data-stream2 }
-  - match: { data_streams.1.timestamp_field.name: '@timestamp2' }
+  - match: { data_streams.1.timestamp_field.name: '@timestamp' }
   - match: { data_streams.0.generation: 1 }
   - length: { data_streams.1.indices: 1 }
   - match: { data_streams.1.indices.0.index_name: '.ds-simple-data-stream2-000001' }
@@ -129,31 +129,26 @@ setup:
       indices.get_data_stream: {}
   - match: { data_streams.0.name: simple-data-stream1 }
   - match: { data_streams.0.timestamp_field.name: '@timestamp' }
-  - match: { data_streams.0.timestamp_field.mapping: {type: date} }
   - match: { data_streams.0.generation: 1 }
   - match: { data_streams.1.name: simple-data-stream2 }
-  - match: { data_streams.1.timestamp_field.name: '@timestamp2' }
-  - match: { data_streams.1.timestamp_field.mapping: {type: date} }
+  - match: { data_streams.1.timestamp_field.name: '@timestamp' }
   - match: { data_streams.1.generation: 1 }
 
   - do:
       indices.get_data_stream:
         name: simple-data-stream1
   - match: { data_streams.0.name: simple-data-stream1 }
   - match: { data_streams.0.timestamp_field.name: '@timestamp' }
-  - match: { data_streams.0.timestamp_field.mapping: {type: date} }
   - match: { data_streams.0.generation: 1 }
 
   - do:
       indices.get_data_stream:
         name: simple-data-stream*
   - match: { data_streams.0.name: simple-data-stream1 }
   - match: { data_streams.0.timestamp_field.name: '@timestamp' }
-  - match: { data_streams.0.timestamp_field.mapping: {type: date} }
   - match: { data_streams.0.generation: 1 }
   - match: { data_streams.1.name: simple-data-stream2 }
-  - match: { data_streams.1.timestamp_field.name: '@timestamp2' }
-  - match: { data_streams.1.timestamp_field.mapping: {type: date} }
+  - match: { data_streams.1.timestamp_field.name: '@timestamp' }
   - match: { data_streams.1.generation: 1 }
 
   - do:
@@ -240,35 +235,35 @@ setup:
           template:
             mappings:
               properties:
-                'timestamp':
+                '@timestamp':
                   type: date
           data_stream:
-            timestamp_field: timestamp
+            timestamp_field: '@timestamp'
 
   - do:
       index:
         index:  logs-foobar
-        body:   { timestamp: '2020-12-12' }
+        body:   { '@timestamp': '2020-12-12' }
   - match: { _index: .ds-logs-foobar-000001 }
 
   - do:
       catch: bad_request
       index:
         index:  .ds-logs-foobar-000001
-        body:   { timestamp: '2020-12-12' }
+        body:   { '@timestamp': '2020-12-12' }
 
   - do:
       bulk:
         body:
           - create:
               _index: .ds-logs-foobar-000001
-          - timestamp: '2020-12-12'
+          - '@timestamp': '2020-12-12'
           - index:
               _index: .ds-logs-foobar-000001
-          - timestamp: '2020-12-12'
+          - '@timestamp': '2020-12-12'
           - create:
               _index: logs-foobar
-          - timestamp: '2020-12-12'
+          - '@timestamp': '2020-12-12'
   - match: { errors: true }
   - match: { items.0.create.status: 400 }
   - match: { items.0.create.error.type: illegal_argument_exception }
@@ -301,10 +296,10 @@ setup:
           template:
             mappings:
               properties:
-                'timestamp':
+                '@timestamp':
                   type: date
           data_stream:
-            timestamp_field: timestamp
+            timestamp_field: '@timestamp'
 
   - do:
       catch: bad_request
@@ -320,19 +315,19 @@ setup:
           - foo: bar
           - create:
               _index: logs-foobar
-          - timestamp: '2020-12-12'
+          - '@timestamp': '2020-12-12'
           - create:
               _index: logs-foobar
-          - timestamp: ['2020-12-12', '2022-12-12']
+          - '@timestamp': ['2020-12-12', '2022-12-12']
   - match: { errors: true }
   - match: { items.0.create.status: 400 }
   - match: { items.0.create.error.caused_by.type: illegal_argument_exception }
-  - match: { items.0.create.error.caused_by.reason: "data stream timestamp field [timestamp] is missing" }
+  - match: { items.0.create.error.caused_by.reason: "data stream timestamp field [@timestamp] is missing" }
   - match: { items.1.create.result: created }
   - match: { items.1.create._index: .ds-logs-foobar-000001 }
   - match: { items.2.create.status: 400 }
   - match: { items.2.create.error.caused_by.type: illegal_argument_exception }
-  - match: { items.2.create.error.caused_by.reason: "data stream timestamp field [timestamp] encountered multiple values" }
+  - match: { items.2.create.error.caused_by.reason: "data stream timestamp field [@timestamp] encountered multiple values" }
 
   - do:
       indices.delete_data_stream:

rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/30_auto_create_data_stream.yml

@@ -13,22 +13,22 @@
         body:
           index_patterns: logs-*
           data_stream:
-            timestamp_field: timestamp
+            timestamp_field: '@timestamp'
           template:
             settings:
               number_of_shards:   1
               number_of_replicas: 0
             mappings:
               properties:
-                timestamp:
+                '@timestamp':
                   type: date
 
   - do:
       index:
         index:  logs-foobar
         refresh: true
         body:
-          'timestamp': '2020-12-12'
+          '@timestamp': '2020-12-12'
           foo: bar
 
   - do:
@@ -43,7 +43,7 @@
       indices.get_data_stream:
         name: logs-foobar
   - match: { data_streams.0.name: logs-foobar }
-  - match: { data_streams.0.timestamp_field.name: 'timestamp' }
+  - match: { data_streams.0.timestamp_field.name: '@timestamp' }
   - length: { data_streams.0.indices: 1 }
   - match: { data_streams.0.indices.0.index_name: '.ds-logs-foobar-000001' }
 

rest-api-spec/src/main/resources/rest-api-spec/test/indices.resolve_index/10_basic_resolve_index.yml

@@ -105,7 +105,7 @@ setup:
   - match: {data_streams.1.name: simple-data-stream2}
   - match: {data_streams.1.backing_indices.0: .ds-simple-data-stream2-000001}
   - match: {data_streams.1.backing_indices.1: .ds-simple-data-stream2-000002}
-  - match: {data_streams.1.timestamp_field: "@timestamp2"}
+  - match: {data_streams.1.timestamp_field: "@timestamp"}
 
   - do:
       indices.delete_data_stream:
@@ -165,7 +165,7 @@ setup:
   - match: {data_streams.1.name: simple-data-stream2}
   - match: {data_streams.1.backing_indices.0: .ds-simple-data-stream2-000001}
   - match: {data_streams.1.backing_indices.1: .ds-simple-data-stream2-000002}
-  - match: {data_streams.1.timestamp_field: "@timestamp2"}
+  - match: {data_streams.1.timestamp_field: "@timestamp"}
 
   - do:
       indices.delete_data_stream: