Skip to content

Commit 001b3fb

Browse files
martijnvgmartijnvg
authored
Add data stream timestamp validation via metadata field mapper (#58582)
This commit adds a new metadata field mapper that validates, that a document has exactly a single timestamp value in the data stream timestamp field and that the timestamp field mapping only has `type`, `meta` or `format` attributes configured. Other attributes can affect the guarantee that an index with this meta field mapper has a useable timestamp field. The MetadataCreateIndexService inserts a data stream timestamp field mapper whenever a new backing index of a data stream is created. Relates to #53100
1 parent 2717697 commit 001b3fb

File tree

24 files changed

+662
-79
lines changed

24 files changed

+662
-79
lines changed

‎docs/reference/indices/rollover-index.asciidoc

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -233,14 +233,14 @@ PUT _index_template/template
233233
"template": {
234234
"mappings": {
235235
"properties": {
236-
"@timestamp": {
236+
"date": {
237237
"type": "date"
238238
}
239239
}
240240
}
241241
},
242242
"data_stream": {
243-
"timestamp_field": "@timestamp"
243+
"timestamp_field": "date"
244244
}
245245
}
246246
-----------------------------------

‎modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/90_data_streams.yml

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,9 @@
3030
index: simple-data-stream1
3131
id: 1
3232
op_type: create
33-
body: { "text": "test" }
33+
body:
34+
foo: bar
35+
'@timestamp': '2020-12-12'
3436

3537
- do:
3638
indices.refresh:

‎modules/reindex/src/test/resources/rest-api-spec/test/reindex/96_data_streams.yml

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,9 @@ teardown:
3434
index:
3535
index: logs-foobar
3636
refresh: true
37-
body: { foo: bar }
37+
body:
38+
foo: bar
39+
timestamp: '2020-12-12'
3840

3941
- do:
4042
reindex:
@@ -65,7 +67,9 @@ teardown:
6567
index:
6668
index: old-logs-index
6769
refresh: true
68-
body: { foo: bar }
70+
body:
71+
foo: bar
72+
timestamp: '2020-12-12'
6973

7074
- do:
7175
reindex:
@@ -96,7 +100,9 @@ teardown:
96100
index:
97101
index: logs-foobar
98102
refresh: true
99-
body: { foo: bar }
103+
body:
104+
foo: bar
105+
timestamp: '2020-12-12'
100106

101107
- do:
102108
reindex:

‎modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/90_data_streams.yml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@
3030
index: simple-data-stream1
3131
id: 1
3232
op_type: create
33-
body: { "number": 4 }
33+
body: { "number": 4, '@timestamp': '2020-12-12' }
3434

3535
# rollover data stream to create new backing index
3636
- do:
@@ -47,7 +47,7 @@
4747
index: simple-data-stream1
4848
id: 2
4949
op_type: create
50-
body: { "number": 1 }
50+
body: { "number": 1, '@timestamp': '2020-12-12' }
5151

5252
# rollover data stream to create another new backing index
5353
- do:
@@ -64,7 +64,7 @@
6464
index: simple-data-stream1
6565
id: 3
6666
op_type: create
67-
body: { "number": 5 }
67+
body: { "number": 5, '@timestamp': '2020-12-12' }
6868

6969
- do:
7070
indices.refresh:

‎rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/10_basic.yml

Lines changed: 63 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,9 @@ setup:
6363
- do:
6464
index:
6565
index: simple-data-stream1
66-
body: { foo: bar }
66+
body:
67+
'@timestamp': '2020-12-12'
68+
foo: bar
6769

6870
- do:
6971
indices.refresh:
@@ -241,27 +243,27 @@ setup:
241243
- do:
242244
index:
243245
index: logs-foobar
244-
body: { foo: bar }
246+
body: { timestamp: '2020-12-12' }
245247
- match: { _index: .ds-logs-foobar-000001 }
246248

247249
- do:
248250
catch: bad_request
249251
index:
250252
index: .ds-logs-foobar-000001
251-
body: { foo: bar }
253+
body: { timestamp: '2020-12-12' }
252254

253255
- do:
254256
bulk:
255257
body:
256258
- create:
257259
_index: .ds-logs-foobar-000001
258-
- foo: bar
260+
- timestamp: '2020-12-12'
259261
- index:
260262
_index: .ds-logs-foobar-000001
261-
- foo: bar
263+
- timestamp: '2020-12-12'
262264
- create:
263265
_index: logs-foobar
264-
- foo: bar
266+
- timestamp: '2020-12-12'
265267
- match: { errors: true }
266268
- match: { items.0.create.status: 400 }
267269
- match: { items.0.create.error.type: illegal_argument_exception }
@@ -276,3 +278,58 @@ setup:
276278
indices.delete_data_stream:
277279
name: logs-foobar
278280
- is_true: acknowledged
281+
282+
---
283+
"Indexing a document into a data stream without a timestamp field":
284+
- skip:
285+
version: " - 7.9.99"
286+
reason: "enable in 7.9+ when backported"
287+
features: allowed_warnings
288+
289+
- do:
290+
allowed_warnings:
291+
- "index template [generic_logs_template] has index patterns [logs-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [generic_logs_template] will take precedence during new index creation"
292+
indices.put_index_template:
293+
name: generic_logs_template
294+
body:
295+
index_patterns: logs-*
296+
template:
297+
mappings:
298+
properties:
299+
'timestamp':
300+
type: date
301+
data_stream:
302+
timestamp_field: timestamp
303+
304+
- do:
305+
catch: bad_request
306+
index:
307+
index: logs-foobar
308+
body: { foo: bar }
309+
310+
- do:
311+
bulk:
312+
body:
313+
- create:
314+
_index: logs-foobar
315+
- foo: bar
316+
- create:
317+
_index: logs-foobar
318+
- timestamp: '2020-12-12'
319+
- create:
320+
_index: logs-foobar
321+
- timestamp: ['2020-12-12', '2022-12-12']
322+
- match: { errors: true }
323+
- match: { items.0.create.status: 400 }
324+
- match: { items.0.create.error.caused_by.type: illegal_argument_exception }
325+
- match: { items.0.create.error.caused_by.reason: "data stream timestamp field [timestamp] is missing" }
326+
- match: { items.1.create.result: created }
327+
- match: { items.1.create._index: .ds-logs-foobar-000001 }
328+
- match: { items.2.create.status: 400 }
329+
- match: { items.2.create.error.caused_by.type: illegal_argument_exception }
330+
- match: { items.2.create.error.caused_by.reason: "data stream timestamp field [timestamp] encountered multiple values" }
331+
332+
- do:
333+
indices.delete_data_stream:
334+
name: logs-foobar
335+
- is_true: acknowledged

‎rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/20_unsupported_apis.yml

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,9 @@
2929
index:
3030
index: logs-foobar
3131
refresh: true
32-
body: { foo: bar }
32+
body:
33+
'@timestamp': '2020-12-12'
34+
foo: bar
3335
- match: {_index: .ds-logs-foobar-000001}
3436

3537
- do:

‎rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/30_auto_create_data_stream.yml

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,9 @@
2727
index:
2828
index: logs-foobar
2929
refresh: true
30-
body: { foo: bar }
30+
body:
31+
'timestamp': '2020-12-12'
32+
foo: bar
3133

3234
- do:
3335
search:

‎server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -233,28 +233,28 @@ public void testMixedAutoCreate() throws Exception {
233233
client().execute(PutComposableIndexTemplateAction.INSTANCE, createTemplateRequest).actionGet();
234234

235235
BulkRequest bulkRequest = new BulkRequest();
236-
bulkRequest.add(new IndexRequest("logs-foobar").opType(CREATE).source("{}", XContentType.JSON));
237-
bulkRequest.add(new IndexRequest("logs-foobaz").opType(CREATE).source("{}", XContentType.JSON));
238-
bulkRequest.add(new IndexRequest("logs-barbaz").opType(CREATE).source("{}", XContentType.JSON));
239-
bulkRequest.add(new IndexRequest("logs-barfoo").opType(CREATE).source("{}", XContentType.JSON));
236+
bulkRequest.add(new IndexRequest("logs-foobar").opType(CREATE).source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON));
237+
bulkRequest.add(new IndexRequest("logs-foobaz").opType(CREATE).source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON));
238+
bulkRequest.add(new IndexRequest("logs-barbaz").opType(CREATE).source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON));
239+
bulkRequest.add(new IndexRequest("logs-barfoo").opType(CREATE).source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON));
240240
BulkResponse bulkResponse = client().bulk(bulkRequest).actionGet();
241241
assertThat("bulk failures: " + Strings.toString(bulkResponse), bulkResponse.hasFailures(), is(false));
242242

243243
bulkRequest = new BulkRequest();
244-
bulkRequest.add(new IndexRequest("logs-foobar").opType(CREATE).source("{}", XContentType.JSON));
245-
bulkRequest.add(new IndexRequest("logs-foobaz2").opType(CREATE).source("{}", XContentType.JSON));
246-
bulkRequest.add(new IndexRequest("logs-barbaz").opType(CREATE).source("{}", XContentType.JSON));
247-
bulkRequest.add(new IndexRequest("logs-barfoo2").opType(CREATE).source("{}", XContentType.JSON));
244+
bulkRequest.add(new IndexRequest("logs-foobar").opType(CREATE).source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON));
245+
bulkRequest.add(new IndexRequest("logs-foobaz2").opType(CREATE).source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON));
246+
bulkRequest.add(new IndexRequest("logs-barbaz").opType(CREATE).source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON));
247+
bulkRequest.add(new IndexRequest("logs-barfoo2").opType(CREATE).source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON));
248248
bulkResponse = client().bulk(bulkRequest).actionGet();
249249
assertThat("bulk failures: " + Strings.toString(bulkResponse), bulkResponse.hasFailures(), is(false));
250250

251251
bulkRequest = new BulkRequest();
252-
bulkRequest.add(new IndexRequest("logs-foobar").opType(CREATE).source("{}", XContentType.JSON));
253-
bulkRequest.add(new IndexRequest("logs-foobaz2").opType(CREATE).source("{}", XContentType.JSON));
254-
bulkRequest.add(new IndexRequest("logs-foobaz3").opType(CREATE).source("{}", XContentType.JSON));
255-
bulkRequest.add(new IndexRequest("logs-barbaz").opType(CREATE).source("{}", XContentType.JSON));
256-
bulkRequest.add(new IndexRequest("logs-barfoo2").opType(CREATE).source("{}", XContentType.JSON));
257-
bulkRequest.add(new IndexRequest("logs-barfoo3").opType(CREATE).source("{}", XContentType.JSON));
252+
bulkRequest.add(new IndexRequest("logs-foobar").opType(CREATE).source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON));
253+
bulkRequest.add(new IndexRequest("logs-foobaz2").opType(CREATE).source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON));
254+
bulkRequest.add(new IndexRequest("logs-foobaz3").opType(CREATE).source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON));
255+
bulkRequest.add(new IndexRequest("logs-barbaz").opType(CREATE).source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON));
256+
bulkRequest.add(new IndexRequest("logs-barfoo2").opType(CREATE).source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON));
257+
bulkRequest.add(new IndexRequest("logs-barfoo3").opType(CREATE).source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON));
258258
bulkResponse = client().bulk(bulkRequest).actionGet();
259259
assertThat("bulk failures: " + Strings.toString(bulkResponse), bulkResponse.hasFailures(), is(false));
260260

0 commit comments

Comments
 (0)