Skip to content

Commit 6a86134

Browse files
jpountzruflin
jpountz
and
ruflin
authored
Better out-of-the-box mappings for logs, metrics and synthetics (#72703)
One of the problems we have today with the default templates is that ip addresses and message fields are not mapped correct. Auto detection of ip addresses would be great: #64400 But in the meantime, we could also match on the naming convention that all `*.ip` fields are of type ip address. Co-authored-by: Nicolas Ruflin <[email protected]>
1 parent b187c75 commit 6a86134

File tree

10 files changed

+195
-125
lines changed

10 files changed

+195
-125
lines changed

‎x-pack/plugin/core/src/main/resources/data-streams-mappings.json

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
{
2+
"template": {
3+
"mappings": {
4+
"dynamic_templates": [
5+
{
6+
"match_ip": {
7+
"match_mapping_type": "string",
8+
"match": "ip",
9+
"mapping": {
10+
"type": "ip"
11+
}
12+
}
13+
},
14+
{
15+
"match_message": {
16+
"match_mapping_type": "string",
17+
"match": "message",
18+
"mapping": {
19+
"type": "match_only_text"
20+
}
21+
}
22+
},
23+
{
24+
"strings_as_keyword": {
25+
"mapping": {
26+
"ignore_above": 1024,
27+
"type": "keyword"
28+
},
29+
"match_mapping_type": "string"
30+
}
31+
}
32+
],
33+
"date_detection": false,
34+
"properties": {
35+
"@timestamp": {
36+
"type": "date"
37+
},
38+
"data_stream": {
39+
"properties": {
40+
"dataset": {
41+
"type": "constant_keyword"
42+
},
43+
"namespace": {
44+
"type": "constant_keyword"
45+
}
46+
}
47+
},
48+
"ecs": {
49+
"properties": {
50+
"version": {
51+
"ignore_above": 1024,
52+
"type": "keyword"
53+
}
54+
}
55+
},
56+
"host": {
57+
"type": "object"
58+
}
59+
}
60+
}
61+
},
62+
"_meta": {
63+
"description": "general mapping conventions for data streams",
64+
"managed": true
65+
},
66+
"version": ${xpack.stack.template.version}
67+
}

‎x-pack/plugin/core/src/main/resources/logs-mappings.json

Lines changed: 0 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -1,53 +1,14 @@
11
{
22
"template": {
33
"mappings": {
4-
"dynamic_templates": [
5-
{
6-
"strings_as_keyword": {
7-
"mapping": {
8-
"ignore_above": 1024,
9-
"type": "keyword"
10-
},
11-
"match_mapping_type": "string"
12-
}
13-
}
14-
],
15-
"date_detection": false,
164
"properties": {
17-
"@timestamp": {
18-
"type": "date"
19-
},
205
"data_stream": {
216
"properties": {
227
"type": {
238
"type": "constant_keyword",
249
"value": "logs"
25-
},
26-
"dataset": {
27-
"type": "constant_keyword"
28-
},
29-
"namespace": {
30-
"type": "constant_keyword"
31-
}
32-
}
33-
},
34-
"ecs": {
35-
"properties": {
36-
"version": {
37-
"ignore_above": 1024,
38-
"type": "keyword"
39-
}
40-
}
41-
},
42-
"host": {
43-
"properties": {
44-
"ip": {
45-
"type": "ip"
4610
}
4711
}
48-
},
49-
"message": {
50-
"type": "text"
5112
}
5213
}
5314
}

‎x-pack/plugin/core/src/main/resources/logs-template.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
"data_stream": {},
55
"composed_of": [
66
"logs-mappings",
7+
"data-streams-mappings",
78
"logs-settings"
89
],
910
"allow_auto_create": true,

‎x-pack/plugin/core/src/main/resources/metrics-mappings.json

Lines changed: 0 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -1,48 +1,12 @@
11
{
22
"template": {
33
"mappings": {
4-
"dynamic_templates": [
5-
{
6-
"strings_as_keyword": {
7-
"mapping": {
8-
"ignore_above": 1024,
9-
"type": "keyword"
10-
},
11-
"match_mapping_type": "string"
12-
}
13-
}
14-
],
15-
"date_detection": false,
164
"properties": {
17-
"@timestamp": {
18-
"type": "date"
19-
},
205
"data_stream": {
216
"properties": {
227
"type": {
238
"type": "constant_keyword",
249
"value": "metrics"
25-
},
26-
"dataset": {
27-
"type": "constant_keyword"
28-
},
29-
"namespace": {
30-
"type": "constant_keyword"
31-
}
32-
}
33-
},
34-
"ecs": {
35-
"properties": {
36-
"version": {
37-
"ignore_above": 1024,
38-
"type": "keyword"
39-
}
40-
}
41-
},
42-
"host": {
43-
"properties": {
44-
"ip": {
45-
"type": "ip"
4610
}
4711
}
4812
}

‎x-pack/plugin/core/src/main/resources/metrics-template.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
"data_stream": {},
55
"composed_of": [
66
"metrics-mappings",
7+
"data-streams-mappings",
78
"metrics-settings"
89
],
910
"allow_auto_create": true,

‎x-pack/plugin/core/src/main/resources/synthetics-mappings.json

Lines changed: 0 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -1,55 +1,12 @@
11
{
22
"template": {
33
"mappings": {
4-
"dynamic_templates": [
5-
{
6-
"strings_as_keyword": {
7-
"mapping": {
8-
"ignore_above": 1024,
9-
"type": "keyword"
10-
},
11-
"match_mapping_type": "string"
12-
}
13-
}
14-
],
15-
"date_detection": false,
164
"properties": {
17-
"@timestamp": {
18-
"type": "date"
19-
},
205
"data_stream": {
216
"properties": {
227
"type": {
238
"type": "constant_keyword",
249
"value": "synthetics"
25-
},
26-
"dataset": {
27-
"type": "constant_keyword"
28-
},
29-
"namespace": {
30-
"type": "constant_keyword"
31-
}
32-
}
33-
},
34-
"ecs": {
35-
"properties": {
36-
"version": {
37-
"ignore_above": 1024,
38-
"type": "keyword"
39-
}
40-
}
41-
},
42-
"host": {
43-
"properties": {
44-
"ip": {
45-
"type": "ip"
46-
}
47-
}
48-
},
49-
"observer": {
50-
"properties": {
51-
"ip": {
52-
"type": "ip"
5310
}
5411
}
5512
}

‎x-pack/plugin/core/src/main/resources/synthetics-template.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
"data_stream": {},
55
"composed_of": [
66
"synthetics-mappings",
7+
"data-streams-mappings",
78
"synthetics-settings"
89
],
910
"allow_auto_create": true,

‎x-pack/plugin/stack/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/stack/10_basic.yml

Lines changed: 106 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,10 @@ setup:
1414
ilm.get_lifecycle:
1515
policy: "metrics"
1616

17+
- do:
18+
cluster.get_component_template:
19+
name: data-streams-mappings
20+
1721
- do:
1822
cluster.get_component_template:
1923
name: logs-mappings
@@ -46,6 +50,11 @@ setup:
4650
body:
4751
"@timestamp": "2020-01-01"
4852
message: "test-log-message"
53+
source.ip: "10.1.2.3"
54+
log.file.path: "/var/log/web/access.log"
55+
data_stream.type: "logs"
56+
data_stream.dataset: "foo"
57+
data_stream.namespace: "bar"
4958

5059
- do:
5160
indices.get_data_stream:
@@ -66,7 +75,16 @@ setup:
6675
- is_true: .$idx0name.settings
6776
- is_true: .$idx0name.mappings
6877
- match: { .$idx0name.settings.index.lifecycle.name: "logs" }
78+
- match: { .$idx0name.mappings.properties.data_stream.properties.type.type: "constant_keyword" }
79+
- match: { .$idx0name.mappings.properties.data_stream.properties.type.value: "logs" }
80+
- match: { .$idx0name.mappings.properties.data_stream.properties.dataset.type: "constant_keyword" }
81+
- match: { .$idx0name.mappings.properties.data_stream.properties.dataset.value: "foo" }
82+
- match: { .$idx0name.mappings.properties.data_stream.properties.namespace.type: "constant_keyword" }
83+
- match: { .$idx0name.mappings.properties.data_stream.properties.namespace.value: "bar" }
6984
- is_true: .$idx0name.mappings.properties.message
85+
- match: { .$idx0name.mappings.properties.message.type: "match_only_text" }
86+
- match: { .$idx0name.mappings.properties.source.properties.ip.type: "ip" }
87+
- match: { .$idx0name.mappings.properties.log.properties.file.properties.path.type: "keyword" }
7088
- match: { .$idx0name.data_stream: "logs-foo-bar" }
7189

7290
- do:
@@ -80,7 +98,10 @@ setup:
8098
index: metrics-foo-bar
8199
body:
82100
"@timestamp": "2020-01-01"
83-
message: "test-log-message"
101+
source.ip: "10.1.2.3"
102+
data_stream.type: "metrics"
103+
data_stream.dataset: "foo"
104+
data_stream.namespace: "bar"
84105

85106
- do:
86107
indices.get_data_stream:
@@ -101,9 +122,92 @@ setup:
101122
- is_true: .$idx0name.settings
102123
- is_true: .$idx0name.mappings
103124
- match: { .$idx0name.settings.index.lifecycle.name: "metrics" }
104-
- is_true: .$idx0name.mappings.properties.message
125+
- match: { .$idx0name.mappings.properties.data_stream.properties.type.type: "constant_keyword" }
126+
- match: { .$idx0name.mappings.properties.data_stream.properties.type.value: "metrics" }
127+
- match: { .$idx0name.mappings.properties.data_stream.properties.dataset.type: "constant_keyword" }
128+
- match: { .$idx0name.mappings.properties.data_stream.properties.dataset.value: "foo" }
129+
- match: { .$idx0name.mappings.properties.data_stream.properties.namespace.type: "constant_keyword" }
130+
- match: { .$idx0name.mappings.properties.data_stream.properties.namespace.value: "bar" }
131+
- match: { .$idx0name.mappings.properties.source.properties.ip.type: "ip" }
105132
- match: { .$idx0name.data_stream: "metrics-foo-bar" }
106133

107134
- do:
108135
indices.delete_data_stream:
109136
name: metrics-foo-bar
137+
138+
---
139+
"Test synthetics index auto creation":
140+
- do:
141+
index:
142+
index: synthetics-foo-bar
143+
body:
144+
"@timestamp": "2020-01-01"
145+
source.ip: "10.1.2.3"
146+
data_stream.type: "synthetics"
147+
data_stream.dataset: "foo"
148+
data_stream.namespace: "bar"
149+
150+
- do:
151+
indices.get_data_stream:
152+
name: synthetics-foo-bar
153+
154+
- match: { data_streams.0.name: synthetics-foo-bar }
155+
- match: { data_streams.0.timestamp_field.name: '@timestamp' }
156+
- match: { data_streams.0.generation: 1 }
157+
- length: { data_streams.0.indices: 1 }
158+
- match: { data_streams.0.indices.0.index_name: '/\.ds-synthetics-foo-bar-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
159+
160+
- set: { data_streams.0.indices.0.index_name: idx0name }
161+
162+
- do:
163+
indices.get:
164+
index: $idx0name
165+
166+
- is_true: .$idx0name.settings
167+
- is_true: .$idx0name.mappings
168+
- match: { .$idx0name.settings.index.lifecycle.name: "synthetics" }
169+
- match: { .$idx0name.mappings.properties.data_stream.properties.type.type: "constant_keyword" }
170+
- match: { .$idx0name.mappings.properties.data_stream.properties.type.value: "synthetics" }
171+
- match: { .$idx0name.mappings.properties.data_stream.properties.dataset.type: "constant_keyword" }
172+
- match: { .$idx0name.mappings.properties.data_stream.properties.dataset.value: "foo" }
173+
- match: { .$idx0name.mappings.properties.data_stream.properties.namespace.type: "constant_keyword" }
174+
- match: { .$idx0name.mappings.properties.data_stream.properties.namespace.value: "bar" }
175+
- match: { .$idx0name.mappings.properties.source.properties.ip.type: "ip" }
176+
- match: { .$idx0name.data_stream: "synthetics-foo-bar" }
177+
178+
- do:
179+
indices.delete_data_stream:
180+
name: synthetics-foo-bar
181+
182+
---
183+
"Test wrong data_stream type":
184+
185+
- do:
186+
catch: bad_request
187+
index:
188+
index: synthetics-dataset0-namespace1
189+
body:
190+
"@timestamp": "2020-01-01"
191+
data_stream.type: "logs"
192+
data_stream.dataset: "dataset0"
193+
data_stream.namespace: "namespace1"
194+
195+
- do:
196+
catch: bad_request
197+
index:
198+
index: logs-dataset0-namespace1
199+
body:
200+
"@timestamp": "2020-01-01"
201+
data_stream.type: "metrics"
202+
data_stream.dataset: "dataset0"
203+
data_stream.namespace: "namespace1"
204+
205+
- do:
206+
catch: bad_request
207+
index:
208+
index: metrics-dataset0-namespace1
209+
body:
210+
"@timestamp": "2020-01-01"
211+
data_stream.type: "synthetics"
212+
data_stream.dataset: "dataset0"
213+
data_stream.namespace: "namespace1"

0 commit comments

Comments
 (0)