Skip to content

Commit 58bc365

Browse files
imotovimotov
committed
Add time series information to field caps
Exposes information about dimensions and metrics via field caps. This information will be needed for PromQL support. Relates to elastic#74660
1 parent 69cf679 commit 58bc365

File tree

20 files changed

+645
-91
lines changed

20 files changed

+645
-91
lines changed

‎client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java

+3-3
Original file line numberDiff line numberDiff line change
@@ -1253,11 +1253,11 @@ public void testFieldCaps() throws IOException {
12531253
assertEquals(2, ratingResponse.size());
12541254

12551255
FieldCapabilities expectedKeywordCapabilities = new FieldCapabilities(
1256-
"rating", "keyword", false, true, true, new String[]{"index2"}, null, null, Collections.emptyMap());
1256+
"rating", "keyword", false, true, true, false, null, null, null, new String[]{"index2"}, null, Collections.emptyMap());
12571257
assertEquals(expectedKeywordCapabilities, ratingResponse.get("keyword"));
12581258

12591259
FieldCapabilities expectedLongCapabilities = new FieldCapabilities(
1260-
"rating", "long", false, true, true, new String[]{"index1"}, null, null, Collections.emptyMap());
1260+
"rating", "long", false, true, true, false, null, null, null, new String[]{"index1"}, null, Collections.emptyMap());
12611261
assertEquals(expectedLongCapabilities, ratingResponse.get("long"));
12621262

12631263
// Check the capabilities for the 'field' field.
@@ -1266,7 +1266,7 @@ public void testFieldCaps() throws IOException {
12661266
assertEquals(1, fieldResponse.size());
12671267

12681268
FieldCapabilities expectedTextCapabilities = new FieldCapabilities(
1269-
"field", "text", false, true, false, null, null, null, Collections.emptyMap());
1269+
"field", "text", false, true, false, false, null, null, null, null, null, Collections.emptyMap());
12701270
assertEquals(expectedTextCapabilities, fieldResponse.get("text"));
12711271
}
12721272

‎docs/reference/search/field-caps.asciidoc

+22-5
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,12 @@ field types are all described as the `keyword` type family.
111111
`aggregatable`::
112112
Whether this field can be aggregated on all indices.
113113

114+
`time_series_dimension`::
115+
Whether this field is used as a time series dimension.
116+
117+
`time_series_metrics`::
118+
Contains metric type if this fields is used as a time series metrics, null if the field is not used as metric.
119+
114120
`indices`::
115121
The list of indices where this field has the same type family, or null if all indices
116122
have the same type family for the field.
@@ -123,6 +129,10 @@ field types are all described as the `keyword` type family.
123129
The list of indices where this field is not aggregatable, or null if all
124130
indices have the same definition for the field.
125131

132+
`non_dimension_indices`::
133+
The list of indices where this field is not marked as a dimension field, or null if all
134+
indices have the same definition for the field.
135+
126136
`meta`::
127137
Merged metadata across all indices as a map of string keys to arrays of values.
128138
A value length of 1 indicates that all indices had the same value for this key,
@@ -163,13 +173,15 @@ The API returns the following response:
163173
"metadata_field": false,
164174
"searchable": true,
165175
"aggregatable": false,
176+
"time_series_dimension": false,
166177
"indices": [ "index1", "index2" ],
167178
"non_aggregatable_indices": [ "index1" ] <2>
168179
},
169180
"keyword": {
170181
"metadata_field": false,
171182
"searchable": false,
172183
"aggregatable": true,
184+
"time_series_dimension": false,
173185
"indices": [ "index3", "index4" ],
174186
"non_searchable_indices": [ "index4" ] <3>
175187
}
@@ -178,8 +190,8 @@ The API returns the following response:
178190
"text": {
179191
"metadata_field": false,
180192
"searchable": true,
181-
"aggregatable": false
182-
193+
"aggregatable": false,
194+
"time_series_dimension": false
183195
}
184196
}
185197
}
@@ -215,35 +227,40 @@ in some indices but not all:
215227
"metadata_field": false,
216228
"searchable": true,
217229
"aggregatable": false,
230+
"time_series_dimension": false,
218231
"indices": [ "index1", "index2" ],
219232
"non_aggregatable_indices": [ "index1" ]
220233
},
221234
"keyword": {
222235
"metadata_field": false,
223236
"searchable": false,
224237
"aggregatable": true,
238+
"time_series_dimension": false,
225239
"indices": [ "index3", "index4" ],
226240
"non_searchable_indices": [ "index4" ]
227241
},
228242
"unmapped": { <1>
229243
"metadata_field": false,
230244
"indices": [ "index5" ],
231245
"searchable": false,
232-
"aggregatable": false
246+
"aggregatable": false,
247+
"time_series_dimension": false
233248
}
234249
},
235250
"title": {
236251
"text": {
237252
"metadata_field": false,
238253
"indices": [ "index1", "index2", "index3", "index4" ],
239254
"searchable": true,
240-
"aggregatable": false
255+
"aggregatable": false,
256+
"time_series_dimension": false
241257
},
242258
"unmapped": { <2>
243259
"metadata_field": false,
244260
"indices": [ "index5" ],
245261
"searchable": false,
246-
"aggregatable": false
262+
"aggregatable": false,
263+
"time_series_dimension": false
247264
}
248265
}
249266
}

‎rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/field_caps/40_time_series.yml

+221
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,221 @@
1+
---
2+
setup:
3+
- skip:
4+
version: " - 7.99.99"
5+
reason: introduced in 8.0.0
6+
7+
- do:
8+
indices.create:
9+
index: tsdb_index1
10+
body:
11+
settings:
12+
index:
13+
number_of_replicas: 0
14+
number_of_shards: 2
15+
mappings:
16+
properties:
17+
"@timestamp":
18+
type: date
19+
metricset:
20+
type: keyword
21+
time_series_dimension: true
22+
non_tsdb_field:
23+
type: keyword
24+
k8s:
25+
properties:
26+
pod:
27+
properties:
28+
availability_zone:
29+
type: short
30+
time_series_dimension: true
31+
uid:
32+
type: keyword
33+
time_series_dimension: true
34+
name:
35+
type: keyword
36+
ip:
37+
type: ip
38+
time_series_dimension: true
39+
network:
40+
properties:
41+
tx:
42+
type: long
43+
time_series_metric: counter
44+
rx:
45+
type: integer
46+
time_series_metric: gauge
47+
packets_dropped:
48+
type: long
49+
time_series_metric: gauge
50+
latency:
51+
type: double
52+
time_series_metric: gauge
53+
54+
- do:
55+
indices.create:
56+
index: tsdb_index2
57+
body:
58+
settings:
59+
index:
60+
number_of_replicas: 0
61+
number_of_shards: 2
62+
mappings:
63+
properties:
64+
"@timestamp":
65+
type: date
66+
metricset:
67+
type: keyword
68+
non_tsdb_field:
69+
type: keyword
70+
k8s:
71+
properties:
72+
pod:
73+
properties:
74+
availability_zone:
75+
type: short
76+
time_series_dimension: true
77+
uid:
78+
type: keyword
79+
time_series_dimension: true
80+
name:
81+
type: keyword
82+
ip:
83+
type: ip
84+
time_series_dimension: true
85+
network:
86+
properties:
87+
tx:
88+
type: long
89+
time_series_metric: gauge
90+
rx:
91+
type: integer
92+
packets_dropped:
93+
type: long
94+
time_series_metric: gauge
95+
latency:
96+
type: double
97+
time_series_metric: gauge
98+
99+
---
100+
"Get simple time series field caps":
101+
102+
- skip:
103+
version: " - 7.99.99"
104+
reason: introduced in 8.0.0
105+
106+
- do:
107+
field_caps:
108+
index: 'tsdb_index1'
109+
fields: [ "metricset", "non_tsdb_field", "k8s.pod.*" ]
110+
111+
- match: {fields.metricset.keyword.searchable: true}
112+
- match: {fields.metricset.keyword.aggregatable: true}
113+
- match: {fields.metricset.keyword.time_series_dimension: true}
114+
- is_false: fields.metricset.keyword.time_series_metric
115+
- is_false: fields.metricset.keyword.indices
116+
- is_false: fields.metricset.keyword.non_searchable_indices
117+
- is_false: fields.metricset.keyword.non_aggregatable_indices
118+
- is_false: fields.metricset.keyword.non_dimension_indices
119+
120+
- match: {fields.non_tsdb_field.keyword.searchable: true}
121+
- match: {fields.non_tsdb_field.keyword.aggregatable: true}
122+
- match: {fields.non_tsdb_field.keyword.time_series_dimension: false}
123+
- is_false: fields.non_tsdb_field.keyword.time_series_metric
124+
- is_false: fields.non_tsdb_field.keyword.indices
125+
- is_false: fields.non_tsdb_field.keyword.non_searchable_indices
126+
- is_false: fields.non_tsdb_field.keyword.non_aggregatable_indices
127+
- is_false: fields.non_tsdb_field.keyword.non_dimension_indices
128+
129+
- match: {fields.k8s\.pod\.availability_zone.short.time_series_dimension: true}
130+
- is_false: fields.k8s\.pod\.availability_zone.short.time_series_metric
131+
- is_false: fields.k8s\.pod\.availability_zone.short.non_dimension_indices
132+
133+
- match: {fields.k8s\.pod\.uid.keyword.time_series_dimension: true}
134+
- is_false: fields.k8s\.pod\.uid.keyword.time_series_metric
135+
- is_false: fields.k8s\.pod\.uid.keyword.non_dimension_indices
136+
137+
- match: {fields.k8s\.pod\.name.keyword.time_series_dimension: false}
138+
- is_false: fields.k8s\.pod\.name.keyword.time_series_metric
139+
- is_false: fields.k8s\.pod\.name.keyword.non_dimension_indices
140+
141+
- match: {fields.k8s\.pod\.ip.ip.time_series_dimension: true}
142+
- is_false: fields.k8s\.pod\.ip.ip.time_series_metric
143+
- is_false: fields.k8s\.pod\.ip.ip.non_dimension_indices
144+
145+
- match: {fields.k8s\.pod\.network\.tx.long.time_series_dimension: false}
146+
- match: {fields.k8s\.pod\.network\.tx.long.time_series_metric: counter}
147+
- is_false: fields.k8s\.pod\.network\.tx.long.non_dimension_indices
148+
149+
- match: {fields.k8s\.pod\.network\.rx.integer.time_series_dimension: false}
150+
- match: {fields.k8s\.pod\.network\.rx.integer.time_series_metric: gauge}
151+
- is_false: fields.k8s\.pod\.network\.rx.integer.non_dimension_indices
152+
153+
- match: {fields.k8s\.pod\.network\.packets_dropped.long.time_series_dimension: false}
154+
- match: {fields.k8s\.pod\.network\.packets_dropped.long.time_series_metric: gauge}
155+
- is_false: fields.k8s\.pod\.network\.packets_dropped.long.non_dimension_indices
156+
157+
- match: {fields.k8s\.pod\.network\.latency.double.time_series_dimension: false}
158+
- match: {fields.k8s\.pod\.network\.latency.double.time_series_metric: gauge}
159+
- is_false: fields.k8s\.pod\.network\.latency.double.non_dimension_indices
160+
161+
---
162+
"Get time series field caps with conflicts":
163+
164+
- skip:
165+
version: " - 7.99.99"
166+
reason: introduced in 8.0.0
167+
168+
- do:
169+
field_caps:
170+
index: tsdb_index1,tsdb_index2
171+
fields: [ "metricset", "non_tsdb_field", "k8s.pod.*" ]
172+
173+
- match: {fields.metricset.keyword.searchable: true}
174+
- match: {fields.metricset.keyword.aggregatable: true}
175+
- match: {fields.metricset.keyword.time_series_dimension: false}
176+
- is_false: fields.metricset.keyword.time_series_metric
177+
- is_false: fields.metricset.keyword.indices
178+
- is_false: fields.metricset.keyword.non_searchable_indices
179+
- is_false: fields.metricset.keyword.non_aggregatable_indices
180+
- match: {fields.metricset.keyword.non_dimension_indices: ["tsdb_index2"]}
181+
182+
- match: {fields.non_tsdb_field.keyword.searchable: true}
183+
- match: {fields.non_tsdb_field.keyword.aggregatable: true}
184+
- match: {fields.non_tsdb_field.keyword.time_series_dimension: false}
185+
- is_false: fields.non_tsdb_field.keyword.time_series_metric
186+
- is_false: fields.non_tsdb_field.keyword.indices
187+
- is_false: fields.non_tsdb_field.keyword.non_searchable_indices
188+
- is_false: fields.non_tsdb_field.keyword.non_aggregatable_indices
189+
- is_false: fields.non_tsdb_field.keyword.non_dimension_indices
190+
191+
- match: {fields.k8s\.pod\.availability_zone.short.time_series_dimension: true}
192+
- is_false: fields.k8s\.pod\.availability_zone.short.time_series_metric
193+
- is_false: fields.k8s\.pod\.availability_zone.short.non_dimension_indices
194+
195+
- match: {fields.k8s\.pod\.uid.keyword.time_series_dimension: true}
196+
- is_false: fields.k8s\.pod\.uid.keyword.time_series_metric
197+
- is_false: fields.k8s\.pod\.uid.keyword.non_dimension_indices
198+
199+
- match: {fields.k8s\.pod\.name.keyword.time_series_dimension: false}
200+
- is_false: fields.k8s\.pod\.name.keyword.time_series_metric
201+
- is_false: fields.k8s\.pod\.name.keyword.non_dimension_indices
202+
203+
- match: {fields.k8s\.pod\.ip.ip.time_series_dimension: true}
204+
- is_false: fields.k8s\.pod\.ip.ip.time_series_metric
205+
- is_false: fields.k8s\.pod\.ip.ip.non_dimension_indices
206+
207+
- match: {fields.k8s\.pod\.network\.tx.long.time_series_dimension: false}
208+
- is_false: fields.k8s\.pod\.network\.tx.long.time_series_metric
209+
- is_false: fields.k8s\.pod\.network\.tx.long.non_dimension_indices
210+
211+
- match: {fields.k8s\.pod\.network\.rx.integer.time_series_dimension: false}
212+
- is_false: fields.k8s\.pod\.network\.rx.integer.time_series_metric
213+
- is_false: fields.k8s\.pod\.network\.rx.integer.non_dimension_indices
214+
215+
- match: {fields.k8s\.pod\.network\.packets_dropped.long.time_series_dimension: false}
216+
- match: {fields.k8s\.pod\.network\.packets_dropped.long.time_series_metric: gauge}
217+
- is_false: fields.k8s\.pod\.network\.packets_dropped.long.non_dimension_indices
218+
219+
- match: {fields.k8s\.pod\.network\.latency.double.time_series_dimension: false}
220+
- match: {fields.k8s\.pod\.network\.latency.double.time_series_metric: gauge}
221+
- is_false: fields.k8s\.pod\.network\.latency.double.non_dimension_indices

0 commit comments

Comments
 (0)