Skip to content

Commit f6034e6

Browse files
imotovimotov
authored
TSDB: Add time series information to field caps (#78790)
Exposes information about dimensions and metrics via field caps. This information will be needed for PromQL support. Relates to #74660
1 parent 1ed9b15 commit f6034e6

File tree

12 files changed

+751
-57
lines changed

12 files changed

+751
-57
lines changed

‎docs/reference/search/field-caps.asciidoc

+14-1
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,12 @@ field types are all described as the `keyword` type family.
111111
`aggregatable`::
112112
Whether this field can be aggregated on all indices.
113113

114+
`time_series_dimension`::
115+
Whether this field is used as a time series dimension.
116+
117+
`time_series_metric`::
118+
Contains metric type if this fields is used as a time series metrics, absent if the field is not used as metric.
119+
114120
`indices`::
115121
The list of indices where this field has the same type family, or null if all indices
116122
have the same type family for the field.
@@ -123,6 +129,14 @@ field types are all described as the `keyword` type family.
123129
The list of indices where this field is not aggregatable, or null if all
124130
indices have the same definition for the field.
125131

132+
`non_dimension_indices`::
133+
If this list is present in response then some indices have the field marked as a dimension and other indices, the
134+
ones in this list, do not.
135+
136+
`metric_conflicts_indices`::
137+
The list of indices where this field is present if these indices don't have the same `time_series_metric` value for
138+
this field.
139+
126140
`meta`::
127141
Merged metadata across all indices as a map of string keys to arrays of values.
128142
A value length of 1 indicates that all indices had the same value for this key,
@@ -179,7 +193,6 @@ The API returns the following response:
179193
"metadata_field": false,
180194
"searchable": true,
181195
"aggregatable": false
182-
183196
}
184197
}
185198
}

‎rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/field_caps/40_time_series.yml

+231
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,231 @@
1+
---
2+
setup:
3+
- skip:
4+
version: " - 7.99.99"
5+
reason: introduced in 8.0.0
6+
7+
- do:
8+
indices.create:
9+
index: tsdb_index1
10+
body:
11+
settings:
12+
index:
13+
number_of_replicas: 0
14+
number_of_shards: 2
15+
mappings:
16+
properties:
17+
"@timestamp":
18+
type: date
19+
metricset:
20+
type: keyword
21+
time_series_dimension: true
22+
non_tsdb_field:
23+
type: keyword
24+
k8s:
25+
properties:
26+
pod:
27+
properties:
28+
availability_zone:
29+
type: short
30+
time_series_dimension: true
31+
uid:
32+
type: keyword
33+
time_series_dimension: true
34+
name:
35+
type: keyword
36+
ip:
37+
type: ip
38+
time_series_dimension: true
39+
network:
40+
properties:
41+
tx:
42+
type: long
43+
time_series_metric: counter
44+
rx:
45+
type: integer
46+
time_series_metric: gauge
47+
packets_dropped:
48+
type: long
49+
time_series_metric: gauge
50+
latency:
51+
type: double
52+
time_series_metric: gauge
53+
54+
- do:
55+
indices.create:
56+
index: tsdb_index2
57+
body:
58+
settings:
59+
index:
60+
number_of_replicas: 0
61+
number_of_shards: 2
62+
mappings:
63+
properties:
64+
"@timestamp":
65+
type: date
66+
metricset:
67+
type: keyword
68+
non_tsdb_field:
69+
type: keyword
70+
k8s:
71+
properties:
72+
pod:
73+
properties:
74+
availability_zone:
75+
type: short
76+
time_series_dimension: true
77+
uid:
78+
type: keyword
79+
time_series_dimension: true
80+
name:
81+
type: keyword
82+
ip:
83+
type: ip
84+
time_series_dimension: true
85+
network:
86+
properties:
87+
tx:
88+
type: long
89+
time_series_metric: gauge
90+
rx:
91+
type: integer
92+
packets_dropped:
93+
type: long
94+
time_series_metric: gauge
95+
latency:
96+
type: double
97+
time_series_metric: gauge
98+
99+
---
100+
"Get simple time series field caps":
101+
102+
- skip:
103+
version: " - 7.99.99"
104+
reason: introduced in 8.0.0
105+
106+
- do:
107+
field_caps:
108+
index: 'tsdb_index1'
109+
fields: [ "metricset", "non_tsdb_field", "k8s.pod.*" ]
110+
111+
- match: {fields.metricset.keyword.searchable: true}
112+
- match: {fields.metricset.keyword.aggregatable: true}
113+
- match: {fields.metricset.keyword.time_series_dimension: true}
114+
- is_false: fields.metricset.keyword.time_series_metric
115+
- is_false: fields.metricset.keyword.indices
116+
- is_false: fields.metricset.keyword.non_searchable_indices
117+
- is_false: fields.metricset.keyword.non_aggregatable_indices
118+
- is_false: fields.metricset.keyword.non_dimension_indices
119+
120+
- match: {fields.non_tsdb_field.keyword.searchable: true}
121+
- match: {fields.non_tsdb_field.keyword.aggregatable: true}
122+
- is_false: fields.non_tsdb_field.keyword.time_series_dimension
123+
- is_false: fields.non_tsdb_field.keyword.time_series_metric
124+
- is_false: fields.non_tsdb_field.keyword.indices
125+
- is_false: fields.non_tsdb_field.keyword.non_searchable_indices
126+
- is_false: fields.non_tsdb_field.keyword.non_aggregatable_indices
127+
- is_false: fields.non_tsdb_field.keyword.non_dimension_indices
128+
129+
- match: {fields.k8s\.pod\.availability_zone.short.time_series_dimension: true}
130+
- is_false: fields.k8s\.pod\.availability_zone.short.time_series_metric
131+
- is_false: fields.k8s\.pod\.availability_zone.short.non_dimension_indices
132+
133+
- match: {fields.k8s\.pod\.uid.keyword.time_series_dimension: true}
134+
- is_false: fields.k8s\.pod\.uid.keyword.time_series_metric
135+
- is_false: fields.k8s\.pod\.uid.keyword.non_dimension_indices
136+
137+
- is_false: fields.k8s\.pod\.name.keyword.time_series_dimension
138+
- is_false: fields.k8s\.pod\.name.keyword.time_series_metric
139+
- is_false: fields.k8s\.pod\.name.keyword.non_dimension_indices
140+
141+
- match: {fields.k8s\.pod\.ip.ip.time_series_dimension: true}
142+
- is_false: fields.k8s\.pod\.ip.ip.time_series_metric
143+
- is_false: fields.k8s\.pod\.ip.ip.non_dimension_indices
144+
145+
- is_false: fields.k8s\.pod\.network\.tx.long.time_series_dimension
146+
- match: {fields.k8s\.pod\.network\.tx.long.time_series_metric: counter}
147+
- is_false: fields.k8s\.pod\.network\.tx.long.non_dimension_indices
148+
149+
- is_false: fields.k8s\.pod\.network\.rx.integer.time_series_dimension
150+
- match: {fields.k8s\.pod\.network\.rx.integer.time_series_metric: gauge}
151+
- is_false: fields.k8s\.pod\.network\.rx.integer.non_dimension_indices
152+
153+
- is_false: fields.k8s\.pod\.network\.packets_dropped.long.time_series_dimension
154+
- match: {fields.k8s\.pod\.network\.packets_dropped.long.time_series_metric: gauge}
155+
- is_false: fields.k8s\.pod\.network\.packets_dropped.long.non_dimension_indices
156+
157+
- is_false: fields.k8s\.pod\.network\.latency.double.time_series_dimension
158+
- match: {fields.k8s\.pod\.network\.latency.double.time_series_metric: gauge}
159+
- is_false: fields.k8s\.pod\.network\.latency.double.non_dimension_indices
160+
161+
---
162+
"Get time series field caps with conflicts":
163+
164+
- skip:
165+
version: " - 7.99.99"
166+
reason: introduced in 8.0.0
167+
168+
- do:
169+
field_caps:
170+
index: tsdb_index1,tsdb_index2
171+
fields: [ "metricset", "non_tsdb_field", "k8s.pod.*" ]
172+
173+
- match: {fields.metricset.keyword.searchable: true}
174+
- match: {fields.metricset.keyword.aggregatable: true}
175+
- is_false: fields.metricset.keyword.time_series_dimension
176+
- is_false: fields.metricset.keyword.time_series_metric
177+
- is_false: fields.metricset.keyword.indices
178+
- is_false: fields.metricset.keyword.non_searchable_indices
179+
- is_false: fields.metricset.keyword.non_aggregatable_indices
180+
- match: {fields.metricset.keyword.non_dimension_indices: ["tsdb_index2"]}
181+
- is_false: fields.metricset.keyword.mertric_conflicts_indices
182+
183+
- match: {fields.non_tsdb_field.keyword.searchable: true}
184+
- match: {fields.non_tsdb_field.keyword.aggregatable: true}
185+
- is_false: fields.non_tsdb_field.keyword.time_series_dimension
186+
- is_false: fields.non_tsdb_field.keyword.time_series_metric
187+
- is_false: fields.non_tsdb_field.keyword.indices
188+
- is_false: fields.non_tsdb_field.keyword.non_searchable_indices
189+
- is_false: fields.non_tsdb_field.keyword.non_aggregatable_indices
190+
- is_false: fields.non_tsdb_field.keyword.non_dimension_indices
191+
- is_false: fields.non_tsdb_field.keyword.mertric_conflicts_indices
192+
193+
- match: {fields.k8s\.pod\.availability_zone.short.time_series_dimension: true}
194+
- is_false: fields.k8s\.pod\.availability_zone.short.time_series_metric
195+
- is_false: fields.k8s\.pod\.availability_zone.short.non_dimension_indices
196+
- is_false: fields.k8s\.pod\.availability_zone.short.mertric_conflicts_indices
197+
198+
- match: {fields.k8s\.pod\.uid.keyword.time_series_dimension: true}
199+
- is_false: fields.k8s\.pod\.uid.keyword.time_series_metric
200+
- is_false: fields.k8s\.pod\.uid.keyword.non_dimension_indices
201+
- is_false: fields.k8s\.pod\.uid.keyword.mertric_conflicts_indices
202+
203+
- is_false: fields.k8s\.pod\.name.keyword.time_series_dimension
204+
- is_false: fields.k8s\.pod\.name.keyword.time_series_metric
205+
- is_false: fields.k8s\.pod\.name.keyword.non_dimension_indices
206+
- is_false: fields.k8s\.pod\.name.keyword.mertric_conflicts_indices
207+
208+
- match: {fields.k8s\.pod\.ip.ip.time_series_dimension: true}
209+
- is_false: fields.k8s\.pod\.ip.ip.time_series_metric
210+
- is_false: fields.k8s\.pod\.ip.ip.non_dimension_indices
211+
- is_false: fields.k8s\.pod\.ip.ip.mertric_conflicts_indices
212+
213+
- is_false: fields.k8s\.pod\.network\.tx.long.time_series_dimension
214+
- is_false: fields.k8s\.pod\.network\.tx.long.time_series_metric
215+
- is_false: fields.k8s\.pod\.network\.tx.long.non_dimension_indices
216+
- match: {fields.k8s\.pod\.network\.tx.long.mertric_conflicts_indices: ["tsdb_index1", "tsdb_index2"]}
217+
218+
- is_false: fields.k8s\.pod\.network\.rx.integer.time_series_dimension
219+
- is_false: fields.k8s\.pod\.network\.rx.integer.time_series_metric
220+
- is_false: fields.k8s\.pod\.network\.rx.integer.non_dimension_indices
221+
- match: {fields.k8s\.pod\.network\.rx.integer.mertric_conflicts_indices: ["tsdb_index1", "tsdb_index2"]}
222+
223+
- is_false: fields.k8s\.pod\.network\.packets_dropped.long.time_series_dimension
224+
- match: {fields.k8s\.pod\.network\.packets_dropped.long.time_series_metric: gauge}
225+
- is_false: fields.k8s\.pod\.network\.packets_dropped.long.non_dimension_indices
226+
- is_false: fields.k8s\.pod\.network\.packets_dropped.long.mertric_conflicts_indices
227+
228+
- is_false: fields.k8s\.pod\.network\.latency.double.time_series_dimension
229+
- match: {fields.k8s\.pod\.network\.latency.double.time_series_metric: gauge}
230+
- is_false: fields.k8s\.pod\.network\.latency.double.non_dimension_indices
231+
- is_false: fields.k8s\.pod\.network\.latency.double.mertric_conflicts_indices

‎server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java

+39-2
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,10 @@
1515
import org.elasticsearch.action.index.IndexRequestBuilder;
1616
import org.elasticsearch.common.io.stream.StreamInput;
1717
import org.elasticsearch.common.io.stream.StreamOutput;
18-
import org.elasticsearch.xcontent.XContentBuilder;
19-
import org.elasticsearch.xcontent.XContentFactory;
2018
import org.elasticsearch.index.mapper.DocumentParserContext;
2119
import org.elasticsearch.index.mapper.KeywordFieldMapper;
2220
import org.elasticsearch.index.mapper.MetadataFieldMapper;
21+
import org.elasticsearch.index.mapper.TimeSeriesParams;
2322
import org.elasticsearch.index.query.AbstractQueryBuilder;
2423
import org.elasticsearch.index.query.QueryBuilder;
2524
import org.elasticsearch.index.query.QueryBuilders;
@@ -30,6 +29,8 @@
3029
import org.elasticsearch.plugins.SearchPlugin;
3130
import org.elasticsearch.test.ESIntegTestCase;
3231
import org.elasticsearch.transport.RemoteTransportException;
32+
import org.elasticsearch.xcontent.XContentBuilder;
33+
import org.elasticsearch.xcontent.XContentFactory;
3334
import org.junit.Before;
3435

3536
import java.io.IOException;
@@ -45,8 +46,10 @@
4546

4647
import static java.util.Collections.singletonList;
4748
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
49+
import static org.hamcrest.Matchers.array;
4850
import static org.hamcrest.Matchers.arrayContainingInAnyOrder;
4951
import static org.hamcrest.Matchers.containsInAnyOrder;
52+
import static org.hamcrest.Matchers.equalTo;
5053

5154
public class FieldCapabilitiesIT extends ESIntegTestCase {
5255

@@ -69,6 +72,14 @@ public void setUp() throws Exception {
6972
.startObject("playlist")
7073
.field("type", "text")
7174
.endObject()
75+
.startObject("some_dimension")
76+
.field("type", "keyword")
77+
.field("time_series_dimension", true)
78+
.endObject()
79+
.startObject("some_metric")
80+
.field("type", "long")
81+
.field("time_series_metric", TimeSeriesParams.MetricType.counter)
82+
.endObject()
7283
.startObject("secret_soundtrack")
7384
.field("type", "alias")
7485
.field("path", "playlist")
@@ -98,6 +109,13 @@ public void setUp() throws Exception {
98109
.startObject("new_field")
99110
.field("type", "long")
100111
.endObject()
112+
.startObject("some_dimension")
113+
.field("type", "keyword")
114+
.endObject()
115+
.startObject("some_metric")
116+
.field("type", "long")
117+
.field("time_series_metric", TimeSeriesParams.MetricType.gauge)
118+
.endObject()
101119
.endObject()
102120
.endObject()
103121
.endObject();
@@ -285,6 +303,25 @@ public void testWithRunntimeMappings() throws InterruptedException {
285303
assertTrue(runtimeField.get("keyword").isAggregatable());
286304
}
287305

306+
public void testFieldMetricsAndDimensions() {
307+
FieldCapabilitiesResponse response = client().prepareFieldCaps("old_index").setFields("some_dimension", "some_metric").get();
308+
assertIndices(response, "old_index");
309+
assertEquals(2, response.get().size());
310+
assertTrue(response.get().containsKey("some_dimension"));
311+
assertTrue(response.get().get("some_dimension").get("keyword").isDimension());
312+
assertNull(response.get().get("some_dimension").get("keyword").nonDimensionIndices());
313+
assertTrue(response.get().containsKey("some_metric"));
314+
assertEquals(TimeSeriesParams.MetricType.counter, response.get().get("some_metric").get("long").getMetricType());
315+
assertNull(response.get().get("some_metric").get("long").metricConflictsIndices());
316+
317+
response = client().prepareFieldCaps("old_index", "new_index").setFields("some_dimension", "some_metric").get();
318+
assertIndices(response, "old_index", "new_index");
319+
assertEquals(2, response.get().size());
320+
assertTrue(response.get().containsKey("some_dimension"));
321+
assertFalse(response.get().get("some_dimension").get("keyword").isDimension());
322+
assertThat(response.get().get("some_dimension").get("keyword").nonDimensionIndices(), array(equalTo("new_index")));
323+
}
324+
288325
public void testFailures() throws InterruptedException {
289326
// in addition to the existing "old_index" and "new_index", create two where the test query throws an error on rewrite
290327
assertAcked(prepareCreate("index1-error"));

0 commit comments

Comments
 (0)