Skip to content

Commit 041f4d6

Browse files
polyfractalGurkan Kaymak
polyfractal
authored and
Gurkan Kaymak
committed
Force selection of calendar or fixed intervals in date histo agg (elastic#33727)
The date_histogram accepts an interval which can be either a calendar interval (DST-aware, leap seconds, arbitrary length of months, etc) or fixed interval (strict multiples of SI units). Unfortunately this is inferred by first trying to parse as a calendar interval, then falling back to fixed if that fails. This leads to confusing arrangement where `1d` == calendar, but `2d` == fixed. And if you want a day of fixed time, you have to specify `24h` (e.g. the next smallest unit). This arrangement is very error-prone for users. This PR adds `calendar_interval` and `fixed_interval` parameters to any code that uses intervals (date_histogram, rollup, composite, datafeed, etc). Calendar only accepts calendar intervals, fixed accepts any combination of units (meaning `1d` can be used to specify `24h` in fixed time), and both are mutually exclusive. The old interval behavior is deprecated and will throw a deprecation warning. It is also mutually exclusive with the two new parameters. In the future the old dual-purpose interval will be removed. The change applies to both REST and java clients.
1 parent 4ded96a commit 041f4d6

File tree

108 files changed

+3268
-670
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

108 files changed

+3268
-670
lines changed

‎client/rest-high-level/src/main/java/org/elasticsearch/client/rollup/job/config/DateHistogramGroupConfig.java

+125-12
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import org.elasticsearch.client.ValidationException;
2323
import org.elasticsearch.common.Nullable;
2424
import org.elasticsearch.common.ParseField;
25+
import org.elasticsearch.common.unit.TimeValue;
2526
import org.elasticsearch.common.xcontent.ConstructingObjectParser;
2627
import org.elasticsearch.common.xcontent.ToXContentObject;
2728
import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -30,8 +31,11 @@
3031
import org.joda.time.DateTimeZone;
3132

3233
import java.io.IOException;
34+
import java.util.Collections;
35+
import java.util.HashSet;
3336
import java.util.Objects;
3437
import java.util.Optional;
38+
import java.util.Set;
3539

3640
import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
3741
import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
@@ -59,14 +63,63 @@ public class DateHistogramGroupConfig implements Validatable, ToXContentObject {
5963
private static final String TIME_ZONE = "time_zone";
6064
private static final String DELAY = "delay";
6165
private static final String DEFAULT_TIMEZONE = "UTC";
66+
private static final String CALENDAR_INTERVAL = "calendar_interval";
67+
private static final String FIXED_INTERVAL = "fixed_interval";
68+
69+
// From DateHistogramAggregationBuilder in core, transplanted and modified to a set
70+
// so we don't need to import a dependency on the class
71+
private static final Set<String> DATE_FIELD_UNITS;
72+
static {
73+
Set<String> dateFieldUnits = new HashSet<>();
74+
dateFieldUnits.add("year");
75+
dateFieldUnits.add("1y");
76+
dateFieldUnits.add("quarter");
77+
dateFieldUnits.add("1q");
78+
dateFieldUnits.add("month");
79+
dateFieldUnits.add("1M");
80+
dateFieldUnits.add("week");
81+
dateFieldUnits.add("1w");
82+
dateFieldUnits.add("day");
83+
dateFieldUnits.add("1d");
84+
dateFieldUnits.add("hour");
85+
dateFieldUnits.add("1h");
86+
dateFieldUnits.add("minute");
87+
dateFieldUnits.add("1m");
88+
dateFieldUnits.add("second");
89+
dateFieldUnits.add("1s");
90+
DATE_FIELD_UNITS = Collections.unmodifiableSet(dateFieldUnits);
91+
}
6292

6393
private static final ConstructingObjectParser<DateHistogramGroupConfig, Void> PARSER;
6494
static {
65-
PARSER = new ConstructingObjectParser<>(NAME, true, a ->
66-
new DateHistogramGroupConfig((String) a[0], (DateHistogramInterval) a[1], (DateHistogramInterval) a[2], (String) a[3]));
95+
PARSER = new ConstructingObjectParser<>(NAME, true, a -> {
96+
DateHistogramInterval oldInterval = (DateHistogramInterval) a[1];
97+
DateHistogramInterval calendarInterval = (DateHistogramInterval) a[2];
98+
DateHistogramInterval fixedInterval = (DateHistogramInterval) a[3];
99+
100+
if (oldInterval != null) {
101+
if (calendarInterval != null || fixedInterval != null) {
102+
throw new IllegalArgumentException("Cannot use [interval] with [fixed_interval] or [calendar_interval] " +
103+
"configuration options.");
104+
}
105+
return new DateHistogramGroupConfig((String) a[0], oldInterval, (DateHistogramInterval) a[4], (String) a[5]);
106+
} else if (calendarInterval != null && fixedInterval == null) {
107+
return new CalendarInterval((String) a[0], calendarInterval, (DateHistogramInterval) a[4], (String) a[5]);
108+
} else if (calendarInterval == null && fixedInterval != null) {
109+
return new FixedInterval((String) a[0], fixedInterval, (DateHistogramInterval) a[4], (String) a[5]);
110+
} else if (calendarInterval != null && fixedInterval != null) {
111+
throw new IllegalArgumentException("Cannot set both [fixed_interval] and [calendar_interval] at the same time");
112+
} else {
113+
throw new IllegalArgumentException("An interval is required. Use [fixed_interval] or [calendar_interval].");
114+
}
115+
});
67116
PARSER.declareString(constructorArg(), new ParseField(FIELD));
68-
PARSER.declareField(constructorArg(), p -> new DateHistogramInterval(p.text()), new ParseField(INTERVAL), ValueType.STRING);
69-
PARSER.declareField(optionalConstructorArg(), p -> new DateHistogramInterval(p.text()), new ParseField(DELAY), ValueType.STRING);
117+
PARSER.declareField(optionalConstructorArg(), p -> new DateHistogramInterval(p.text()), new ParseField(INTERVAL), ValueType.STRING);
118+
PARSER.declareField(optionalConstructorArg(), p -> new DateHistogramInterval(p.text()),
119+
new ParseField(CALENDAR_INTERVAL), ValueType.STRING);
120+
PARSER.declareField(optionalConstructorArg(), p -> new DateHistogramInterval(p.text()),
121+
new ParseField(FIXED_INTERVAL), ValueType.STRING);
122+
PARSER.declareField(optionalConstructorArg(), p -> new DateHistogramInterval(p.text()), new ParseField(DELAY), ValueType.STRING);
70123
PARSER.declareString(optionalConstructorArg(), new ParseField(TIME_ZONE));
71124
}
72125

@@ -75,27 +128,81 @@ public class DateHistogramGroupConfig implements Validatable, ToXContentObject {
75128
private final DateHistogramInterval delay;
76129
private final String timeZone;
77130

131+
/**
132+
* FixedInterval is a {@link DateHistogramGroupConfig} that uses a fixed time interval for rolling up data.
133+
* The fixed time interval is one or multiples of SI units and has no calendar-awareness (e.g. doesn't account
134+
* for leap corrections, does not have variable length months, etc).
135+
*
136+
* For calendar-aware rollups, use {@link CalendarInterval}
137+
*/
138+
public static class FixedInterval extends DateHistogramGroupConfig {
139+
public FixedInterval(String field, DateHistogramInterval interval) {
140+
this(field, interval, null, null);
141+
}
142+
143+
public FixedInterval(String field, DateHistogramInterval interval, DateHistogramInterval delay, String timeZone) {
144+
super(field, interval, delay, timeZone);
145+
// validate fixed time
146+
TimeValue.parseTimeValue(interval.toString(), NAME + ".FixedInterval");
147+
}
148+
}
149+
150+
/**
151+
* CalendarInterval is a {@link DateHistogramGroupConfig} that uses calendar-aware intervals for rolling up data.
152+
* Calendar time intervals understand leap corrections and contextual differences in certain calendar units (e.g.
153+
* months are variable length depending on the month). Calendar units are only available in singular quantities:
154+
* 1s, 1m, 1h, 1d, 1w, 1q, 1M, 1y
155+
*
156+
* For fixed time rollups, use {@link FixedInterval}
157+
*/
158+
public static class CalendarInterval extends DateHistogramGroupConfig {
159+
public CalendarInterval(String field, DateHistogramInterval interval) {
160+
this(field, interval, null, null);
161+
162+
}
163+
164+
public CalendarInterval(String field, DateHistogramInterval interval, DateHistogramInterval delay, String timeZone) {
165+
super(field, interval, delay, timeZone);
166+
if (DATE_FIELD_UNITS.contains(interval.toString()) == false) {
167+
throw new IllegalArgumentException("The supplied interval [" + interval +"] could not be parsed " +
168+
"as a calendar interval.");
169+
}
170+
}
171+
172+
}
173+
78174
/**
79175
* Create a new {@link DateHistogramGroupConfig} using the given field and interval parameters.
176+
*
177+
* @deprecated Build a DateHistoConfig using {@link DateHistogramGroupConfig.CalendarInterval}
178+
* or {@link DateHistogramGroupConfig.FixedInterval} instead
179+
*
180+
* @since 7.2.0
80181
*/
182+
@Deprecated
81183
public DateHistogramGroupConfig(final String field, final DateHistogramInterval interval) {
82184
this(field, interval, null, null);
83185
}
84186

85187
/**
86188
* Create a new {@link DateHistogramGroupConfig} using the given configuration parameters.
87189
* <p>
88-
* The {@code field} and {@code interval} are required to compute the date histogram for the rolled up documents.
89-
* The {@code delay} is optional and can be set to {@code null}. It defines how long to wait before rolling up new documents.
90-
* The {@code timeZone} is optional and can be set to {@code null}. When configured, the time zone value is resolved using
91-
* ({@link DateTimeZone#forID(String)} and must match a time zone identifier provided by the Joda Time library.
190+
* The {@code field} and {@code interval} are required to compute the date histogram for the rolled up documents.
191+
* The {@code delay} is optional and can be set to {@code null}. It defines how long to wait before rolling up new documents.
192+
* The {@code timeZone} is optional and can be set to {@code null}. When configured, the time zone value is resolved using
193+
* ({@link DateTimeZone#forID(String)} and must match a time zone identifier provided by the Joda Time library.
92194
* </p>
93-
*
94-
* @param field the name of the date field to use for the date histogram (required)
195+
* @param field the name of the date field to use for the date histogram (required)
95196
* @param interval the interval to use for the date histogram (required)
96-
* @param delay the time delay (optional)
197+
* @param delay the time delay (optional)
97198
* @param timeZone the id of time zone to use to calculate the date histogram (optional). When {@code null}, the UTC timezone is used.
199+
*
200+
* @deprecated Build a DateHistoConfig using {@link DateHistogramGroupConfig.CalendarInterval}
201+
* or {@link DateHistogramGroupConfig.FixedInterval} instead
202+
*
203+
* @since 7.2.0
98204
*/
205+
@Deprecated
99206
public DateHistogramGroupConfig(final String field,
100207
final DateHistogramInterval interval,
101208
final @Nullable DateHistogramInterval delay,
@@ -153,7 +260,13 @@ public String getTimeZone() {
153260
public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException {
154261
builder.startObject();
155262
{
156-
builder.field(INTERVAL, interval.toString());
263+
if (this.getClass().equals(CalendarInterval.class)) {
264+
builder.field(CALENDAR_INTERVAL, interval.toString());
265+
} else if (this.getClass().equals(FixedInterval.class)) {
266+
builder.field(FIXED_INTERVAL, interval.toString());
267+
} else {
268+
builder.field(INTERVAL, interval.toString());
269+
}
157270
builder.field(FIELD, field);
158271
if (delay != null) {
159272
builder.field(DELAY, delay.toString());

‎client/rest-high-level/src/test/java/org/elasticsearch/client/RollupIT.java

+6-6
Original file line numberDiff line numberDiff line change
@@ -152,7 +152,7 @@ public int indexDocs() throws Exception {
152152

153153

154154
public void testDeleteRollupJob() throws Exception {
155-
final GroupConfig groups = new GroupConfig(new DateHistogramGroupConfig("date", DateHistogramInterval.DAY));
155+
final GroupConfig groups = new GroupConfig(new DateHistogramGroupConfig.CalendarInterval("date", DateHistogramInterval.DAY));
156156
final List<MetricConfig> metrics = Collections.singletonList(new MetricConfig("value", SUPPORTED_METRICS));
157157
final TimeValue timeout = TimeValue.timeValueSeconds(randomIntBetween(30, 600));
158158
PutRollupJobRequest putRollupJobRequest =
@@ -174,7 +174,7 @@ public void testDeleteMissingRollupJob() {
174174

175175
public void testPutStartAndGetRollupJob() throws Exception {
176176
// TODO expand this to also test with histogram and terms?
177-
final GroupConfig groups = new GroupConfig(new DateHistogramGroupConfig("date", DateHistogramInterval.DAY));
177+
final GroupConfig groups = new GroupConfig(new DateHistogramGroupConfig.CalendarInterval("date", DateHistogramInterval.DAY));
178178
final List<MetricConfig> metrics = Collections.singletonList(new MetricConfig("value", SUPPORTED_METRICS));
179179
final TimeValue timeout = TimeValue.timeValueSeconds(randomIntBetween(30, 600));
180180

@@ -333,7 +333,7 @@ public void testGetRollupCaps() throws Exception {
333333
final String cron = "*/1 * * * * ?";
334334
final int pageSize = randomIntBetween(numDocs, numDocs * 10);
335335
// TODO expand this to also test with histogram and terms?
336-
final GroupConfig groups = new GroupConfig(new DateHistogramGroupConfig("date", DateHistogramInterval.DAY));
336+
final GroupConfig groups = new GroupConfig(new DateHistogramGroupConfig.CalendarInterval("date", DateHistogramInterval.DAY));
337337
final List<MetricConfig> metrics = Collections.singletonList(new MetricConfig("value", SUPPORTED_METRICS));
338338
final TimeValue timeout = TimeValue.timeValueSeconds(randomIntBetween(30, 600));
339339

@@ -377,7 +377,7 @@ public void testGetRollupCaps() throws Exception {
377377
case "delay":
378378
assertThat(entry.getValue(), equalTo("foo"));
379379
break;
380-
case "interval":
380+
case "calendar_interval":
381381
assertThat(entry.getValue(), equalTo("1d"));
382382
break;
383383
case "time_zone":
@@ -445,7 +445,7 @@ public void testGetRollupIndexCaps() throws Exception {
445445
final String cron = "*/1 * * * * ?";
446446
final int pageSize = randomIntBetween(numDocs, numDocs * 10);
447447
// TODO expand this to also test with histogram and terms?
448-
final GroupConfig groups = new GroupConfig(new DateHistogramGroupConfig("date", DateHistogramInterval.DAY));
448+
final GroupConfig groups = new GroupConfig(new DateHistogramGroupConfig.CalendarInterval("date", DateHistogramInterval.DAY));
449449
final List<MetricConfig> metrics = Collections.singletonList(new MetricConfig("value", SUPPORTED_METRICS));
450450
final TimeValue timeout = TimeValue.timeValueSeconds(randomIntBetween(30, 600));
451451

@@ -489,7 +489,7 @@ public void testGetRollupIndexCaps() throws Exception {
489489
case "delay":
490490
assertThat(entry.getValue(), equalTo("foo"));
491491
break;
492-
case "interval":
492+
case "calendar_interval":
493493
assertThat(entry.getValue(), equalTo("1d"));
494494
break;
495495
case "time_zone":

‎client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/RollupDocumentationIT.java

+8-6
Original file line numberDiff line numberDiff line change
@@ -399,8 +399,8 @@ public void onFailure(Exception e) {
399399
public void testGetRollupCaps() throws Exception {
400400
RestHighLevelClient client = highLevelClient();
401401

402-
DateHistogramGroupConfig dateHistogram =
403-
new DateHistogramGroupConfig("timestamp", DateHistogramInterval.HOUR, new DateHistogramInterval("7d"), "UTC"); // <1>
402+
DateHistogramGroupConfig dateHistogram = new DateHistogramGroupConfig.FixedInterval(
403+
"timestamp", DateHistogramInterval.HOUR, new DateHistogramInterval("7d"), "UTC"); // <1>
404404
TermsGroupConfig terms = new TermsGroupConfig("hostname", "datacenter");
405405
HistogramGroupConfig histogram = new HistogramGroupConfig(5L, "load", "net_in", "net_out");
406406
GroupConfig groups = new GroupConfig(dateHistogram, histogram, terms);
@@ -473,7 +473,8 @@ public void testGetRollupCaps() throws Exception {
473473
// item represents a different aggregation that can be run against the "timestamp"
474474
// field, and any additional details specific to that agg (interval, etc)
475475
List<Map<String, Object>> timestampCaps = fieldCaps.get("timestamp").getAggs();
476-
assert timestampCaps.get(0).toString().equals("{agg=date_histogram, delay=7d, interval=1h, time_zone=UTC}");
476+
logger.error(timestampCaps.get(0).toString());
477+
assert timestampCaps.get(0).toString().equals("{agg=date_histogram, fixed_interval=1h, delay=7d, time_zone=UTC}");
477478

478479
// In contrast to the timestamp field, the temperature field has multiple aggs configured
479480
List<Map<String, Object>> temperatureCaps = fieldCaps.get("temperature").getAggs();
@@ -515,8 +516,8 @@ public void onFailure(Exception e) {
515516
public void testGetRollupIndexCaps() throws Exception {
516517
RestHighLevelClient client = highLevelClient();
517518

518-
DateHistogramGroupConfig dateHistogram =
519-
new DateHistogramGroupConfig("timestamp", DateHistogramInterval.HOUR, new DateHistogramInterval("7d"), "UTC"); // <1>
519+
DateHistogramGroupConfig dateHistogram = new DateHistogramGroupConfig.FixedInterval(
520+
"timestamp", DateHistogramInterval.HOUR, new DateHistogramInterval("7d"), "UTC"); // <1>
520521
TermsGroupConfig terms = new TermsGroupConfig("hostname", "datacenter");
521522
HistogramGroupConfig histogram = new HistogramGroupConfig(5L, "load", "net_in", "net_out");
522523
GroupConfig groups = new GroupConfig(dateHistogram, histogram, terms);
@@ -587,7 +588,8 @@ public void testGetRollupIndexCaps() throws Exception {
587588
// item represents a different aggregation that can be run against the "timestamp"
588589
// field, and any additional details specific to that agg (interval, etc)
589590
List<Map<String, Object>> timestampCaps = fieldCaps.get("timestamp").getAggs();
590-
assert timestampCaps.get(0).toString().equals("{agg=date_histogram, delay=7d, interval=1h, time_zone=UTC}");
591+
logger.error(timestampCaps.get(0).toString());
592+
assert timestampCaps.get(0).toString().equals("{agg=date_histogram, fixed_interval=1h, delay=7d, time_zone=UTC}");
591593

592594
// In contrast to the timestamp field, the temperature field has multiple aggs configured
593595
List<Map<String, Object>> temperatureCaps = fieldCaps.get("temperature").getAggs();

‎client/rest-high-level/src/test/java/org/elasticsearch/client/ml/datafeed/DatafeedConfigTests.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import org.elasticsearch.index.query.QueryBuilders;
2929
import org.elasticsearch.search.aggregations.AggregationBuilders;
3030
import org.elasticsearch.search.aggregations.AggregatorFactories;
31+
import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
3132
import org.elasticsearch.search.aggregations.metrics.MaxAggregationBuilder;
3233
import org.elasticsearch.search.builder.SearchSourceBuilder.ScriptField;
3334
import org.elasticsearch.test.AbstractXContentTestCase;
@@ -79,7 +80,7 @@ public static DatafeedConfig.Builder createRandomBuilder() {
7980
aggHistogramInterval = aggHistogramInterval <= 0 ? 1 : aggHistogramInterval;
8081
MaxAggregationBuilder maxTime = AggregationBuilders.max("time").field("time");
8182
aggs.addAggregator(AggregationBuilders.dateHistogram("buckets")
82-
.interval(aggHistogramInterval).subAggregation(maxTime).field("time"));
83+
.fixedInterval(new DateHistogramInterval(aggHistogramInterval + "ms")).subAggregation(maxTime).field("time"));
8384
try {
8485
builder.setAggregations(aggs);
8586
} catch (IOException e) {

‎client/rest-high-level/src/test/java/org/elasticsearch/client/rollup/GetRollupJobResponseTests.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ public void testFromXContent() throws IOException {
4444
this::createTestInstance,
4545
this::toXContent,
4646
GetRollupJobResponse::fromXContent)
47-
.supportsUnknownFields(true)
47+
.supportsUnknownFields(false)
4848
.randomFieldsExcludeFilter(field ->
4949
field.endsWith("status.current_position"))
5050
.test();

‎client/rest-high-level/src/test/java/org/elasticsearch/client/rollup/PutRollupJobRequestTests.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ protected PutRollupJobRequest doParseInstance(final XContentParser parser) throw
4949

5050
@Override
5151
protected boolean supportsUnknownFields() {
52-
return true;
52+
return false;
5353
}
5454

5555
public void testRequireConfiguration() {

0 commit comments

Comments
 (0)