Skip to content

Commit 85fa2bd

Browse files
not-napoleonnot-napoleon
authored
Unit tests for Range and DateRange aggs (#52380)
1 parent 7722360 commit 85fa2bd

File tree

2 files changed

+615
-0
lines changed

2 files changed

+615
-0
lines changed

‎server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/DateRangeAggregatorTests.java

Lines changed: 307 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,307 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.elasticsearch.search.aggregations.bucket.range;
21+
22+
import org.apache.lucene.document.NumericDocValuesField;
23+
import org.apache.lucene.document.SortedNumericDocValuesField;
24+
import org.apache.lucene.document.SortedSetDocValuesField;
25+
import org.apache.lucene.index.DirectoryReader;
26+
import org.apache.lucene.index.IndexReader;
27+
import org.apache.lucene.index.RandomIndexWriter;
28+
import org.apache.lucene.search.IndexSearcher;
29+
import org.apache.lucene.search.MatchAllDocsQuery;
30+
import org.apache.lucene.search.Query;
31+
import org.apache.lucene.store.Directory;
32+
import org.apache.lucene.util.BytesRef;
33+
import org.elasticsearch.ElasticsearchParseException;
34+
import org.elasticsearch.common.CheckedConsumer;
35+
import org.elasticsearch.index.mapper.DateFieldMapper;
36+
import org.elasticsearch.index.mapper.KeywordFieldMapper;
37+
import org.elasticsearch.index.mapper.MappedFieldType;
38+
import org.elasticsearch.index.mapper.NumberFieldMapper;
39+
import org.elasticsearch.search.aggregations.AggregatorTestCase;
40+
import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper;
41+
42+
import java.io.IOException;
43+
import java.time.ZoneOffset;
44+
import java.time.ZonedDateTime;
45+
import java.util.List;
46+
import java.util.function.Consumer;
47+
48+
import static java.util.Collections.singleton;
49+
50+
public class DateRangeAggregatorTests extends AggregatorTestCase {
51+
52+
private String NUMBER_FIELD_NAME = "number";
53+
private String UNMAPPED_FIELD_NAME = "field_not_appearing_in_this_index";
54+
private String DATE_FIELD_NAME = "date";
55+
56+
private long milli1 = ZonedDateTime.of(2015, 11, 13, 16, 14, 34, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
57+
private long milli2 = ZonedDateTime.of(2016, 11, 13, 16, 14, 34, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
58+
59+
public void testNoMatchingField() throws IOException {
60+
testBothResolutions(new MatchAllDocsQuery(), iw -> {
61+
iw.addDocument(singleton(new SortedNumericDocValuesField("bogus_field_name", 7)));
62+
iw.addDocument(singleton(new SortedNumericDocValuesField("bogus_field_name", 2)));
63+
iw.addDocument(singleton(new SortedNumericDocValuesField("bogus_field_name", 3)));
64+
}, range -> {
65+
List<? extends InternalRange.Bucket> ranges = range.getBuckets();
66+
assertEquals(2, ranges.size());
67+
assertEquals(0, ranges.get(0).getDocCount());
68+
assertEquals(0, ranges.get(1).getDocCount());
69+
assertFalse(AggregationInspectionHelper.hasValue(range));
70+
});
71+
}
72+
73+
public void testMatchesSortedNumericDocValues() throws IOException {
74+
testBothResolutions(new MatchAllDocsQuery(), iw -> {
75+
iw.addDocument(singleton(new SortedNumericDocValuesField(DATE_FIELD_NAME, milli1)));
76+
iw.addDocument(singleton(new SortedNumericDocValuesField(DATE_FIELD_NAME, milli2)));
77+
}, range -> {
78+
List<? extends InternalRange.Bucket> ranges = range.getBuckets();
79+
assertEquals(2, ranges.size());
80+
assertEquals(1, ranges.get(0).getDocCount());
81+
assertEquals(0, ranges.get(1).getDocCount());
82+
assertTrue(AggregationInspectionHelper.hasValue(range));
83+
});
84+
}
85+
86+
public void testMatchesNumericDocValues() throws IOException {
87+
testBothResolutions(new MatchAllDocsQuery(), iw -> {
88+
iw.addDocument(singleton(new NumericDocValuesField(DATE_FIELD_NAME, milli1)));
89+
iw.addDocument(singleton(new NumericDocValuesField(DATE_FIELD_NAME, milli2)));
90+
}, range -> {
91+
List<? extends InternalRange.Bucket> ranges = range.getBuckets();
92+
assertEquals(2, ranges.size());
93+
assertEquals(1, ranges.get(0).getDocCount());
94+
assertEquals(0, ranges.get(1).getDocCount());
95+
assertTrue(AggregationInspectionHelper.hasValue(range));
96+
});
97+
}
98+
99+
public void testMissingDateStringWithDateField() throws IOException {
100+
DateFieldMapper.Builder builder = new DateFieldMapper.Builder(DATE_FIELD_NAME)
101+
.withResolution(DateFieldMapper.Resolution.MILLISECONDS);
102+
DateFieldMapper.DateFieldType fieldType = builder.fieldType();
103+
fieldType.setHasDocValues(true);
104+
fieldType.setName(DATE_FIELD_NAME);
105+
106+
DateRangeAggregationBuilder aggregationBuilder = new DateRangeAggregationBuilder("date_range")
107+
.field(DATE_FIELD_NAME)
108+
.missing("2015-11-13T16:14:34")
109+
.addRange("2015-11-13", "2015-11-14");
110+
111+
testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> {
112+
iw.addDocument(singleton(new SortedNumericDocValuesField(DATE_FIELD_NAME, milli1)));
113+
iw.addDocument(singleton(new SortedNumericDocValuesField(DATE_FIELD_NAME, milli2)));
114+
// Missing will apply to this document
115+
iw.addDocument(singleton(new SortedNumericDocValuesField(NUMBER_FIELD_NAME, 7)));
116+
}, range -> {
117+
List<? extends InternalRange.Bucket> ranges = range.getBuckets();
118+
assertEquals(1, ranges.size());
119+
assertEquals(2, ranges.get(0).getDocCount());
120+
assertTrue(AggregationInspectionHelper.hasValue(range));
121+
}, fieldType);
122+
}
123+
124+
public void testNumberFieldDateRanges() throws IOException {
125+
DateRangeAggregationBuilder aggregationBuilder = new DateRangeAggregationBuilder("date_range")
126+
.field(NUMBER_FIELD_NAME)
127+
.addRange("2015-11-13", "2015-11-14");
128+
129+
MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.INTEGER);
130+
fieldType.setName(NUMBER_FIELD_NAME);
131+
132+
expectThrows(NumberFormatException.class,
133+
() -> testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> {
134+
iw.addDocument(singleton(new NumericDocValuesField(NUMBER_FIELD_NAME, 7)));
135+
iw.addDocument(singleton(new NumericDocValuesField(NUMBER_FIELD_NAME, 1)));
136+
}, range -> fail("Should have thrown exception"), fieldType));
137+
}
138+
139+
public void testNumberFieldNumberRanges() throws IOException {
140+
DateRangeAggregationBuilder aggregationBuilder = new DateRangeAggregationBuilder("date_range")
141+
.field(NUMBER_FIELD_NAME)
142+
.addRange(0, 5);
143+
144+
MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.INTEGER);
145+
fieldType.setName(NUMBER_FIELD_NAME);
146+
147+
testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> {
148+
iw.addDocument(singleton(new NumericDocValuesField(NUMBER_FIELD_NAME, 7)));
149+
iw.addDocument(singleton(new NumericDocValuesField(NUMBER_FIELD_NAME, 1)));
150+
}, range -> {
151+
List<? extends InternalRange.Bucket> ranges = range.getBuckets();
152+
assertEquals(1, ranges.size());
153+
assertEquals(1, ranges.get(0).getDocCount());
154+
assertTrue(AggregationInspectionHelper.hasValue(range));
155+
}, fieldType);
156+
}
157+
158+
public void testMissingDateStringWithNumberField() throws IOException {
159+
DateRangeAggregationBuilder aggregationBuilder = new DateRangeAggregationBuilder("date_range")
160+
.field(NUMBER_FIELD_NAME)
161+
.addRange("2015-11-13", "2015-11-14")
162+
.missing("1979-01-01T00:00:00");
163+
164+
MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.INTEGER);
165+
fieldType.setName(NUMBER_FIELD_NAME);
166+
167+
expectThrows(NumberFormatException.class,
168+
() -> testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> {
169+
iw.addDocument(singleton(new NumericDocValuesField(NUMBER_FIELD_NAME, 7)));
170+
iw.addDocument(singleton(new NumericDocValuesField(NUMBER_FIELD_NAME, 1)));
171+
}, range -> fail("Should have thrown exception"), fieldType));
172+
}
173+
174+
public void testUnmappedWithMissingNumber() throws IOException {
175+
DateRangeAggregationBuilder aggregationBuilder = new DateRangeAggregationBuilder("date_range")
176+
.field("does_not_exist")
177+
.addRange("2015-11-13", "2015-11-14")
178+
.missing(1447438575000L); // 2015-11-13 6:16:15
179+
180+
MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.INTEGER);
181+
fieldType.setName(NUMBER_FIELD_NAME);
182+
183+
testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> {
184+
iw.addDocument(singleton(new NumericDocValuesField(NUMBER_FIELD_NAME, 7)));
185+
iw.addDocument(singleton(new NumericDocValuesField(NUMBER_FIELD_NAME, 1)));
186+
}, range -> {
187+
List<? extends InternalRange.Bucket> ranges = range.getBuckets();
188+
assertEquals(1, ranges.size());
189+
assertEquals(2, ranges.get(0).getDocCount());
190+
assertTrue(AggregationInspectionHelper.hasValue(range));
191+
}, fieldType);
192+
}
193+
194+
public void testUnmappedWithMissingDate() throws IOException {
195+
DateRangeAggregationBuilder aggregationBuilder = new DateRangeAggregationBuilder("date_range")
196+
.field("does_not_exist")
197+
.addRange("2015-11-13", "2015-11-14")
198+
.missing("2015-11-13T10:11:12");
199+
200+
MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.INTEGER);
201+
fieldType.setName(NUMBER_FIELD_NAME);
202+
203+
testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> {
204+
iw.addDocument(singleton(new NumericDocValuesField(NUMBER_FIELD_NAME, 7)));
205+
iw.addDocument(singleton(new NumericDocValuesField(NUMBER_FIELD_NAME, 1)));
206+
}, range -> {
207+
List<? extends InternalRange.Bucket> ranges = range.getBuckets();
208+
assertEquals(1, ranges.size());
209+
assertEquals(2, ranges.get(0).getDocCount());
210+
assertTrue(AggregationInspectionHelper.hasValue(range));
211+
}, fieldType);
212+
}
213+
214+
public void testKeywordField() {
215+
DateRangeAggregationBuilder aggregationBuilder = new DateRangeAggregationBuilder("date_range")
216+
.field("not_a_number")
217+
.addRange("2015-11-13", "2015-11-14");
218+
219+
MappedFieldType fieldType = new KeywordFieldMapper.KeywordFieldType();
220+
fieldType.setName("not_a_number");
221+
fieldType.setHasDocValues(true);
222+
223+
IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
224+
() -> testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> {
225+
iw.addDocument(singleton(new SortedSetDocValuesField("string", new BytesRef("foo"))));
226+
}, range -> fail("Should have thrown exception"), fieldType));
227+
// I believe this error is coming from improperly parsing the range, not the field.
228+
assertEquals("For input string: \"2015-11-13\"", e.getMessage());
229+
}
230+
231+
public void testBadMissingField() {
232+
DateRangeAggregationBuilder aggregationBuilder = new DateRangeAggregationBuilder("date_range")
233+
.field(NUMBER_FIELD_NAME)
234+
.addRange("2020-01-01T00:00:00", "2020-01-02T00:00:00")
235+
.missing("bogus");
236+
237+
MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.INTEGER);
238+
fieldType.setName(NUMBER_FIELD_NAME);
239+
240+
expectThrows(NumberFormatException.class,
241+
() -> testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> {
242+
iw.addDocument(singleton(new NumericDocValuesField(NUMBER_FIELD_NAME, 7)));
243+
iw.addDocument(singleton(new NumericDocValuesField(NUMBER_FIELD_NAME, 1)));
244+
}, range -> fail("Should have thrown exception"), fieldType));
245+
}
246+
247+
public void testUnmappedWithBadMissingField() {
248+
DateRangeAggregationBuilder aggregationBuilder = new DateRangeAggregationBuilder("date_range")
249+
.field("does_not_exist")
250+
.addRange("2020-01-01T00:00:00", "2020-01-02T00:00:00")
251+
.missing("bogus");
252+
253+
MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.INTEGER);
254+
fieldType.setName(NUMBER_FIELD_NAME);
255+
256+
expectThrows(ElasticsearchParseException.class,
257+
() -> testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> {
258+
iw.addDocument(singleton(new NumericDocValuesField(NUMBER_FIELD_NAME, 7)));
259+
iw.addDocument(singleton(new NumericDocValuesField(NUMBER_FIELD_NAME, 1)));
260+
}, range -> fail("Should have thrown exception"), fieldType));
261+
}
262+
263+
private void testBothResolutions(Query query,
264+
CheckedConsumer<RandomIndexWriter, IOException> buildIndex,
265+
Consumer<InternalRange<? extends InternalRange.Bucket, ? extends InternalRange>> verify)
266+
throws IOException {
267+
testCase(query, buildIndex, verify, DateFieldMapper.Resolution.MILLISECONDS);
268+
testCase(query, buildIndex, verify, DateFieldMapper.Resolution.NANOSECONDS);
269+
}
270+
271+
private void testCase(Query query,
272+
CheckedConsumer<RandomIndexWriter, IOException> buildIndex,
273+
Consumer<InternalRange<? extends InternalRange.Bucket, ? extends InternalRange>> verify,
274+
DateFieldMapper.Resolution resolution) throws IOException {
275+
DateFieldMapper.Builder builder = new DateFieldMapper.Builder(DATE_FIELD_NAME)
276+
.withResolution(resolution);
277+
DateFieldMapper.DateFieldType fieldType = builder.fieldType();
278+
fieldType.setHasDocValues(true);
279+
fieldType.setName(DATE_FIELD_NAME);
280+
DateRangeAggregationBuilder aggregationBuilder = new DateRangeAggregationBuilder("test_range_agg");
281+
aggregationBuilder.field(DATE_FIELD_NAME);
282+
aggregationBuilder.addRange("2015-01-01", "2015-12-31");
283+
aggregationBuilder.addRange("2019-01-01", "2019-12-31");
284+
testCase(aggregationBuilder, query, buildIndex, verify, fieldType);
285+
}
286+
287+
private void testCase(DateRangeAggregationBuilder aggregationBuilder,
288+
Query query,
289+
CheckedConsumer<RandomIndexWriter, IOException> buildIndex,
290+
Consumer<InternalRange<? extends InternalRange.Bucket, ? extends InternalRange>> verify,
291+
MappedFieldType fieldType) throws IOException {
292+
try (Directory directory = newDirectory()) {
293+
RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory);
294+
buildIndex.accept(indexWriter);
295+
indexWriter.close();
296+
297+
try (IndexReader indexReader = DirectoryReader.open(directory)) {
298+
IndexSearcher indexSearcher = newSearcher(indexReader, true, true);
299+
300+
InternalRange<? extends InternalRange.Bucket, ? extends InternalRange> agg = searchAndReduce(indexSearcher,
301+
query, aggregationBuilder, fieldType);
302+
verify.accept(agg);
303+
304+
}
305+
}
306+
}
307+
}

0 commit comments

Comments
 (0)