Skip to content

Commit a862dff

Browse files
csoulioscsoulios
authored
Add tests to MedianAbsoluteDeviationAggregator (#54884)
1 parent f1fbb40 commit a862dff

File tree

3 files changed

+170
-61
lines changed

3 files changed

+170
-61
lines changed

‎server/src/test/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationAggregatorTests.java

+142-22
Original file line numberDiff line numberDiff line change
@@ -32,22 +32,37 @@
3232
import org.apache.lucene.search.Query;
3333
import org.apache.lucene.store.Directory;
3434
import org.elasticsearch.common.CheckedConsumer;
35+
import org.elasticsearch.common.settings.Settings;
3536
import org.elasticsearch.index.mapper.MappedFieldType;
3637
import org.elasticsearch.index.mapper.NumberFieldMapper;
38+
import org.elasticsearch.script.MockScriptEngine;
39+
import org.elasticsearch.script.Script;
40+
import org.elasticsearch.script.ScriptEngine;
41+
import org.elasticsearch.script.ScriptModule;
42+
import org.elasticsearch.script.ScriptService;
43+
import org.elasticsearch.script.ScriptType;
44+
import org.elasticsearch.search.aggregations.AggregationBuilder;
45+
import org.elasticsearch.search.aggregations.Aggregator;
3746
import org.elasticsearch.search.aggregations.AggregatorTestCase;
3847
import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper;
48+
import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
49+
import org.elasticsearch.search.aggregations.support.ValuesSourceType;
3950
import org.hamcrest.Description;
4051
import org.hamcrest.TypeSafeMatcher;
4152

4253
import java.io.IOException;
4354
import java.util.ArrayList;
4455
import java.util.Arrays;
56+
import java.util.Collections;
57+
import java.util.HashMap;
4558
import java.util.List;
59+
import java.util.Map;
4660
import java.util.function.Consumer;
4761
import java.util.function.Function;
4862
import java.util.stream.IntStream;
4963

5064
import static java.util.Collections.singleton;
65+
import static java.util.Collections.singletonList;
5166
import static org.elasticsearch.search.aggregations.metrics.MedianAbsoluteDeviationAggregatorTests.ExactMedianAbsoluteDeviation.calculateMAD;
5267
import static org.elasticsearch.search.aggregations.metrics.MedianAbsoluteDeviationAggregatorTests.IsCloseToRelative.closeToRelative;
5368
import static org.hamcrest.Matchers.equalTo;
@@ -56,6 +71,11 @@ public class MedianAbsoluteDeviationAggregatorTests extends AggregatorTestCase {
5671

5772
private static final int SAMPLE_MIN = -1000000;
5873
private static final int SAMPLE_MAX = 1000000;
74+
public static final String FIELD_NAME = "number";
75+
76+
/** Script to return the {@code _value} provided by aggs framework. */
77+
private static final String VALUE_SCRIPT = "_value";
78+
private static final String SINGLE_SCRIPT = "single";
5979

6080
private static <T extends IndexableField> CheckedConsumer<RandomIndexWriter, IOException> randomSample(
6181
int size,
@@ -96,10 +116,10 @@ public void testSomeMatchesSortedNumericDocValues() throws IOException {
96116
final int size = randomIntBetween(100, 1000);
97117
final List<Long> sample = new ArrayList<>(size);
98118
testCase(
99-
new DocValuesFieldExistsQuery("number"),
119+
new DocValuesFieldExistsQuery(FIELD_NAME),
100120
randomSample(size, point -> {
101121
sample.add(point);
102-
return singleton(new SortedNumericDocValuesField("number", point));
122+
return singleton(new SortedNumericDocValuesField(FIELD_NAME, point));
103123
}),
104124
agg -> {
105125
assertThat(agg.getMedianAbsoluteDeviation(), closeToRelative(calculateMAD(sample)));
@@ -112,10 +132,10 @@ public void testSomeMatchesNumericDocValues() throws IOException {
112132
final int size = randomIntBetween(100, 1000);
113133
final List<Long> sample = new ArrayList<>(size);
114134
testCase(
115-
new DocValuesFieldExistsQuery("number"),
135+
new DocValuesFieldExistsQuery(FIELD_NAME),
116136
randomSample(size, point -> {
117137
sample.add(point);
118-
return singleton(new NumericDocValuesField("number", point));
138+
return singleton(new NumericDocValuesField(FIELD_NAME, point));
119139
}),
120140
agg -> {
121141
assertThat(agg.getMedianAbsoluteDeviation(), closeToRelative(calculateMAD(sample)));
@@ -130,10 +150,10 @@ public void testQueryFiltering() throws IOException {
130150
final int[] sample = IntStream.rangeClosed(1, 1000).toArray();
131151
final int[] filteredSample = Arrays.stream(sample).filter(point -> point >= lowerRange && point <= upperRange).toArray();
132152
testCase(
133-
IntPoint.newRangeQuery("number", lowerRange, upperRange),
153+
IntPoint.newRangeQuery(FIELD_NAME, lowerRange, upperRange),
134154
writer -> {
135155
for (int point : sample) {
136-
writer.addDocument(Arrays.asList(new IntPoint("number", point), new SortedNumericDocValuesField("number", point)));
156+
writer.addDocument(Arrays.asList(new IntPoint(FIELD_NAME, point), new SortedNumericDocValuesField(FIELD_NAME, point)));
137157
}
138158
},
139159
agg -> {
@@ -145,10 +165,10 @@ public void testQueryFiltering() throws IOException {
145165

146166
public void testQueryFiltersAll() throws IOException {
147167
testCase(
148-
IntPoint.newRangeQuery("number", -1, 0),
168+
IntPoint.newRangeQuery(FIELD_NAME, -1, 0),
149169
writer -> {
150-
writer.addDocument(Arrays.asList(new IntPoint("number", 1), new SortedNumericDocValuesField("number", 1)));
151-
writer.addDocument(Arrays.asList(new IntPoint("number", 2), new SortedNumericDocValuesField("number", 2)));
170+
writer.addDocument(Arrays.asList(new IntPoint(FIELD_NAME, 1), new SortedNumericDocValuesField(FIELD_NAME, 1)));
171+
writer.addDocument(Arrays.asList(new IntPoint(FIELD_NAME, 2), new SortedNumericDocValuesField(FIELD_NAME, 2)));
152172
},
153173
agg -> {
154174
assertThat(agg.getMedianAbsoluteDeviation(), equalTo(Double.NaN));
@@ -157,34 +177,110 @@ public void testQueryFiltersAll() throws IOException {
157177
);
158178
}
159179

180+
public void testUnmapped() throws IOException {
181+
MedianAbsoluteDeviationAggregationBuilder aggregationBuilder = new MedianAbsoluteDeviationAggregationBuilder("foo")
182+
.field(FIELD_NAME);
183+
184+
testCase(aggregationBuilder, new DocValuesFieldExistsQuery(FIELD_NAME), iw -> {
185+
iw.addDocument(singleton(new NumericDocValuesField(FIELD_NAME, 7)));
186+
iw.addDocument(singleton(new NumericDocValuesField(FIELD_NAME, 1)));
187+
}, agg -> {
188+
assertEquals(Double.NaN, agg.getMedianAbsoluteDeviation(),0);
189+
assertFalse(AggregationInspectionHelper.hasValue(agg));
190+
}, null);
191+
}
192+
193+
public void testUnmappedMissing() throws IOException {
194+
MedianAbsoluteDeviationAggregationBuilder aggregationBuilder = new MedianAbsoluteDeviationAggregationBuilder("foo")
195+
.field(FIELD_NAME)
196+
.missing(1234);
197+
198+
testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> {
199+
iw.addDocument(singleton(new NumericDocValuesField("unrelatedField", 7)));
200+
iw.addDocument(singleton(new NumericDocValuesField("unrelatedField", 8)));
201+
iw.addDocument(singleton(new NumericDocValuesField("unrelatedField", 9)));
202+
}, agg -> {
203+
assertEquals(0, agg.getMedianAbsoluteDeviation(), 0);
204+
assertTrue(AggregationInspectionHelper.hasValue(agg));
205+
}, null);
206+
}
207+
208+
public void testValueScript() throws IOException {
209+
MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
210+
fieldType.setName(FIELD_NAME);
211+
fieldType.setHasDocValues(true);
212+
213+
MedianAbsoluteDeviationAggregationBuilder aggregationBuilder = new MedianAbsoluteDeviationAggregationBuilder("foo")
214+
.field(FIELD_NAME)
215+
.script(new Script(ScriptType.INLINE, MockScriptEngine.NAME, VALUE_SCRIPT, Collections.emptyMap()));
216+
217+
final int size = randomIntBetween(100, 1000);
218+
final List<Long> sample = new ArrayList<>(size);
219+
testCase(aggregationBuilder,
220+
new MatchAllDocsQuery(),
221+
randomSample(size, point -> {
222+
sample.add(point);
223+
return singleton(new SortedNumericDocValuesField(FIELD_NAME, point));
224+
}),
225+
agg -> {
226+
assertThat(agg.getMedianAbsoluteDeviation(), closeToRelative(calculateMAD(sample)));
227+
assertTrue(AggregationInspectionHelper.hasValue(agg));
228+
}, fieldType);
229+
}
230+
231+
public void testSingleScript() throws IOException {
232+
MedianAbsoluteDeviationAggregationBuilder aggregationBuilder = new MedianAbsoluteDeviationAggregationBuilder("foo")
233+
.script(new Script(ScriptType.INLINE, MockScriptEngine.NAME, SINGLE_SCRIPT, Collections.emptyMap()));
234+
235+
MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
236+
fieldType.setName(FIELD_NAME);
237+
238+
final int size = randomIntBetween(100, 1000);
239+
final List<Long> sample = new ArrayList<>(size);
240+
testCase(aggregationBuilder,
241+
new MatchAllDocsQuery(),
242+
iw -> {
243+
for (int i = 0; i < 10; i++) {
244+
iw.addDocument(singleton(new NumericDocValuesField(FIELD_NAME, i + 1)));
245+
}
246+
},
247+
agg -> {
248+
assertEquals(0, agg.getMedianAbsoluteDeviation(), 0);
249+
assertTrue(AggregationInspectionHelper.hasValue(agg));
250+
}, fieldType);
251+
}
252+
160253
private void testCase(Query query,
161-
CheckedConsumer<RandomIndexWriter, IOException> buildIndex,
254+
CheckedConsumer<RandomIndexWriter,
255+
IOException> buildIndex,
162256
Consumer<InternalMedianAbsoluteDeviation> verify) throws IOException {
257+
MedianAbsoluteDeviationAggregationBuilder builder = new MedianAbsoluteDeviationAggregationBuilder("mad")
258+
.field(FIELD_NAME)
259+
.compression(randomDoubleBetween(20, 1000, true));
260+
261+
MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
262+
fieldType.setName(FIELD_NAME);
263+
264+
testCase(builder, query, buildIndex, verify, fieldType);
265+
}
163266

267+
private void testCase(MedianAbsoluteDeviationAggregationBuilder aggregationBuilder, Query query,
268+
CheckedConsumer<RandomIndexWriter, IOException> indexer,
269+
Consumer<InternalMedianAbsoluteDeviation> verify, MappedFieldType fieldType) throws IOException {
164270
try (Directory directory = newDirectory()) {
165271
try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) {
166-
buildIndex.accept(indexWriter);
272+
indexer.accept(indexWriter);
167273
}
168274

169275
try (IndexReader indexReader = DirectoryReader.open(directory)) {
170276
IndexSearcher indexSearcher = newSearcher(indexReader, true, true);
171-
172-
MedianAbsoluteDeviationAggregationBuilder builder = new MedianAbsoluteDeviationAggregationBuilder("mad")
173-
.field("number")
174-
.compression(randomDoubleBetween(20, 1000, true));
175-
176-
MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
177-
fieldType.setName("number");
178-
179-
MedianAbsoluteDeviationAggregator aggregator = createAggregator(builder, indexSearcher, fieldType);
277+
Aggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType);
180278
aggregator.preCollection();
181279
indexSearcher.search(query, aggregator);
182280
aggregator.postCollection();
183-
184281
verify.accept((InternalMedianAbsoluteDeviation) aggregator.buildAggregation(0L));
185282
}
186283
}
187-
188284
}
189285

190286
public static class IsCloseToRelative extends TypeSafeMatcher<Double> {
@@ -271,6 +367,30 @@ private static double calculateMedian(double[] sample) {
271367
}
272368
return median;
273369
}
370+
}
371+
372+
@Override
373+
protected List<ValuesSourceType> getSupportedValuesSourceTypes() {
374+
return singletonList(CoreValuesSourceType.NUMERIC);
375+
}
376+
377+
@Override
378+
protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldType, String fieldName) {
379+
return new MedianAbsoluteDeviationAggregationBuilder("foo").field(fieldName);
380+
}
381+
382+
@Override
383+
protected ScriptService getMockScriptService() {
384+
Map<String, Function<Map<String, Object>, Object>> scripts = new HashMap<>();
385+
386+
scripts.put(VALUE_SCRIPT, vars -> ((Number) vars.get("_value")).doubleValue() + 1);
387+
scripts.put(SINGLE_SCRIPT, vars -> 1);
388+
389+
MockScriptEngine scriptEngine = new MockScriptEngine(MockScriptEngine.NAME,
390+
scripts,
391+
Collections.emptyMap());
392+
Map<String, ScriptEngine> engines = Collections.singletonMap(scriptEngine.getType(), scriptEngine);
274393

394+
return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS);
275395
}
276396
}

‎server/src/test/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationIT.java

+1-14
Original file line numberDiff line numberDiff line change
@@ -182,20 +182,7 @@ public void testEmptyAggregation() throws Exception {
182182

183183
@Override
184184
public void testUnmapped() throws Exception {
185-
final SearchResponse response = client()
186-
.prepareSearch("idx_unmapped")
187-
.setQuery(matchAllQuery())
188-
.addAggregation(
189-
randomBuilder()
190-
.field("value"))
191-
.get();
192-
193-
assertHitCount(response, 0);
194-
195-
final MedianAbsoluteDeviation mad = response.getAggregations().get("mad");
196-
assertThat(mad, notNullValue());
197-
assertThat(mad.getName(), is("mad"));
198-
assertThat(mad.getMedianAbsoluteDeviation(), is(Double.NaN));
185+
// Test moved to MedianAbsoluteDeviationAggregatorTests.testUnmapped()
199186
}
200187

201188
@Override

‎server/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountAggregatorTests.java

+27-25
Original file line numberDiff line numberDiff line change
@@ -218,9 +218,9 @@ public void testUnmappedMissingString() throws IOException {
218218
iw.addDocument(singleton(new NumericDocValuesField("unrelatedField", 7)));
219219
iw.addDocument(singleton(new NumericDocValuesField("unrelatedField", 8)));
220220
iw.addDocument(singleton(new NumericDocValuesField("unrelatedField", 9)));
221-
}, card -> {
222-
assertEquals(3, card.getValue(), 0);
223-
assertTrue(AggregationInspectionHelper.hasValue(card));
221+
}, valueCount -> {
222+
assertEquals(3, valueCount.getValue(), 0);
223+
assertTrue(AggregationInspectionHelper.hasValue(valueCount));
224224
}, null);
225225
}
226226

@@ -232,9 +232,9 @@ public void testUnmappedMissingNumber() throws IOException {
232232
iw.addDocument(singleton(new NumericDocValuesField("unrelatedField", 7)));
233233
iw.addDocument(singleton(new NumericDocValuesField("unrelatedField", 8)));
234234
iw.addDocument(singleton(new NumericDocValuesField("unrelatedField", 9)));
235-
}, card -> {
236-
assertEquals(3, card.getValue(), 0);
237-
assertTrue(AggregationInspectionHelper.hasValue(card));
235+
}, valueCount -> {
236+
assertEquals(3, valueCount.getValue(), 0);
237+
assertTrue(AggregationInspectionHelper.hasValue(valueCount));
238238
}, null);
239239
}
240240

@@ -246,9 +246,9 @@ public void testUnmappedMissingGeoPoint() throws IOException {
246246
iw.addDocument(singleton(new NumericDocValuesField("unrelatedField", 7)));
247247
iw.addDocument(singleton(new NumericDocValuesField("unrelatedField", 8)));
248248
iw.addDocument(singleton(new NumericDocValuesField("unrelatedField", 9)));
249-
}, card -> {
250-
assertEquals(3, card.getValue(), 0);
251-
assertTrue(AggregationInspectionHelper.hasValue(card));
249+
}, valueCount -> {
250+
assertEquals(3, valueCount.getValue(), 0);
251+
assertTrue(AggregationInspectionHelper.hasValue(valueCount));
252252
}, null);
253253
}
254254

@@ -284,15 +284,15 @@ public void testValueScriptNumber() throws IOException {
284284
iw.addDocument(singleton(new NumericDocValuesField(FIELD_NAME, 7)));
285285
iw.addDocument(singleton(new NumericDocValuesField(FIELD_NAME, 8)));
286286
iw.addDocument(singleton(new NumericDocValuesField(FIELD_NAME, 9)));
287-
}, card -> {
288-
assertEquals(3, card.getValue(), 0);
289-
assertTrue(AggregationInspectionHelper.hasValue(card));
287+
}, valueCount -> {
288+
assertEquals(3, valueCount.getValue(), 0);
289+
assertTrue(AggregationInspectionHelper.hasValue(valueCount));
290290
}, fieldType);
291291
}
292292

293293
public void testSingleScriptNumber() throws IOException {
294294
ValueCountAggregationBuilder aggregationBuilder = new ValueCountAggregationBuilder("name")
295-
.field(FIELD_NAME);
295+
.script(new Script(ScriptType.INLINE, MockScriptEngine.NAME, SINGLE_SCRIPT, Collections.emptyMap()));
296296

297297
MappedFieldType fieldType = createMappedFieldType(ValueType.NUMERIC);
298298
fieldType.setName(FIELD_NAME);
@@ -313,10 +313,11 @@ public void testSingleScriptNumber() throws IOException {
313313
doc.add(new SortedNumericDocValuesField(FIELD_NAME, 1));
314314
doc.add(new SortedNumericDocValuesField(FIELD_NAME, 1));
315315
iw.addDocument(doc);
316-
}, card -> {
317-
// note: this is 6, even though the script returns a single value. ValueCount does not de-dedupe
318-
assertEquals(6, card.getValue(), 0);
319-
assertTrue(AggregationInspectionHelper.hasValue(card));
316+
}, valueCount -> {
317+
// Note: The field values won't be taken into account. The script will only be called
318+
// once per document, and only expect a count of 3
319+
assertEquals(3, valueCount.getValue(), 0);
320+
assertTrue(AggregationInspectionHelper.hasValue(valueCount));
320321
}, fieldType);
321322
}
322323

@@ -333,15 +334,15 @@ public void testValueScriptString() throws IOException {
333334
iw.addDocument(singleton(new SortedDocValuesField(FIELD_NAME, new BytesRef("1"))));
334335
iw.addDocument(singleton(new SortedDocValuesField(FIELD_NAME, new BytesRef("2"))));
335336
iw.addDocument(singleton(new SortedDocValuesField(FIELD_NAME, new BytesRef("3"))));
336-
}, card -> {
337-
assertEquals(3, card.getValue(), 0);
338-
assertTrue(AggregationInspectionHelper.hasValue(card));
337+
}, valueCount -> {
338+
assertEquals(3, valueCount.getValue(), 0);
339+
assertTrue(AggregationInspectionHelper.hasValue(valueCount));
339340
}, fieldType);
340341
}
341342

342343
public void testSingleScriptString() throws IOException {
343344
ValueCountAggregationBuilder aggregationBuilder = new ValueCountAggregationBuilder("name")
344-
.field(FIELD_NAME);
345+
.script(new Script(ScriptType.INLINE, MockScriptEngine.NAME, SINGLE_SCRIPT, Collections.emptyMap()));
345346

346347
MappedFieldType fieldType = createMappedFieldType(ValueType.STRING);
347348
fieldType.setName(FIELD_NAME);
@@ -363,10 +364,11 @@ public void testSingleScriptString() throws IOException {
363364
doc.add(new SortedSetDocValuesField(FIELD_NAME, new BytesRef("5")));
364365
doc.add(new SortedSetDocValuesField(FIELD_NAME, new BytesRef("6")));
365366
iw.addDocument(doc);
366-
}, card -> {
367-
// note: this is 6, even though the script returns a single value. ValueCount does not de-dedupe
368-
assertEquals(6, card.getValue(), 0);
369-
assertTrue(AggregationInspectionHelper.hasValue(card));
367+
}, valueCount -> {
368+
// Note: The field values won't be taken into account. The script will only be called
369+
// once per document, and only expect a count of 3
370+
assertEquals(3, valueCount.getValue(), 0);
371+
assertTrue(AggregationInspectionHelper.hasValue(valueCount));
370372
}, fieldType);
371373
}
372374

0 commit comments

Comments
 (0)