|
| 1 | +package org.elasticsearch.xpack.transform.integration.continuous; |
| 2 | + |
| 3 | +import org.elasticsearch.action.search.SearchRequest; |
| 4 | +import org.elasticsearch.action.search.SearchResponse; |
| 5 | +import org.elasticsearch.action.support.IndicesOptions; |
| 6 | +import org.elasticsearch.client.transform.transforms.DestConfig; |
| 7 | +import org.elasticsearch.client.transform.transforms.SourceConfig; |
| 8 | +import org.elasticsearch.client.transform.transforms.TransformConfig; |
| 9 | +import org.elasticsearch.client.transform.transforms.pivot.GroupConfig; |
| 10 | +import org.elasticsearch.client.transform.transforms.pivot.HistogramGroupSource; |
| 11 | +import org.elasticsearch.client.transform.transforms.pivot.PivotConfig; |
| 12 | +import org.elasticsearch.common.xcontent.support.XContentMapValues; |
| 13 | +import org.elasticsearch.search.SearchHit; |
| 14 | +import org.elasticsearch.search.aggregations.AggregatorFactories; |
| 15 | +import org.elasticsearch.search.aggregations.BucketOrder; |
| 16 | +import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; |
| 17 | +import org.elasticsearch.search.aggregations.bucket.histogram.Histogram.Bucket; |
| 18 | +import org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregationBuilder; |
| 19 | +import org.elasticsearch.search.builder.SearchSourceBuilder; |
| 20 | + |
| 21 | +import java.io.IOException; |
| 22 | +import java.util.Iterator; |
| 23 | +import java.util.List; |
| 24 | +import java.util.Map; |
| 25 | + |
| 26 | +import static org.hamcrest.Matchers.equalTo; |
| 27 | + |
| 28 | +public class HistogramGroupByIT extends ContinuousTestCase { |
| 29 | + private static final String NAME = "continuous-histogram-pivot-test"; |
| 30 | + |
| 31 | + @Override |
| 32 | + public String getName() { |
| 33 | + return NAME; |
| 34 | + } |
| 35 | + |
| 36 | + @Override |
| 37 | + public TransformConfig createConfig() { |
| 38 | + TransformConfig.Builder transformConfigBuilder = new TransformConfig.Builder(); |
| 39 | + addCommonBuilderParameters(transformConfigBuilder); |
| 40 | + transformConfigBuilder.setSource(new SourceConfig(CONTINUOUS_EVENTS_SOURCE_INDEX)); |
| 41 | + transformConfigBuilder.setDest(new DestConfig(NAME, INGEST_PIPELINE)); |
| 42 | + transformConfigBuilder.setId(NAME); |
| 43 | + PivotConfig.Builder pivotConfigBuilder = new PivotConfig.Builder(); |
| 44 | + pivotConfigBuilder.setGroups( |
| 45 | + new GroupConfig.Builder().groupBy("metric", new HistogramGroupSource.Builder().setField("metric").setInterval(50.0).build()) |
| 46 | + .build() |
| 47 | + ); |
| 48 | + AggregatorFactories.Builder aggregations = new AggregatorFactories.Builder(); |
| 49 | + addCommonAggregations(aggregations); |
| 50 | + |
| 51 | + pivotConfigBuilder.setAggregations(aggregations); |
| 52 | + transformConfigBuilder.setPivotConfig(pivotConfigBuilder.build()); |
| 53 | + return transformConfigBuilder.build(); |
| 54 | + } |
| 55 | + |
| 56 | + @Override |
| 57 | + public void testIteration(int iteration) throws IOException { |
| 58 | + SearchRequest searchRequestSource = new SearchRequest(CONTINUOUS_EVENTS_SOURCE_INDEX).allowPartialSearchResults(false) |
| 59 | + .indicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN); |
| 60 | + SearchSourceBuilder sourceBuilderSource = new SearchSourceBuilder().size(0); |
| 61 | + HistogramAggregationBuilder metricBuckets = new HistogramAggregationBuilder("metric").field("metric") |
| 62 | + .interval(50.0) |
| 63 | + .order(BucketOrder.key(true)); |
| 64 | + sourceBuilderSource.aggregation(metricBuckets); |
| 65 | + searchRequestSource.source(sourceBuilderSource); |
| 66 | + SearchResponse responseSource = search(searchRequestSource); |
| 67 | + |
| 68 | + SearchRequest searchRequestDest = new SearchRequest(NAME).allowPartialSearchResults(false) |
| 69 | + .indicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN); |
| 70 | + SearchSourceBuilder sourceBuilderDest = new SearchSourceBuilder().size(10000).sort("metric"); |
| 71 | + searchRequestDest.source(sourceBuilderDest); |
| 72 | + SearchResponse responseDest = search(searchRequestDest); |
| 73 | + |
| 74 | + List<? extends Bucket> buckets = ((Histogram) responseSource.getAggregations().get("metric")).getBuckets(); |
| 75 | + |
| 76 | + Iterator<? extends Bucket> sourceIterator = buckets.iterator(); |
| 77 | + Iterator<SearchHit> destIterator = responseDest.getHits().iterator(); |
| 78 | + |
| 79 | + while (sourceIterator.hasNext() && destIterator.hasNext()) { |
| 80 | + Bucket bucket = sourceIterator.next(); |
| 81 | + SearchHit searchHit = destIterator.next(); |
| 82 | + Map<String, Object> source = searchHit.getSourceAsMap(); |
| 83 | + |
| 84 | + Long transformBucketKey = ((Integer) XContentMapValues.extractValue("metric", source)).longValue(); |
| 85 | + |
| 86 | + // aggs return buckets with 0 doc_count while composite aggs skip over them |
| 87 | + while (bucket.getDocCount() == 0L) { |
| 88 | + assertTrue(sourceIterator.hasNext()); |
| 89 | + bucket = sourceIterator.next(); |
| 90 | + } |
| 91 | + long bucketKey = ((Double) bucket.getKey()).longValue(); |
| 92 | + |
| 93 | + // test correctness, the results from the aggregation and the results from the transform should be the same |
| 94 | + assertThat( |
| 95 | + "Buckets did not match, source: " + source + ", expected: " + bucketKey + ", iteration: " + iteration, |
| 96 | + transformBucketKey, |
| 97 | + equalTo(bucketKey) |
| 98 | + ); |
| 99 | + assertThat( |
| 100 | + "Doc count did not match, source: " + source + ", expected: " + bucket.getDocCount() + ", iteration: " + iteration, |
| 101 | + ((Integer) XContentMapValues.extractValue("count", source)).longValue(), |
| 102 | + equalTo(bucket.getDocCount()) |
| 103 | + ); |
| 104 | + |
| 105 | + // TODO: gh#63801 transform is not optimized for histogram it, it should only rewrite documents that require it |
| 106 | + } |
| 107 | + |
| 108 | + assertFalse(sourceIterator.hasNext()); |
| 109 | + assertFalse(destIterator.hasNext()); |
| 110 | + } |
| 111 | + |
| 112 | +} |
0 commit comments