Skip to content

Commit 4e48f03

Browse files
nik9000nik9000
committed
Add size support to top_metrics (backport of elastic#52662)
This adds support for returning the top "n" metrics instead of just the very top. Relates to elastic#51813
1 parent 2d01c00 commit 4e48f03

File tree

45 files changed

+1693
-553
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

45 files changed

+1693
-553
lines changed

‎client/rest-high-level/src/main/java/org/elasticsearch/client/analytics/TopMetricsAggregationBuilder.java

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,17 +48,20 @@ public class TopMetricsAggregationBuilder extends AbstractAggregationBuilder<Top
4848
public static final String NAME = "top_metrics";
4949

5050
private final SortBuilder<?> sort;
51+
private final int size;
5152
private final String metric;
5253

5354
/**
5455
* Build the request.
5556
* @param name the name of the metric
5657
* @param sort the sort key used to select the top metrics
58+
* @param size number of results to return per bucket
5759
* @param metric the name of the field to select
5860
*/
59-
public TopMetricsAggregationBuilder(String name, SortBuilder<?> sort, String metric) {
61+
public TopMetricsAggregationBuilder(String name, SortBuilder<?> sort, int size, String metric) {
6062
super(name);
6163
this.sort = sort;
64+
this.size = size;
6265
this.metric = metric;
6366
}
6467

@@ -74,6 +77,7 @@ protected XContentBuilder internalXContent(XContentBuilder builder, Params param
7477
builder.startArray("sort");
7578
sort.toXContent(builder, params);
7679
builder.endArray();
80+
builder.field("size", size);
7781
builder.startObject("metric").field("field", metric).endObject();
7882
}
7983
return builder.endObject();

‎client/rest-high-level/src/test/java/org/elasticsearch/client/analytics/AnalyticsAggsIT.java

Lines changed: 26 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -61,19 +61,39 @@ public void testStringStats() throws IOException {
6161
assertThat(stats.getDistribution(), hasEntry(equalTo("t"), closeTo(.09, .005)));
6262
}
6363

64-
public void testBasic() throws IOException {
65-
BulkRequest bulk = new BulkRequest("test").setRefreshPolicy(RefreshPolicy.IMMEDIATE);
66-
bulk.add(new IndexRequest().source(XContentType.JSON, "s", 1, "v", 2));
67-
bulk.add(new IndexRequest().source(XContentType.JSON, "s", 2, "v", 3));
68-
highLevelClient().bulk(bulk, RequestOptions.DEFAULT);
64+
public void testTopMetricsSizeOne() throws IOException {
65+
indexTopMetricsData();
6966
SearchRequest search = new SearchRequest("test");
7067
search.source().aggregation(new TopMetricsAggregationBuilder(
71-
"test", new FieldSortBuilder("s").order(SortOrder.DESC), "v"));
68+
"test", new FieldSortBuilder("s").order(SortOrder.DESC), 1, "v"));
7269
SearchResponse response = highLevelClient().search(search, RequestOptions.DEFAULT);
7370
ParsedTopMetrics top = response.getAggregations().get("test");
7471
assertThat(top.getTopMetrics(), hasSize(1));
7572
ParsedTopMetrics.TopMetrics metric = top.getTopMetrics().get(0);
7673
assertThat(metric.getSort(), equalTo(singletonList(2)));
7774
assertThat(metric.getMetrics(), equalTo(singletonMap("v", 3.0)));
7875
}
76+
77+
public void testTopMetricsSizeTwo() throws IOException {
78+
indexTopMetricsData();
79+
SearchRequest search = new SearchRequest("test");
80+
search.source().aggregation(new TopMetricsAggregationBuilder(
81+
"test", new FieldSortBuilder("s").order(SortOrder.DESC), 2, "v"));
82+
SearchResponse response = highLevelClient().search(search, RequestOptions.DEFAULT);
83+
ParsedTopMetrics top = response.getAggregations().get("test");
84+
assertThat(top.getTopMetrics(), hasSize(2));
85+
ParsedTopMetrics.TopMetrics metric = top.getTopMetrics().get(0);
86+
assertThat(metric.getSort(), equalTo(singletonList(2)));
87+
assertThat(metric.getMetrics(), equalTo(singletonMap("v", 3.0)));
88+
metric = top.getTopMetrics().get(1);
89+
assertThat(metric.getSort(), equalTo(singletonList(1)));
90+
assertThat(metric.getMetrics(), equalTo(singletonMap("v", 2.0)));
91+
}
92+
93+
private void indexTopMetricsData() throws IOException {
94+
BulkRequest bulk = new BulkRequest("test").setRefreshPolicy(RefreshPolicy.IMMEDIATE);
95+
bulk.add(new IndexRequest().source(XContentType.JSON, "s", 1, "v", 2));
96+
bulk.add(new IndexRequest().source(XContentType.JSON, "s", 2, "v", 3));
97+
highLevelClient().bulk(bulk, RequestOptions.DEFAULT);
98+
}
7999
}

‎docs/reference/aggregations/metrics/top-metrics-aggregation.asciidoc

Lines changed: 66 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ POST /test/_bulk?refresh
1414
{"index": {}}
1515
{"s": 1, "v": 3.1415}
1616
{"index": {}}
17-
{"s": 2, "v": 1}
17+
{"s": 2, "v": 1.0}
1818
{"index": {}}
1919
{"s": 3, "v": 2.71828}
2020
POST /test/_search?filter_path=aggregations
@@ -74,8 +74,73 @@ At this point `metric` supports only `{"field": "field_name"}` and all metrics
7474
are returned as double precision floating point numbers. Expect more to
7575
come here.
7676

77+
==== `size`
78+
79+
`top_metrics` can return the top few document's worth of metrics using the size parameter:
80+
81+
[source,console,id=search-aggregations-metrics-top-metrics-size]
82+
----
83+
POST /test/_bulk?refresh
84+
{"index": {}}
85+
{"s": 1, "v": 3.1415}
86+
{"index": {}}
87+
{"s": 2, "v": 1.0}
88+
{"index": {}}
89+
{"s": 3, "v": 2.71828}
90+
POST /test/_search?filter_path=aggregations
91+
{
92+
"aggs": {
93+
"tm": {
94+
"top_metrics": {
95+
"metric": {"field": "v"},
96+
"sort": {"s": "desc"},
97+
"size": 2
98+
}
99+
}
100+
}
101+
}
102+
----
103+
104+
Which returns:
105+
106+
[source,js]
107+
----
108+
{
109+
"aggregations": {
110+
"tm": {
111+
"top": [
112+
{"sort": [3], "metrics": {"v": 2.718280076980591 } },
113+
{"sort": [2], "metrics": {"v": 1.0 } }
114+
]
115+
}
116+
}
117+
}
118+
----
119+
// TESTRESPONSE
120+
121+
The default `size` is 1. The maximum default size is `10` because the aggregation's
122+
working storage is "dense", meaning we allocate `size` slots for every bucket. `10`
123+
is a *very* conservative default maximum and you can raise it if you need to by
124+
changing the `top_metrics_max_size` index setting. But know that large sizes can
125+
take a fair bit of memory, especially if they are inside of an aggregation which
126+
makes many buckes like a large
127+
<<search-aggregations-metrics-top-metrics-example-terms, terms aggregation>>.
128+
129+
[source,console]
130+
----
131+
PUT /test/_settings
132+
{
133+
"top_metrics_max_size": 100
134+
}
135+
----
136+
// TEST[continued]
137+
138+
NOTE: If `size` is more than `1` the `top_metrics` aggregation can't be the target of a sort.
139+
140+
77141
==== Examples
78142

143+
[[search-aggregations-metrics-top-metrics-example-terms]]
79144
===== Use with terms
80145

81146
This aggregation should be quite useful inside of <<search-aggregations-bucket-terms-aggregation, `terms`>>

‎modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -527,8 +527,9 @@ public SortField sortField(@Nullable Object missingValue, MultiValueMode sortMod
527527

528528
@Override
529529
public BucketedSort newBucketedSort(BigArrays bigArrays, Object missingValue, MultiValueMode sortMode, Nested nested,
530-
SortOrder sortOrder, DocValueFormat format) {
531-
return new DoubleValuesComparatorSource(this, missingValue, sortMode, nested).newBucketedSort(bigArrays, sortOrder, format);
530+
SortOrder sortOrder, DocValueFormat format, int bucketSize, BucketedSort.ExtraData extra) {
531+
return new DoubleValuesComparatorSource(this, missingValue, sortMode, nested)
532+
.newBucketedSort(bigArrays, sortOrder, format, bucketSize, extra);
532533
}
533534

534535
@Override

‎server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@ public interface IndexFieldData<FD extends AtomicFieldData> extends IndexCompone
8080
* Build a sort implementation specialized for aggregations.
8181
*/
8282
BucketedSort newBucketedSort(BigArrays bigArrays, @Nullable Object missingValue, MultiValueMode sortMode,
83-
Nested nested, SortOrder sortOrder, DocValueFormat format);
83+
Nested nested, SortOrder sortOrder, DocValueFormat format, int bucketSize, BucketedSort.ExtraData extra);
8484

8585
/**
8686
* Clears any resources associated with this field data.
@@ -241,7 +241,8 @@ public Object missingValue(boolean reversed) {
241241
/**
242242
* Create a {@linkplain BucketedSort} which is useful for sorting inside of aggregations.
243243
*/
244-
public abstract BucketedSort newBucketedSort(BigArrays bigArrays, SortOrder sortOrder, DocValueFormat format);
244+
public abstract BucketedSort newBucketedSort(BigArrays bigArrays, SortOrder sortOrder, DocValueFormat format,
245+
int bucketSize, BucketedSort.ExtraData extra);
245246
}
246247

247248
interface Builder {

‎server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -140,7 +140,8 @@ public void setScorer(Scorable scorer) {
140140
}
141141

142142
@Override
143-
public BucketedSort newBucketedSort(BigArrays bigArrays, SortOrder sortOrder, DocValueFormat format) {
143+
public BucketedSort newBucketedSort(BigArrays bigArrays, SortOrder sortOrder, DocValueFormat format,
144+
int bucketSize, BucketedSort.ExtraData extra) {
144145
throw new IllegalArgumentException("only supported on numeric fields");
145146
}
146147

‎server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleValuesComparatorSource.java

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -96,23 +96,29 @@ public void setScorer(Scorable scorer) {
9696
}
9797

9898
@Override
99-
public BucketedSort newBucketedSort(BigArrays bigArrays, SortOrder sortOrder, DocValueFormat format) {
100-
return new BucketedSort.ForDoubles(bigArrays, sortOrder, format) {
99+
public BucketedSort newBucketedSort(BigArrays bigArrays, SortOrder sortOrder, DocValueFormat format,
100+
int bucketSize, BucketedSort.ExtraData extra) {
101+
return new BucketedSort.ForDoubles(bigArrays, sortOrder, format, bucketSize, extra) {
101102
private final double dMissingValue = (Double) missingObject(missingValue, sortOrder == SortOrder.DESC);
102103

103104
@Override
104105
public Leaf forLeaf(LeafReaderContext ctx) throws IOException {
105-
return new Leaf() {
106-
private final NumericDoubleValues values = getNumericDocValues(ctx, dMissingValue);
106+
return new Leaf(ctx) {
107+
private final NumericDoubleValues docValues = getNumericDocValues(ctx, dMissingValue);
108+
private double docValue;
107109

108110
@Override
109111
protected boolean advanceExact(int doc) throws IOException {
110-
return values.advanceExact(doc);
112+
if (docValues.advanceExact(doc)) {
113+
docValue = docValues.doubleValue();
114+
return true;
115+
}
116+
return false;
111117
}
112118

113119
@Override
114-
protected double docValue() throws IOException {
115-
return values.doubleValue();
120+
protected double docValue() {
121+
return docValue;
116122
}
117123
};
118124
}

‎server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/FloatValuesComparatorSource.java

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -85,29 +85,35 @@ protected NumericDocValues getNumericDocValues(LeafReaderContext context, String
8585
}
8686

8787
@Override
88-
public BucketedSort newBucketedSort(BigArrays bigArrays, SortOrder sortOrder, DocValueFormat format) {
89-
return new BucketedSort.ForFloats(bigArrays, sortOrder, format) {
88+
public BucketedSort newBucketedSort(BigArrays bigArrays, SortOrder sortOrder, DocValueFormat format,
89+
int bucketSize, BucketedSort.ExtraData extra) {
90+
return new BucketedSort.ForFloats(bigArrays, sortOrder, format, bucketSize, extra) {
9091
private final float dMissingValue = (Float) missingObject(missingValue, sortOrder == SortOrder.DESC);
9192

9293
@Override
9394
public boolean needsScores() { return false; }
9495

9596
@Override
9697
public Leaf forLeaf(LeafReaderContext ctx) throws IOException {
97-
return new Leaf() {
98-
private final NumericDoubleValues values = getNumericDocValues(ctx, dMissingValue);
98+
return new Leaf(ctx) {
99+
private final NumericDoubleValues docValues = getNumericDocValues(ctx, dMissingValue);
100+
private float docValue;
99101

100102
@Override
101103
public void setScorer(Scorable scorer) {}
102104

103105
@Override
104106
protected boolean advanceExact(int doc) throws IOException {
105-
return values.advanceExact(doc);
107+
if (docValues.advanceExact(doc)) {
108+
docValue = (float) docValues.doubleValue();
109+
return true;
110+
}
111+
return false;
106112
}
107113

108114
@Override
109-
protected float docValue() throws IOException {
110-
return (float) values.doubleValue();
115+
protected float docValue() {
116+
return docValue;
111117
}
112118
};
113119
}

‎server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/LongValuesComparatorSource.java

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -104,23 +104,29 @@ protected NumericDocValues getNumericDocValues(LeafReaderContext context, String
104104
}
105105

106106
@Override
107-
public BucketedSort newBucketedSort(BigArrays bigArrays, SortOrder sortOrder, DocValueFormat format) {
108-
return new BucketedSort.ForLongs(bigArrays, sortOrder, format) {
107+
public BucketedSort newBucketedSort(BigArrays bigArrays, SortOrder sortOrder, DocValueFormat format,
108+
int bucketSize, BucketedSort.ExtraData extra) {
109+
return new BucketedSort.ForLongs(bigArrays, sortOrder, format, bucketSize, extra) {
109110
private final long lMissingValue = (Long) missingObject(missingValue, sortOrder == SortOrder.DESC);
110111

111112
@Override
112113
public Leaf forLeaf(LeafReaderContext ctx) throws IOException {
113-
return new Leaf() {
114-
private final NumericDocValues values = getNumericDocValues(ctx, lMissingValue);
114+
return new Leaf(ctx) {
115+
private final NumericDocValues docValues = getNumericDocValues(ctx, lMissingValue);
116+
private long docValue;
115117

116118
@Override
117119
protected boolean advanceExact(int doc) throws IOException {
118-
return values.advanceExact(doc);
120+
if (docValues.advanceExact(doc)) {
121+
docValue = docValues.longValue();
122+
return true;
123+
}
124+
return false;
119125
}
120126

121127
@Override
122-
protected long docValue() throws IOException {
123-
return values.longValue();
128+
protected long docValue() {
129+
return docValue;
124130
}
125131
};
126132
}

‎server/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsIndexFieldData.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,7 @@ public SortField sortField(@Nullable Object missingValue, MultiValueMode sortMod
108108

109109
@Override
110110
public BucketedSort newBucketedSort(BigArrays bigArrays, Object missingValue, MultiValueMode sortMode, Nested nested,
111-
SortOrder sortOrder, DocValueFormat format) {
111+
SortOrder sortOrder, DocValueFormat format, int bucketSize, BucketedSort.ExtraData extra) {
112112
throw new IllegalArgumentException("only supported on numeric fields");
113113
}
114114

@@ -198,7 +198,7 @@ public SortField sortField(@Nullable Object missingValue, MultiValueMode sortMod
198198

199199
@Override
200200
public BucketedSort newBucketedSort(BigArrays bigArrays, Object missingValue, MultiValueMode sortMode, Nested nested,
201-
SortOrder sortOrder, DocValueFormat format) {
201+
SortOrder sortOrder, DocValueFormat format, int bucketSize, BucketedSort.ExtraData extra) {
202202
throw new IllegalArgumentException("only supported on numeric fields");
203203
}
204204

‎server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractLatLonPointDVIndexFieldData.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ public SortField sortField(@Nullable Object missingValue, MultiValueMode sortMod
5555

5656
@Override
5757
public BucketedSort newBucketedSort(BigArrays bigArrays, Object missingValue, MultiValueMode sortMode, Nested nested,
58-
SortOrder sortOrder, DocValueFormat format) {
58+
SortOrder sortOrder, DocValueFormat format, int bucketSize, BucketedSort.ExtraData extra) {
5959
throw new IllegalArgumentException("can't sort on geo_point field without using specific sorting feature, like geo_distance");
6060
}
6161

‎server/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVIndexFieldData.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ public SortField sortField(@Nullable Object missingValue, MultiValueMode sortMod
7272

7373
@Override
7474
public BucketedSort newBucketedSort(BigArrays bigArrays, Object missingValue, MultiValueMode sortMode, Nested nested,
75-
SortOrder sortOrder, DocValueFormat format) {
75+
SortOrder sortOrder, DocValueFormat format, int bucketSize, BucketedSort.ExtraData extra) {
7676
throw new IllegalArgumentException("only supported on numeric fields");
7777
}
7878
}

‎server/src/main/java/org/elasticsearch/index/fielddata/plain/BytesBinaryDVIndexFieldData.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ public SortField sortField(@Nullable Object missingValue, MultiValueMode sortMod
5252

5353
@Override
5454
public BucketedSort newBucketedSort(BigArrays bigArrays, Object missingValue, MultiValueMode sortMode, Nested nested,
55-
SortOrder sortOrder, DocValueFormat format) {
55+
SortOrder sortOrder, DocValueFormat format, int bucketSize, BucketedSort.ExtraData extra) {
5656
throw new IllegalArgumentException("can't sort on binary field");
5757
}
5858

‎server/src/main/java/org/elasticsearch/index/fielddata/plain/ConstantIndexFieldData.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -165,7 +165,7 @@ public SortField sortField(@Nullable Object missingValue, MultiValueMode sortMod
165165

166166
@Override
167167
public BucketedSort newBucketedSort(BigArrays bigArrays, Object missingValue, MultiValueMode sortMode, Nested nested,
168-
SortOrder sortOrder, DocValueFormat format) {
168+
SortOrder sortOrder, DocValueFormat format, int bucketSize, BucketedSort.ExtraData extra) {
169169
throw new IllegalArgumentException("only supported on numeric fields");
170170
}
171171

‎server/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesIndexFieldData.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,7 @@ public SortField sortField(@Nullable Object missingValue, MultiValueMode sortMod
9191

9292
@Override
9393
public BucketedSort newBucketedSort(BigArrays bigArrays, Object missingValue, MultiValueMode sortMode, Nested nested,
94-
SortOrder sortOrder, DocValueFormat format) {
94+
SortOrder sortOrder, DocValueFormat format, int bucketSize, BucketedSort.ExtraData extra) {
9595
throw new IllegalArgumentException("only supported on numeric fields");
9696
}
9797

0 commit comments

Comments
 (0)