Skip to content

Commit 5e1e129

Browse files
committed
Support geotile_grid aggregation in composite agg sources (elastic#45810)
Adds support for `geotile_grid` as a source in composite aggs. Part of this change includes adding a new docFormat of `GEOTILE` that formats a hashed `long` value into a geotile formatting string `zoom/x/y`.
1 parent caf3e4d commit 5e1e129

16 files changed

+514
-30
lines changed

rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/230_composite.yml

+90-4
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@ setup:
1212
type: keyword
1313
long:
1414
type: long
15+
geo_point:
16+
type: geo_point
1517
nested:
1618
type: nested
1719
properties:
@@ -38,25 +40,25 @@ setup:
3840
index:
3941
index: test
4042
id: 1
41-
body: { "keyword": "foo", "long": [10, 20], "nested": [{"nested_long": 10}, {"nested_long": 20}] }
43+
body: { "keyword": "foo", "long": [10, 20], "geo_point": "37.2343,-115.8067", "nested": [{"nested_long": 10}, {"nested_long": 20}] }
4244

4345
- do:
4446
index:
4547
index: test
4648
id: 2
47-
body: { "keyword": ["foo", "bar"] }
49+
body: { "keyword": ["foo", "bar"], "geo_point": "41.12,-71.34" }
4850

4951
- do:
5052
index:
5153
index: test
5254
id: 3
53-
body: { "keyword": "bar", "long": [100, 0], "nested": [{"nested_long": 10}, {"nested_long": 0}] }
55+
body: { "keyword": "bar", "long": [100, 0], "geo_point": "90.0,0.0", "nested": [{"nested_long": 10}, {"nested_long": 0}] }
5456

5557
- do:
5658
index:
5759
index: test
5860
id: 4
59-
body: { "keyword": "bar", "long": [1000, 0], "nested": [{"nested_long": 1000}, {"nested_long": 20}] }
61+
body: { "keyword": "bar", "long": [1000, 0], "geo_point": "41.12,-71.34", "nested": [{"nested_long": 1000}, {"nested_long": 20}] }
6062

6163
- do:
6264
index:
@@ -650,3 +652,87 @@ setup:
650652
}
651653
]
652654

655+
---
656+
"Simple Composite aggregation with GeoTile grid":
657+
- skip:
658+
version: " - 7.99.99"
659+
reason: geotile_grid is not supported until 8.0.0
660+
- do:
661+
search:
662+
rest_total_hits_as_int: true
663+
index: test
664+
body:
665+
aggregations:
666+
test:
667+
composite:
668+
sources: [
669+
"geo": {
670+
"geotile_grid": {
671+
"field": "geo_point",
672+
"precision": 12
673+
}
674+
},
675+
{
676+
"kw": {
677+
"terms": {
678+
"field": "keyword"
679+
}
680+
}
681+
}
682+
]
683+
684+
- match: {hits.total: 6}
685+
- length: { aggregations.test.buckets: 4 }
686+
- match: { aggregations.test.buckets.0.key.geo: "12/730/1590" }
687+
- match: { aggregations.test.buckets.0.key.kw: "foo" }
688+
- match: { aggregations.test.buckets.0.doc_count: 1 }
689+
- match: { aggregations.test.buckets.1.key.geo: "12/1236/1533" }
690+
- match: { aggregations.test.buckets.1.key.kw: "bar" }
691+
- match: { aggregations.test.buckets.1.doc_count: 2 }
692+
- match: { aggregations.test.buckets.2.key.geo: "12/1236/1533" }
693+
- match: { aggregations.test.buckets.2.key.kw: "foo" }
694+
- match: { aggregations.test.buckets.2.doc_count: 1 }
695+
- match: { aggregations.test.buckets.3.key.geo: "12/2048/0" }
696+
- match: { aggregations.test.buckets.3.key.kw: "bar" }
697+
- match: { aggregations.test.buckets.3.doc_count: 1 }
698+
---
699+
"Simple Composite aggregation with geotile grid add aggregate after":
700+
- skip:
701+
version: " - 7.99.99"
702+
reason: geotile_grid is not supported until 8.0.0
703+
- do:
704+
search:
705+
rest_total_hits_as_int: true
706+
index: test
707+
body:
708+
aggregations:
709+
test:
710+
composite:
711+
sources: [
712+
"geo": {
713+
"geotile_grid": {
714+
"field": "geo_point",
715+
"precision": 12
716+
}
717+
},
718+
{
719+
"kw": {
720+
"terms": {
721+
"field": "keyword"
722+
}
723+
}
724+
}
725+
]
726+
after: { "geo": "12/730/1590", "kw": "foo" }
727+
728+
- match: {hits.total: 6}
729+
- length: { aggregations.test.buckets: 3 }
730+
- match: { aggregations.test.buckets.0.key.geo: "12/1236/1533" }
731+
- match: { aggregations.test.buckets.0.key.kw: "bar" }
732+
- match: { aggregations.test.buckets.0.doc_count: 2 }
733+
- match: { aggregations.test.buckets.1.key.geo: "12/1236/1533" }
734+
- match: { aggregations.test.buckets.1.key.kw: "foo" }
735+
- match: { aggregations.test.buckets.1.doc_count: 1 }
736+
- match: { aggregations.test.buckets.2.key.geo: "12/2048/0" }
737+
- match: { aggregations.test.buckets.2.key.kw: "bar" }
738+
- match: { aggregations.test.buckets.2.doc_count: 1 }

server/src/main/java/org/elasticsearch/search/DocValueFormat.java

+23
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
import org.elasticsearch.common.time.DateUtils;
3333
import org.elasticsearch.geometry.utils.Geohash;
3434
import org.elasticsearch.index.mapper.DateFieldMapper;
35+
import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils;
3536

3637
import java.io.IOException;
3738
import java.net.InetAddress;
@@ -257,6 +258,28 @@ public String format(double value) {
257258
}
258259
};
259260

261+
DocValueFormat GEOTILE = new DocValueFormat() {
262+
263+
@Override
264+
public String getWriteableName() {
265+
return "geo_tile";
266+
}
267+
268+
@Override
269+
public void writeTo(StreamOutput out) {
270+
}
271+
272+
@Override
273+
public String format(long value) {
274+
return GeoTileUtils.stringEncode(value);
275+
}
276+
277+
@Override
278+
public String format(double value) {
279+
return format((long) value);
280+
}
281+
};
282+
260283
DocValueFormat BOOLEAN = new DocValueFormat() {
261284

262285
@Override

server/src/main/java/org/elasticsearch/search/SearchModule.java

+1
Original file line numberDiff line numberDiff line change
@@ -704,6 +704,7 @@ private void registerValueFormats() {
704704
registerValueFormat(DocValueFormat.DateTime.NAME, DocValueFormat.DateTime::new);
705705
registerValueFormat(DocValueFormat.Decimal.NAME, DocValueFormat.Decimal::new);
706706
registerValueFormat(DocValueFormat.GEOHASH.getWriteableName(), in -> DocValueFormat.GEOHASH);
707+
registerValueFormat(DocValueFormat.GEOTILE.getWriteableName(), in -> DocValueFormat.GEOTILE);
707708
registerValueFormat(DocValueFormat.IP.getWriteableName(), in -> DocValueFormat.IP);
708709
registerValueFormat(DocValueFormat.RAW.getWriteableName(), in -> DocValueFormat.RAW);
709710
registerValueFormat(DocValueFormat.BINARY.getWriteableName(), in -> DocValueFormat.BINARY);

server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregator.java

+12
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@
4242
import org.elasticsearch.search.aggregations.MultiBucketCollector;
4343
import org.elasticsearch.search.aggregations.MultiBucketConsumerService;
4444
import org.elasticsearch.search.aggregations.bucket.BucketsAggregator;
45+
import org.elasticsearch.search.aggregations.bucket.geogrid.CellIdSource;
4546
import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
4647
import org.elasticsearch.search.aggregations.support.ValuesSource;
4748
import org.elasticsearch.search.internal.SearchContext;
@@ -299,6 +300,17 @@ private SingleDimensionValuesSource<?> createValuesSource(BigArrays bigArrays, I
299300
reverseMul
300301
);
301302

303+
} else if (config.valuesSource() instanceof CellIdSource) {
304+
final CellIdSource cis = (CellIdSource) config.valuesSource();
305+
return new GeoTileValuesSource(
306+
bigArrays,
307+
config.fieldType(),
308+
cis::longValues,
309+
LongUnaryOperator.identity(),
310+
config.format(),
311+
config.missingBucket(),
312+
size,
313+
reverseMul);
302314
} else if (config.valuesSource() instanceof ValuesSource.Numeric) {
303315
final ValuesSource.Numeric vs = (ValuesSource.Numeric) config.valuesSource();
304316
if (vs.isFloatingPoint()) {

server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeValuesSourceParserHelper.java

+12
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
package org.elasticsearch.search.aggregations.bucket.composite;
2121

22+
import org.elasticsearch.Version;
2223
import org.elasticsearch.common.ParseField;
2324
import org.elasticsearch.common.ParsingException;
2425
import org.elasticsearch.common.io.stream.StreamInput;
@@ -67,6 +68,12 @@ public static void writeTo(CompositeValuesSourceBuilder<?> builder, StreamOutput
6768
code = 1;
6869
} else if (builder.getClass() == HistogramValuesSourceBuilder.class) {
6970
code = 2;
71+
} else if (builder.getClass() == GeoTileGridValuesSourceBuilder.class) {
72+
if (out.getVersion().before(Version.V_8_0_0)) {
73+
throw new IOException("Attempting to serialize [" + builder.getClass().getSimpleName()
74+
+ "] to a node with unsupported version [" + out.getVersion() + "]");
75+
}
76+
code = 3;
7077
} else {
7178
throw new IOException("invalid builder type: " + builder.getClass().getSimpleName());
7279
}
@@ -83,6 +90,8 @@ public static CompositeValuesSourceBuilder<?> readFrom(StreamInput in) throws IO
8390
return new DateHistogramValuesSourceBuilder(in);
8491
case 2:
8592
return new HistogramValuesSourceBuilder(in);
93+
case 3:
94+
return new GeoTileGridValuesSourceBuilder(in);
8695
default:
8796
throw new IOException("Invalid code " + code);
8897
}
@@ -112,6 +121,9 @@ public static CompositeValuesSourceBuilder<?> fromXContent(XContentParser parser
112121
case HistogramValuesSourceBuilder.TYPE:
113122
builder = HistogramValuesSourceBuilder.parse(name, parser);
114123
break;
124+
case GeoTileGridValuesSourceBuilder.TYPE:
125+
builder = GeoTileGridValuesSourceBuilder.parse(name, parser);
126+
break;
115127
default:
116128
throw new ParsingException(parser.getTokenLocation(), "invalid source type: " + type);
117129
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.elasticsearch.search.aggregations.bucket.composite;
21+
22+
import org.elasticsearch.common.ParseField;
23+
import org.elasticsearch.common.io.stream.StreamInput;
24+
import org.elasticsearch.common.io.stream.StreamOutput;
25+
import org.elasticsearch.common.xcontent.ObjectParser;
26+
import org.elasticsearch.common.xcontent.XContentBuilder;
27+
import org.elasticsearch.common.xcontent.XContentParser;
28+
import org.elasticsearch.index.mapper.MappedFieldType;
29+
import org.elasticsearch.search.DocValueFormat;
30+
import org.elasticsearch.search.aggregations.bucket.geogrid.CellIdSource;
31+
import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder;
32+
import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils;
33+
import org.elasticsearch.search.aggregations.support.ValueType;
34+
import org.elasticsearch.search.aggregations.support.ValuesSource;
35+
import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
36+
import org.elasticsearch.search.internal.SearchContext;
37+
38+
import java.io.IOException;
39+
import java.util.Objects;
40+
41+
public class GeoTileGridValuesSourceBuilder extends CompositeValuesSourceBuilder<GeoTileGridValuesSourceBuilder> {
42+
static final String TYPE = "geotile_grid";
43+
44+
private static final ObjectParser<GeoTileGridValuesSourceBuilder, Void> PARSER;
45+
static {
46+
PARSER = new ObjectParser<>(GeoTileGridValuesSourceBuilder.TYPE);
47+
PARSER.declareInt(GeoTileGridValuesSourceBuilder::precision, new ParseField("precision"));
48+
CompositeValuesSourceParserHelper.declareValuesSourceFields(PARSER, ValueType.NUMERIC);
49+
}
50+
51+
static GeoTileGridValuesSourceBuilder parse(String name, XContentParser parser) throws IOException {
52+
return PARSER.parse(parser, new GeoTileGridValuesSourceBuilder(name), null);
53+
}
54+
55+
private int precision = GeoTileGridAggregationBuilder.DEFAULT_PRECISION;
56+
57+
GeoTileGridValuesSourceBuilder(String name) {
58+
super(name);
59+
}
60+
61+
GeoTileGridValuesSourceBuilder(StreamInput in) throws IOException {
62+
super(in);
63+
this.precision = in.readInt();
64+
}
65+
66+
public GeoTileGridValuesSourceBuilder precision(int precision) {
67+
this.precision = GeoTileUtils.checkPrecisionRange(precision);
68+
return this;
69+
}
70+
71+
@Override
72+
public GeoTileGridValuesSourceBuilder format(String format) {
73+
throw new IllegalArgumentException("[format] is not supported for [" + TYPE + "]");
74+
}
75+
76+
@Override
77+
protected void innerWriteTo(StreamOutput out) throws IOException {
78+
out.writeInt(precision);
79+
}
80+
81+
@Override
82+
protected void doXContentBody(XContentBuilder builder, Params params) throws IOException {
83+
builder.field("precision", precision);
84+
}
85+
86+
@Override
87+
String type() {
88+
return TYPE;
89+
}
90+
91+
@Override
92+
public int hashCode() {
93+
return Objects.hash(super.hashCode(), precision);
94+
}
95+
96+
@Override
97+
public boolean equals(Object obj) {
98+
if (this == obj) return true;
99+
if (obj == null || getClass() != obj.getClass()) return false;
100+
if (super.equals(obj) == false) return false;
101+
GeoTileGridValuesSourceBuilder other = (GeoTileGridValuesSourceBuilder) obj;
102+
return precision == other.precision;
103+
}
104+
105+
@Override
106+
protected CompositeValuesSourceConfig innerBuild(SearchContext context, ValuesSourceConfig<?> config) throws IOException {
107+
ValuesSource orig = config.toValuesSource(context.getQueryShardContext());
108+
if (orig == null) {
109+
orig = ValuesSource.GeoPoint.EMPTY;
110+
}
111+
if (orig instanceof ValuesSource.GeoPoint) {
112+
ValuesSource.GeoPoint geoPoint = (ValuesSource.GeoPoint) orig;
113+
// is specified in the builder.
114+
final MappedFieldType fieldType = config.fieldContext() != null ? config.fieldContext().fieldType() : null;
115+
CellIdSource cellIdSource = new CellIdSource(geoPoint, precision, GeoTileUtils::longEncode);
116+
return new CompositeValuesSourceConfig(name, fieldType, cellIdSource, DocValueFormat.GEOTILE, order(), missingBucket());
117+
} else {
118+
throw new IllegalArgumentException("invalid source, expected geo_point, got " + orig.getClass().getSimpleName());
119+
}
120+
}
121+
122+
}

0 commit comments

Comments
 (0)