Skip to content

Commit 9d3467a

Browse files
authored
Support geotile_grid aggregation in composite agg sources (#45810)
Adds support for `geotile_grid` as a source in composite aggs. Part of this change includes adding a new docFormat of `GEOTILE` that formats a hashed `long` value into a geotile formatting string `zoom/x/y`.
1 parent ca8dbe5 commit 9d3467a

16 files changed

+514
-30
lines changed

rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/230_composite.yml

+90-4
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@ setup:
1212
type: keyword
1313
long:
1414
type: long
15+
geo_point:
16+
type: geo_point
1517
nested:
1618
type: nested
1719
properties:
@@ -38,25 +40,25 @@ setup:
3840
index:
3941
index: test
4042
id: 1
41-
body: { "keyword": "foo", "long": [10, 20], "nested": [{"nested_long": 10}, {"nested_long": 20}] }
43+
body: { "keyword": "foo", "long": [10, 20], "geo_point": "37.2343,-115.8067", "nested": [{"nested_long": 10}, {"nested_long": 20}] }
4244

4345
- do:
4446
index:
4547
index: test
4648
id: 2
47-
body: { "keyword": ["foo", "bar"] }
49+
body: { "keyword": ["foo", "bar"], "geo_point": "41.12,-71.34" }
4850

4951
- do:
5052
index:
5153
index: test
5254
id: 3
53-
body: { "keyword": "bar", "long": [100, 0], "nested": [{"nested_long": 10}, {"nested_long": 0}] }
55+
body: { "keyword": "bar", "long": [100, 0], "geo_point": "90.0,0.0", "nested": [{"nested_long": 10}, {"nested_long": 0}] }
5456

5557
- do:
5658
index:
5759
index: test
5860
id: 4
59-
body: { "keyword": "bar", "long": [1000, 0], "nested": [{"nested_long": 1000}, {"nested_long": 20}] }
61+
body: { "keyword": "bar", "long": [1000, 0], "geo_point": "41.12,-71.34", "nested": [{"nested_long": 1000}, {"nested_long": 20}] }
6062

6163
- do:
6264
index:
@@ -615,3 +617,87 @@ setup:
615617
}
616618
]
617619

620+
---
621+
"Simple Composite aggregation with GeoTile grid":
622+
- skip:
623+
version: " - 7.99.99"
624+
reason: geotile_grid is not supported until 8.0.0
625+
- do:
626+
search:
627+
rest_total_hits_as_int: true
628+
index: test
629+
body:
630+
aggregations:
631+
test:
632+
composite:
633+
sources: [
634+
"geo": {
635+
"geotile_grid": {
636+
"field": "geo_point",
637+
"precision": 12
638+
}
639+
},
640+
{
641+
"kw": {
642+
"terms": {
643+
"field": "keyword"
644+
}
645+
}
646+
}
647+
]
648+
649+
- match: {hits.total: 6}
650+
- length: { aggregations.test.buckets: 4 }
651+
- match: { aggregations.test.buckets.0.key.geo: "12/730/1590" }
652+
- match: { aggregations.test.buckets.0.key.kw: "foo" }
653+
- match: { aggregations.test.buckets.0.doc_count: 1 }
654+
- match: { aggregations.test.buckets.1.key.geo: "12/1236/1533" }
655+
- match: { aggregations.test.buckets.1.key.kw: "bar" }
656+
- match: { aggregations.test.buckets.1.doc_count: 2 }
657+
- match: { aggregations.test.buckets.2.key.geo: "12/1236/1533" }
658+
- match: { aggregations.test.buckets.2.key.kw: "foo" }
659+
- match: { aggregations.test.buckets.2.doc_count: 1 }
660+
- match: { aggregations.test.buckets.3.key.geo: "12/2048/0" }
661+
- match: { aggregations.test.buckets.3.key.kw: "bar" }
662+
- match: { aggregations.test.buckets.3.doc_count: 1 }
663+
---
664+
"Simple Composite aggregation with geotile grid add aggregate after":
665+
- skip:
666+
version: " - 7.99.99"
667+
reason: geotile_grid is not supported until 8.0.0
668+
- do:
669+
search:
670+
rest_total_hits_as_int: true
671+
index: test
672+
body:
673+
aggregations:
674+
test:
675+
composite:
676+
sources: [
677+
"geo": {
678+
"geotile_grid": {
679+
"field": "geo_point",
680+
"precision": 12
681+
}
682+
},
683+
{
684+
"kw": {
685+
"terms": {
686+
"field": "keyword"
687+
}
688+
}
689+
}
690+
]
691+
after: { "geo": "12/730/1590", "kw": "foo" }
692+
693+
- match: {hits.total: 6}
694+
- length: { aggregations.test.buckets: 3 }
695+
- match: { aggregations.test.buckets.0.key.geo: "12/1236/1533" }
696+
- match: { aggregations.test.buckets.0.key.kw: "bar" }
697+
- match: { aggregations.test.buckets.0.doc_count: 2 }
698+
- match: { aggregations.test.buckets.1.key.geo: "12/1236/1533" }
699+
- match: { aggregations.test.buckets.1.key.kw: "foo" }
700+
- match: { aggregations.test.buckets.1.doc_count: 1 }
701+
- match: { aggregations.test.buckets.2.key.geo: "12/2048/0" }
702+
- match: { aggregations.test.buckets.2.key.kw: "bar" }
703+
- match: { aggregations.test.buckets.2.doc_count: 1 }

server/src/main/java/org/elasticsearch/search/DocValueFormat.java

+23
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
import org.elasticsearch.common.time.DateMathParser;
3131
import org.elasticsearch.geometry.utils.Geohash;
3232
import org.elasticsearch.index.mapper.DateFieldMapper;
33+
import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils;
3334

3435
import java.io.IOException;
3536
import java.net.InetAddress;
@@ -246,6 +247,28 @@ public String format(double value) {
246247
}
247248
};
248249

250+
DocValueFormat GEOTILE = new DocValueFormat() {
251+
252+
@Override
253+
public String getWriteableName() {
254+
return "geo_tile";
255+
}
256+
257+
@Override
258+
public void writeTo(StreamOutput out) {
259+
}
260+
261+
@Override
262+
public String format(long value) {
263+
return GeoTileUtils.stringEncode(value);
264+
}
265+
266+
@Override
267+
public String format(double value) {
268+
return format((long) value);
269+
}
270+
};
271+
249272
DocValueFormat BOOLEAN = new DocValueFormat() {
250273

251274
@Override

server/src/main/java/org/elasticsearch/search/SearchModule.java

+1
Original file line numberDiff line numberDiff line change
@@ -667,6 +667,7 @@ private void registerValueFormats() {
667667
registerValueFormat(DocValueFormat.DateTime.NAME, DocValueFormat.DateTime::new);
668668
registerValueFormat(DocValueFormat.Decimal.NAME, DocValueFormat.Decimal::new);
669669
registerValueFormat(DocValueFormat.GEOHASH.getWriteableName(), in -> DocValueFormat.GEOHASH);
670+
registerValueFormat(DocValueFormat.GEOTILE.getWriteableName(), in -> DocValueFormat.GEOTILE);
670671
registerValueFormat(DocValueFormat.IP.getWriteableName(), in -> DocValueFormat.IP);
671672
registerValueFormat(DocValueFormat.RAW.getWriteableName(), in -> DocValueFormat.RAW);
672673
registerValueFormat(DocValueFormat.BINARY.getWriteableName(), in -> DocValueFormat.BINARY);

server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregator.java

+12
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@
4242
import org.elasticsearch.search.aggregations.MultiBucketCollector;
4343
import org.elasticsearch.search.aggregations.MultiBucketConsumerService;
4444
import org.elasticsearch.search.aggregations.bucket.BucketsAggregator;
45+
import org.elasticsearch.search.aggregations.bucket.geogrid.CellIdSource;
4546
import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
4647
import org.elasticsearch.search.aggregations.support.ValuesSource;
4748
import org.elasticsearch.search.internal.SearchContext;
@@ -299,6 +300,17 @@ private SingleDimensionValuesSource<?> createValuesSource(BigArrays bigArrays, I
299300
reverseMul
300301
);
301302

303+
} else if (config.valuesSource() instanceof CellIdSource) {
304+
final CellIdSource cis = (CellIdSource) config.valuesSource();
305+
return new GeoTileValuesSource(
306+
bigArrays,
307+
config.fieldType(),
308+
cis::longValues,
309+
LongUnaryOperator.identity(),
310+
config.format(),
311+
config.missingBucket(),
312+
size,
313+
reverseMul);
302314
} else if (config.valuesSource() instanceof ValuesSource.Numeric) {
303315
final ValuesSource.Numeric vs = (ValuesSource.Numeric) config.valuesSource();
304316
if (vs.isFloatingPoint()) {

server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeValuesSourceParserHelper.java

+12
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
package org.elasticsearch.search.aggregations.bucket.composite;
2121

22+
import org.elasticsearch.Version;
2223
import org.elasticsearch.common.ParseField;
2324
import org.elasticsearch.common.ParsingException;
2425
import org.elasticsearch.common.io.stream.StreamInput;
@@ -67,6 +68,12 @@ public static void writeTo(CompositeValuesSourceBuilder<?> builder, StreamOutput
6768
code = 1;
6869
} else if (builder.getClass() == HistogramValuesSourceBuilder.class) {
6970
code = 2;
71+
} else if (builder.getClass() == GeoTileGridValuesSourceBuilder.class) {
72+
if (out.getVersion().before(Version.V_8_0_0)) {
73+
throw new IOException("Attempting to serialize [" + builder.getClass().getSimpleName()
74+
+ "] to a node with unsupported version [" + out.getVersion() + "]");
75+
}
76+
code = 3;
7077
} else {
7178
throw new IOException("invalid builder type: " + builder.getClass().getSimpleName());
7279
}
@@ -83,6 +90,8 @@ public static CompositeValuesSourceBuilder<?> readFrom(StreamInput in) throws IO
8390
return new DateHistogramValuesSourceBuilder(in);
8491
case 2:
8592
return new HistogramValuesSourceBuilder(in);
93+
case 3:
94+
return new GeoTileGridValuesSourceBuilder(in);
8695
default:
8796
throw new IOException("Invalid code " + code);
8897
}
@@ -112,6 +121,9 @@ public static CompositeValuesSourceBuilder<?> fromXContent(XContentParser parser
112121
case HistogramValuesSourceBuilder.TYPE:
113122
builder = HistogramValuesSourceBuilder.parse(name, parser);
114123
break;
124+
case GeoTileGridValuesSourceBuilder.TYPE:
125+
builder = GeoTileGridValuesSourceBuilder.parse(name, parser);
126+
break;
115127
default:
116128
throw new ParsingException(parser.getTokenLocation(), "invalid source type: " + type);
117129
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.elasticsearch.search.aggregations.bucket.composite;
21+
22+
import org.elasticsearch.common.ParseField;
23+
import org.elasticsearch.common.io.stream.StreamInput;
24+
import org.elasticsearch.common.io.stream.StreamOutput;
25+
import org.elasticsearch.common.xcontent.ObjectParser;
26+
import org.elasticsearch.common.xcontent.XContentBuilder;
27+
import org.elasticsearch.common.xcontent.XContentParser;
28+
import org.elasticsearch.index.mapper.MappedFieldType;
29+
import org.elasticsearch.search.DocValueFormat;
30+
import org.elasticsearch.search.aggregations.bucket.geogrid.CellIdSource;
31+
import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder;
32+
import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils;
33+
import org.elasticsearch.search.aggregations.support.ValueType;
34+
import org.elasticsearch.search.aggregations.support.ValuesSource;
35+
import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
36+
import org.elasticsearch.search.internal.SearchContext;
37+
38+
import java.io.IOException;
39+
import java.util.Objects;
40+
41+
public class GeoTileGridValuesSourceBuilder extends CompositeValuesSourceBuilder<GeoTileGridValuesSourceBuilder> {
42+
static final String TYPE = "geotile_grid";
43+
44+
private static final ObjectParser<GeoTileGridValuesSourceBuilder, Void> PARSER;
45+
static {
46+
PARSER = new ObjectParser<>(GeoTileGridValuesSourceBuilder.TYPE);
47+
PARSER.declareInt(GeoTileGridValuesSourceBuilder::precision, new ParseField("precision"));
48+
CompositeValuesSourceParserHelper.declareValuesSourceFields(PARSER, ValueType.NUMERIC);
49+
}
50+
51+
static GeoTileGridValuesSourceBuilder parse(String name, XContentParser parser) throws IOException {
52+
return PARSER.parse(parser, new GeoTileGridValuesSourceBuilder(name), null);
53+
}
54+
55+
private int precision = GeoTileGridAggregationBuilder.DEFAULT_PRECISION;
56+
57+
GeoTileGridValuesSourceBuilder(String name) {
58+
super(name);
59+
}
60+
61+
GeoTileGridValuesSourceBuilder(StreamInput in) throws IOException {
62+
super(in);
63+
this.precision = in.readInt();
64+
}
65+
66+
public GeoTileGridValuesSourceBuilder precision(int precision) {
67+
this.precision = GeoTileUtils.checkPrecisionRange(precision);
68+
return this;
69+
}
70+
71+
@Override
72+
public GeoTileGridValuesSourceBuilder format(String format) {
73+
throw new IllegalArgumentException("[format] is not supported for [" + TYPE + "]");
74+
}
75+
76+
@Override
77+
protected void innerWriteTo(StreamOutput out) throws IOException {
78+
out.writeInt(precision);
79+
}
80+
81+
@Override
82+
protected void doXContentBody(XContentBuilder builder, Params params) throws IOException {
83+
builder.field("precision", precision);
84+
}
85+
86+
@Override
87+
String type() {
88+
return TYPE;
89+
}
90+
91+
@Override
92+
public int hashCode() {
93+
return Objects.hash(super.hashCode(), precision);
94+
}
95+
96+
@Override
97+
public boolean equals(Object obj) {
98+
if (this == obj) return true;
99+
if (obj == null || getClass() != obj.getClass()) return false;
100+
if (super.equals(obj) == false) return false;
101+
GeoTileGridValuesSourceBuilder other = (GeoTileGridValuesSourceBuilder) obj;
102+
return precision == other.precision;
103+
}
104+
105+
@Override
106+
protected CompositeValuesSourceConfig innerBuild(SearchContext context, ValuesSourceConfig<?> config) throws IOException {
107+
ValuesSource orig = config.toValuesSource(context.getQueryShardContext());
108+
if (orig == null) {
109+
orig = ValuesSource.GeoPoint.EMPTY;
110+
}
111+
if (orig instanceof ValuesSource.GeoPoint) {
112+
ValuesSource.GeoPoint geoPoint = (ValuesSource.GeoPoint) orig;
113+
// is specified in the builder.
114+
final MappedFieldType fieldType = config.fieldContext() != null ? config.fieldContext().fieldType() : null;
115+
CellIdSource cellIdSource = new CellIdSource(geoPoint, precision, GeoTileUtils::longEncode);
116+
return new CompositeValuesSourceConfig(name, fieldType, cellIdSource, DocValueFormat.GEOTILE, order(), missingBucket());
117+
} else {
118+
throw new IllegalArgumentException("invalid source, expected geo_point, got " + orig.getClass().getSimpleName());
119+
}
120+
}
121+
122+
}

0 commit comments

Comments
 (0)