Skip to content

Commit c927e15

Browse files
nyuriknyurik
committed
Quadkey implementation
Implements support for the quadkey geo aggregation.
1 parent 8848778 commit c927e15

11 files changed

+635
-3
lines changed

‎server/src/main/java/org/elasticsearch/common/geo/QuadkeyUtils.java

Lines changed: 187 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,187 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package org.elasticsearch.common.geo;
20+
21+
import org.apache.lucene.util.BitUtil;
22+
import org.elasticsearch.ElasticsearchParseException;
23+
import org.elasticsearch.common.xcontent.XContentParser;
24+
import org.elasticsearch.common.xcontent.support.XContentMapValues;
25+
26+
import java.io.IOException;
27+
28+
import static org.elasticsearch.common.geo.GeoUtils.normalizeLat;
29+
import static org.elasticsearch.common.geo.GeoUtils.normalizeLon;
30+
31+
/**
32+
* Implements quad key hashing, same as used by map tiles.
33+
* The string key is formatted as "zoom/x/y"
34+
* The hash value (long) contains all three of those values.
35+
*/
36+
public class QuadkeyUtils {
37+
38+
/**
39+
* Largest number of tiles (precision) to use.
40+
* This value cannot be more than (64-5)/2 = 29, because 5 bits are used for zoom level itself
41+
* If zoom is not stored inside hash, it would be possible to use up to 32.
42+
* Another consideration is that index optimizes lat/lng storage, loosing some precision.
43+
* E.g. hash lng=140.74779717298918D lat=45.61884022447444D == "18/233561/93659", but shown as "18/233561/93658"
44+
*/
45+
public static final int MAX_ZOOM = 29;
46+
47+
/**
48+
* Bit position of the zoom value within hash. Must be >= 2*MAX_ZOOM
49+
* Keeping it at a constant place allows MAX_ZOOM to be increased
50+
* without breaking serialization binary compatibility
51+
* (still, the newer version should not use higher MAX_ZOOM in the mixed cases)
52+
*/
53+
private static final int ZOOM_SHIFT = 29 * 2;
54+
55+
/**
56+
* Mask of all the bits used by the quadkey in a hash
57+
*/
58+
private static final long QUADKEY_MASK = (1L << ZOOM_SHIFT) - 1;
59+
60+
/**
61+
* Parse quadkey hash as zoom, x, y integers.
62+
*/
63+
private static int[] parseHash(final long hash) {
64+
final int zoom = checkPrecisionRange((int) (hash >>> ZOOM_SHIFT));
65+
final int tiles = 1 << zoom;
66+
67+
// decode the quadkey bits as interleaved xTile and yTile
68+
long val = hash & QUADKEY_MASK;
69+
int xTile = (int) BitUtil.deinterleave(val);
70+
int yTile = (int) BitUtil.deinterleave(val >>> 1);
71+
if (xTile < 0 || yTile < 0 || xTile >= tiles || yTile >= tiles) {
72+
throw new IllegalArgumentException("hash-tile");
73+
}
74+
75+
return new int[]{zoom, xTile, yTile};
76+
}
77+
78+
/**
79+
* Parse a precision that can be expressed as an integer or a distance measure like "1km", "10m".
80+
*
81+
* The precision is expressed as a zoom level between 0 and MAX_ZOOM.
82+
*
83+
* @param parser {@link XContentParser} to parse the value from
84+
* @return int representing precision
85+
*/
86+
public static int parsePrecision(XContentParser parser) throws IOException, ElasticsearchParseException {
87+
XContentParser.Token token = parser.currentToken();
88+
if (token.equals(XContentParser.Token.VALUE_NUMBER)) {
89+
return XContentMapValues.nodeIntegerValue(parser.intValue());
90+
} else {
91+
String precision = parser.text();
92+
try {
93+
// we want to treat simple integer strings as precision levels, not distances
94+
return XContentMapValues.nodeIntegerValue(precision);
95+
} catch (NumberFormatException e) {
96+
// try to parse as a distance value
97+
final int parsedPrecision = GeoUtils.quadTreeLevelsForPrecision(precision);
98+
try {
99+
return checkPrecisionRange(parsedPrecision);
100+
} catch (IllegalArgumentException e2) {
101+
// this happens when distance too small, so precision > max.
102+
// We'd like to see the original string
103+
throw new IllegalArgumentException("precision too high [" + precision + "]", e2);
104+
}
105+
}
106+
}
107+
}
108+
109+
public static int checkPrecisionRange(int precision) {
110+
if (precision < 0 || precision > MAX_ZOOM) {
111+
throw new IllegalArgumentException("Invalid quadkey aggregation precision of " +
112+
precision + ". Must be between 0 and " + MAX_ZOOM + ".");
113+
}
114+
return precision;
115+
}
116+
117+
/**
118+
* Encode lon/lat to the quadkey based long format.
119+
* The resulting hash contains interleaved tile X and Y coordinates.
120+
* The precision itself is also encoded as a few high bits.
121+
*/
122+
public static long longEncode(double longitude, double latitude, int precision) {
123+
// Adapted from https://wiki.openstreetmap.org/wiki/Slippy_map_tilenames#Java
124+
125+
// How many tiles in X and in Y
126+
final int tiles = 1 << checkPrecisionRange(precision);
127+
final double lon = normalizeLon(longitude);
128+
final double lat = normalizeLat(latitude);
129+
130+
int xTile = (int) Math.floor((lon + 180) / 360 * tiles);
131+
int yTile = (int) Math.floor(
132+
(1 - Math.log(
133+
Math.tan(Math.toRadians(lat)) + 1 / Math.cos(Math.toRadians(lat))
134+
) / Math.PI) / 2 * tiles);
135+
if (xTile < 0) {
136+
xTile = 0;
137+
}
138+
if (xTile >= tiles) {
139+
xTile = tiles - 1;
140+
}
141+
if (yTile < 0) {
142+
yTile = 0;
143+
}
144+
if (yTile >= tiles) {
145+
yTile = tiles - 1;
146+
}
147+
148+
// Zoom value is placed in front of all the bits used for the quadkey
149+
// e.g. if max zoom is 26, the largest index would use 52 bits (51st..0th),
150+
// leaving 12 bits unused for zoom. See MAX_ZOOM comment above.
151+
return BitUtil.interleave(xTile, yTile) | ((long) precision << ZOOM_SHIFT);
152+
}
153+
154+
/**
155+
* Encode to a quadkey string from the quadkey based long format
156+
*/
157+
public static String stringEncode(long hash) {
158+
int[] res = parseHash(hash);
159+
return "" + res[0] + "/" + res[1] + "/" + res[2];
160+
}
161+
162+
public static GeoPoint hashToGeoPoint(String hashAsString) {
163+
Throwable cause = null;
164+
try {
165+
final String[] parts = hashAsString.split("/", 4);
166+
if (parts.length == 3) {
167+
final int zoom = Integer.parseInt(parts[0]);
168+
final int xTile = Integer.parseInt(parts[1]);
169+
final int yTile = Integer.parseInt(parts[2]);
170+
171+
final int maxTiles = 1 << checkPrecisionRange(zoom);
172+
if (xTile >= 0 && xTile < maxTiles && yTile >= 0 && yTile < maxTiles) {
173+
final double tiles = Math.pow(2.0, zoom);
174+
final double n = Math.PI - (2.0 * Math.PI * (yTile + 0.5)) / tiles;
175+
final double lat = Math.toDegrees(Math.atan(Math.sinh(n)));
176+
final double lon = ((xTile + 0.5) / tiles * 360.0) - 180;
177+
return new GeoPoint(lat, lon);
178+
}
179+
}
180+
} catch (IllegalArgumentException e) {
181+
// This will also handle NumberFormatException
182+
cause = e;
183+
}
184+
throw new IllegalArgumentException("Invalid quadkey hash string of " +
185+
hashAsString + ". Must be three integers in a form \"zoom/x/y\".", cause);
186+
}
187+
}

‎server/src/main/java/org/elasticsearch/search/SearchModule.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,8 @@
110110
import org.elasticsearch.search.aggregations.bucket.filter.InternalFilters;
111111
import org.elasticsearch.search.aggregations.bucket.geogrid.GeoHashGridAggregationBuilder;
112112
import org.elasticsearch.search.aggregations.bucket.geogrid.InternalGeoHashGrid;
113+
import org.elasticsearch.search.aggregations.bucket.geogrid.QuadkeyGridAggregationBuilder;
114+
import org.elasticsearch.search.aggregations.bucket.geogrid.InternalQuadkeyGrid;
113115
import org.elasticsearch.search.aggregations.bucket.global.GlobalAggregationBuilder;
114116
import org.elasticsearch.search.aggregations.bucket.global.InternalGlobal;
115117
import org.elasticsearch.search.aggregations.bucket.histogram.AutoDateHistogramAggregationBuilder;
@@ -422,6 +424,8 @@ private void registerAggregations(List<SearchPlugin> plugins) {
422424
GeoDistanceAggregationBuilder::parse).addResultReader(InternalGeoDistance::new));
423425
registerAggregation(new AggregationSpec(GeoHashGridAggregationBuilder.NAME, GeoHashGridAggregationBuilder::new,
424426
GeoHashGridAggregationBuilder::parse).addResultReader(InternalGeoHashGrid::new));
427+
registerAggregation(new AggregationSpec(QuadkeyGridAggregationBuilder.NAME, QuadkeyGridAggregationBuilder::new,
428+
QuadkeyGridAggregationBuilder::parse).addResultReader(InternalQuadkeyGrid::new));
425429
registerAggregation(new AggregationSpec(NestedAggregationBuilder.NAME, NestedAggregationBuilder::new,
426430
NestedAggregationBuilder::parse).addResultReader(InternalNested::new));
427431
registerAggregation(new AggregationSpec(ReverseNestedAggregationBuilder.NAME, ReverseNestedAggregationBuilder::new,

‎server/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@
3030
import org.elasticsearch.search.aggregations.bucket.filter.FiltersAggregator.KeyedFilter;
3131
import org.elasticsearch.search.aggregations.bucket.geogrid.InternalGeoHashGrid;
3232
import org.elasticsearch.search.aggregations.bucket.geogrid.GeoHashGridAggregationBuilder;
33+
import org.elasticsearch.search.aggregations.bucket.geogrid.InternalQuadkeyGrid;
34+
import org.elasticsearch.search.aggregations.bucket.geogrid.QuadkeyGridAggregationBuilder;
3335
import org.elasticsearch.search.aggregations.bucket.global.Global;
3436
import org.elasticsearch.search.aggregations.bucket.global.GlobalAggregationBuilder;
3537
import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
@@ -250,6 +252,13 @@ public static GeoHashGridAggregationBuilder geohashGrid(String name) {
250252
return new GeoHashGridAggregationBuilder(name);
251253
}
252254

255+
/**
256+
* Create a new {@link InternalQuadkeyGrid} aggregation with the given name.
257+
*/
258+
public static QuadkeyGridAggregationBuilder quadkeyGrid(String name) {
259+
return new QuadkeyGridAggregationBuilder(name);
260+
}
261+
253262
/**
254263
* Create a new {@link SignificantTerms} aggregation with the given name.
255264
*/

‎server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalGeoHashGrid.java

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,6 @@
3333
*/
3434
public class InternalGeoHashGrid extends InternalGeoGrid<InternalGeoHashGridBucket> {
3535

36-
private static final String NAME = "geohash_grid";
37-
3836
InternalGeoHashGrid(String name, int requiredSize, List<InternalGeoGridBucket> buckets,
3937
List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
4038
super(name, requiredSize, buckets, pipelineAggregators, metaData);
@@ -66,6 +64,6 @@ Reader getBucketReader() {
6664

6765
@Override
6866
public String getWriteableName() {
69-
return NAME;
67+
return GeoHashGridAggregationBuilder.NAME;
7068
}
7169
}

‎server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalQuadkeyGrid.java

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package org.elasticsearch.search.aggregations.bucket.geogrid;
20+
21+
import org.elasticsearch.common.io.stream.StreamInput;
22+
import org.elasticsearch.search.aggregations.InternalAggregations;
23+
import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
24+
25+
import java.io.IOException;
26+
import java.util.List;
27+
import java.util.Map;
28+
29+
/**
30+
* Represents a grid of cells where each cell's location is determined by a geohash.
31+
* All geohashes in a grid are of the same precision and held internally as a single long
32+
* for efficiency's sake.
33+
*/
34+
public class InternalQuadkeyGrid extends InternalGeoGrid<InternalQuadkeyGridBucket> {
35+
36+
InternalQuadkeyGrid(String name, int requiredSize, List<InternalGeoGridBucket> buckets,
37+
List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
38+
super(name, requiredSize, buckets, pipelineAggregators, metaData);
39+
}
40+
41+
public InternalQuadkeyGrid(StreamInput in) throws IOException {
42+
super(in);
43+
}
44+
45+
@Override
46+
public InternalGeoGrid create(List<InternalGeoGridBucket> buckets) {
47+
return new InternalQuadkeyGrid(name, requiredSize, buckets, pipelineAggregators(), metaData);
48+
}
49+
50+
@Override
51+
public InternalGeoGridBucket createBucket(InternalAggregations aggregations, InternalGeoGridBucket prototype) {
52+
return new InternalQuadkeyGridBucket(prototype.hashAsLong, prototype.docCount, aggregations);
53+
}
54+
55+
@Override
56+
InternalGeoGrid create(String name, int requiredSize, List buckets, List list, Map metaData) {
57+
return new InternalQuadkeyGrid(name, requiredSize, buckets, list, metaData);
58+
}
59+
60+
@Override
61+
Reader getBucketReader() {
62+
return InternalQuadkeyGridBucket::new;
63+
}
64+
65+
@Override
66+
public String getWriteableName() {
67+
return QuadkeyGridAggregationBuilder.NAME;
68+
}
69+
}

‎server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalQuadkeyGridBucket.java

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.elasticsearch.search.aggregations.bucket.geogrid;
21+
22+
import org.elasticsearch.common.geo.QuadkeyUtils;
23+
import org.elasticsearch.common.geo.GeoPoint;
24+
import org.elasticsearch.common.io.stream.StreamInput;
25+
import org.elasticsearch.search.aggregations.InternalAggregations;
26+
27+
import java.io.IOException;
28+
29+
public class InternalQuadkeyGridBucket extends InternalGeoGridBucket<InternalQuadkeyGridBucket> {
30+
InternalQuadkeyGridBucket(long geohashAsLong, long docCount, InternalAggregations aggregations) {
31+
super(geohashAsLong, docCount, aggregations);
32+
}
33+
34+
/**
35+
* Read from a stream.
36+
*/
37+
public InternalQuadkeyGridBucket(StreamInput in) throws IOException {
38+
super(in);
39+
}
40+
41+
@Override
42+
InternalQuadkeyGridBucket buildBucket(InternalGeoGridBucket bucket, long geoHashAsLong, long docCount,
43+
InternalAggregations aggregations) {
44+
return new InternalQuadkeyGridBucket(geoHashAsLong, docCount, aggregations);
45+
}
46+
47+
@Override
48+
public String getKeyAsString() {
49+
return QuadkeyUtils.stringEncode(geohashAsLong);
50+
}
51+
52+
@Override
53+
public GeoPoint getKey() {
54+
return GeoPoint.fromGeohash(geohashAsLong);
55+
}
56+
}

0 commit comments

Comments
 (0)