Skip to content

Commit 62a2937

Browse files
imotovimotov
authored
SQL: Add basic support for geo point (#31257)
Adds basic support for geo point type. For now, the geopoint is represented as a string and returned in the same format it was stored in the source. Relates to #29872
1 parent b512e55 commit 62a2937

File tree

11 files changed

+154
-33
lines changed

11 files changed

+154
-33
lines changed

‎x-pack/plugin/sql/sql-shared-proto/src/main/java/org/elasticsearch/xpack/sql/type/DataType.java

+4-2
Original file line numberDiff line numberDiff line change
@@ -42,14 +42,16 @@ public enum DataType {
4242
// see https://github.com/elastic/elasticsearch/issues/30386#issuecomment-386807288
4343
DATE( JDBCType.TIMESTAMP, Timestamp.class, Long.BYTES, 24, 24),
4444
// TODO: This should map to some Geography class instead of String
45-
GEO_SHAPE( JDBCType.OTHER, String.class, Integer.MAX_VALUE, Integer.MAX_VALUE, 0, false, false, false);
45+
GEO_SHAPE( JDBCType.OTHER, String.class, Integer.MAX_VALUE, Integer.MAX_VALUE, 0, false, false, false),
46+
GEO_POINT( JDBCType.OTHER, String.class, Double.BYTES*2, Integer.MAX_VALUE, 0, false, false, false);
4647
// @formatter:on
4748

4849
private static final Map<JDBCType, DataType> jdbcToEs;
4950

5051
static {
5152
jdbcToEs = Arrays.stream(DataType.values())
52-
.filter(dataType -> dataType != TEXT && dataType != NESTED && dataType != SCALED_FLOAT && dataType != GEO_SHAPE)
53+
.filter(dataType -> dataType != TEXT && dataType != NESTED && dataType != SCALED_FLOAT && dataType != GEO_SHAPE &&
54+
dataType != GEO_POINT)
5355
.collect(Collectors.toMap(dataType -> dataType.jdbcType, dataType -> dataType));
5456
}
5557

‎x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysParserTests.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -58,11 +58,11 @@ public void testSysTypes() throws Exception {
5858
Command cmd = sql("SYS TYPES").v1();
5959

6060
List<String> names = asList("BYTE", "LONG", "BINARY", "NULL", "INTEGER", "SHORT", "HALF_FLOAT", "SCALED_FLOAT", "FLOAT", "DOUBLE",
61-
"KEYWORD", "TEXT", "BOOLEAN", "DATE", "UNSUPPORTED", "GEO_SHAPE", "OBJECT", "NESTED");
61+
"KEYWORD", "TEXT", "BOOLEAN", "DATE", "UNSUPPORTED", "GEO_SHAPE", "GEO_POINT", "OBJECT", "NESTED");
6262

6363
cmd.execute(null, ActionListener.wrap(r -> {
6464
assertEquals(19, r.columnCount());
65-
assertEquals(18, r.size());
65+
assertEquals(names.size(), r.size());
6666
assertFalse(r.schema().types().contains(DataType.NULL));
6767
// test numeric as signed
6868
assertFalse(r.column(9, Boolean.class));

‎x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/TypesTests.java

+8-5
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515

1616
import static java.util.Collections.emptyMap;
1717
import static org.elasticsearch.xpack.sql.type.DataType.DATE;
18+
import static org.elasticsearch.xpack.sql.type.DataType.GEO_POINT;
1819
import static org.elasticsearch.xpack.sql.type.DataType.GEO_SHAPE;
1920
import static org.elasticsearch.xpack.sql.type.DataType.INTEGER;
2021
import static org.elasticsearch.xpack.sql.type.DataType.KEYWORD;
@@ -39,13 +40,12 @@ public void testEmptyMap() {
3940

4041
public void testBasicMapping() {
4142
Map<String, EsField> mapping = loadMapping("mapping-basic.json");
42-
assertThat(mapping.size(), is(7));
43+
assertThat(mapping.size(), is(6));
4344
assertThat(mapping.get("emp_no").getDataType(), is(INTEGER));
4445
assertThat(mapping.get("first_name"), instanceOf(TextEsField.class));
4546
assertThat(mapping.get("last_name").getDataType(), is(TEXT));
4647
assertThat(mapping.get("gender").getDataType(), is(KEYWORD));
4748
assertThat(mapping.get("salary").getDataType(), is(INTEGER));
48-
assertThat(mapping.get("site").getDataType(), is(GEO_SHAPE));
4949
}
5050

5151
public void testDefaultStringMapping() {
@@ -181,8 +181,11 @@ public void testNestedDoc() {
181181

182182
public void testGeoField() {
183183
Map<String, EsField> mapping = loadMapping("mapping-geo.json");
184-
EsField dt = mapping.get("location");
185-
assertThat(dt.getDataType().esType, is("unsupported"));
184+
assertThat(mapping.size(), is(2));
185+
EsField gp = mapping.get("location");
186+
assertThat(gp.getDataType().esType, is("geo_point"));
187+
EsField gs = mapping.get("site");
188+
assertThat(gs.getDataType().esType, is("geo_shape"));
186189
}
187190

188191
public void testUnsupportedTypes() {
@@ -202,4 +205,4 @@ public static Map<String, EsField> loadMapping(String name, boolean ordered) {
202205
assertNotNull("Could not find mapping resource:" + name, stream);
203206
return Types.fromEs(XContentHelper.convertToMap(JsonXContent.jsonXContent, stream, ordered));
204207
}
205-
}
208+
}

‎x-pack/plugin/sql/src/test/resources/mapping-basic.json

-3
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,6 @@
1717
},
1818
"salary" : {
1919
"type" : "integer"
20-
},
21-
"site": {
22-
"type": "geo_shape"
2320
}
2421
}
2522
}

‎x-pack/plugin/sql/src/test/resources/mapping-geo.json

+3
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,9 @@
22
"properties" : {
33
"location" : {
44
"type" : "geo_point"
5+
},
6+
"site": {
7+
"type" : "geo_shape"
58
}
69
}
710
}

‎x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/geo/GeoCsvSpecTestCase.java

+7-2
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
package org.elasticsearch.xpack.qa.sql.geo;
88

99
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
10+
import org.elasticsearch.client.Request;
1011
import org.elasticsearch.xpack.qa.sql.jdbc.CsvTestUtils.CsvTestCase;
1112
import org.elasticsearch.xpack.qa.sql.jdbc.SpecBaseIntegrationTestCase;
1213
import org.elasticsearch.xpack.sql.jdbc.jdbc.JdbcConfiguration;
@@ -34,6 +35,7 @@ public static List<Object[]> readScriptSpec() throws Exception {
3435
Parser parser = specParser();
3536
List<Object[]> tests = new ArrayList<>();
3637
tests.addAll(readScriptSpec("/ogc/ogc.csv-spec", parser));
38+
tests.addAll(readScriptSpec("/geo/geosql.csv-spec", parser));
3739
return tests;
3840
}
3941

@@ -45,8 +47,11 @@ public GeoCsvSpecTestCase(String fileName, String groupName, String testName, In
4547

4648
@Before
4749
public void setupTestGeoDataIfNeeded() throws Exception {
48-
if (client().performRequest("HEAD", "/ogc").getStatusLine().getStatusCode() == 404) {
49-
GeoDataLoader.loadDatasetIntoEs(client());
50+
if (client().performRequest(new Request("HEAD", "/ogc")).getStatusLine().getStatusCode() == 404) {
51+
GeoDataLoader.loadOGCDatasetIntoEs(client(), "ogc");
52+
}
53+
if (client().performRequest(new Request("HEAD", "/geo")).getStatusLine().getStatusCode() == 404) {
54+
GeoDataLoader.loadGeoDatasetIntoEs(client(), "geo");
5055
}
5156
}
5257

‎x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/geo/GeoDataLoader.java

+29-17
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
import org.apache.http.entity.ContentType;
1212
import org.apache.http.entity.StringEntity;
1313
import org.apache.http.util.EntityUtils;
14+
import org.elasticsearch.client.Request;
1415
import org.elasticsearch.client.Response;
1516
import org.elasticsearch.client.RestClient;
1617
import org.elasticsearch.common.Strings;
@@ -37,26 +38,28 @@ public class GeoDataLoader {
3738

3839
public static void main(String[] args) throws Exception {
3940
try (RestClient client = RestClient.builder(new HttpHost("localhost", 9200)).build()) {
40-
loadDatasetIntoEs(client);
41+
loadOGCDatasetIntoEs(client, "ogc");
42+
loadGeoDatasetIntoEs(client, "geo");
4143
Loggers.getLogger(GeoDataLoader.class).info("Geo data loaded");
4244
}
4345
}
4446

45-
protected static void loadDatasetIntoEs(RestClient client) throws Exception {
46-
loadDatasetIntoEs(client, "ogc");
47-
makeFilteredAlias(client, "lakes", "ogc", "\"term\" : { \"ogc_type\" : \"lakes\" }");
48-
makeFilteredAlias(client, "road_segments", "ogc", "\"term\" : { \"ogc_type\" : \"road_segments\" }");
49-
makeFilteredAlias(client, "divided_routes", "ogc", "\"term\" : { \"ogc_type\" : \"divided_routes\" }");
50-
makeFilteredAlias(client, "forests", "ogc", "\"term\" : { \"ogc_type\" : \"forests\" }");
51-
makeFilteredAlias(client, "bridges", "ogc", "\"term\" : { \"ogc_type\" : \"bridges\" }");
52-
makeFilteredAlias(client, "streams", "ogc", "\"term\" : { \"ogc_type\" : \"streams\" }");
53-
makeFilteredAlias(client, "buildings", "ogc", "\"term\" : { \"ogc_type\" : \"buildings\" }");
54-
makeFilteredAlias(client, "ponds", "ogc", "\"term\" : { \"ogc_type\" : \"ponds\" }");
55-
makeFilteredAlias(client, "named_places", "ogc", "\"term\" : { \"ogc_type\" : \"named_places\" }");
56-
makeFilteredAlias(client, "map_neatlines", "ogc", "\"term\" : { \"ogc_type\" : \"map_neatlines\" }");
47+
protected static void loadOGCDatasetIntoEs(RestClient client, String index) throws Exception {
48+
createIndex(client, index, createOGCIndexRequest());
49+
loadData(client, index, readResource("/ogc/ogc.json"));
50+
makeFilteredAlias(client, "lakes", index, "\"term\" : { \"ogc_type\" : \"lakes\" }");
51+
makeFilteredAlias(client, "road_segments", index, "\"term\" : { \"ogc_type\" : \"road_segments\" }");
52+
makeFilteredAlias(client, "divided_routes", index, "\"term\" : { \"ogc_type\" : \"divided_routes\" }");
53+
makeFilteredAlias(client, "forests", index, "\"term\" : { \"ogc_type\" : \"forests\" }");
54+
makeFilteredAlias(client, "bridges", index, "\"term\" : { \"ogc_type\" : \"bridges\" }");
55+
makeFilteredAlias(client, "streams", index, "\"term\" : { \"ogc_type\" : \"streams\" }");
56+
makeFilteredAlias(client, "buildings", index, "\"term\" : { \"ogc_type\" : \"buildings\" }");
57+
makeFilteredAlias(client, "ponds", index, "\"term\" : { \"ogc_type\" : \"ponds\" }");
58+
makeFilteredAlias(client, "named_places", index, "\"term\" : { \"ogc_type\" : \"named_places\" }");
59+
makeFilteredAlias(client, "map_neatlines", index, "\"term\" : { \"ogc_type\" : \"map_neatlines\" }");
5760
}
5861

59-
protected static void loadDatasetIntoEs(RestClient client, String index) throws Exception {
62+
private static String createOGCIndexRequest() throws Exception {
6063
XContentBuilder createIndex = JsonXContent.contentBuilder().startObject();
6164
createIndex.startObject("settings");
6265
{
@@ -101,12 +104,21 @@ protected static void loadDatasetIntoEs(RestClient client, String index) throws
101104
createIndex.endObject();
102105
}
103106
createIndex.endObject().endObject();
107+
return Strings.toString(createIndex);
108+
}
104109

105-
client.performRequest("PUT", "/" + index, emptyMap(), new StringEntity(Strings.toString(createIndex),
106-
ContentType.APPLICATION_JSON));
110+
private static void createIndex(RestClient client, String index, String settingsMappings) throws IOException {
111+
Request createIndexRequest = new Request("PUT", "/" + index);
112+
createIndexRequest.setEntity(new StringEntity(settingsMappings, ContentType.APPLICATION_JSON));
113+
client.performRequest(createIndexRequest);
114+
}
107115

108-
String bulk = readResource("/ogc/ogc.json");
116+
static void loadGeoDatasetIntoEs(RestClient client, String index) throws Exception {
117+
createIndex(client, index, readResource("/geo/geosql.json"));
118+
loadData(client, index, readResource("/geo/geosql-bulk.json"));
119+
}
109120

121+
private static void loadData(RestClient client, String index, String bulk) throws IOException {
110122
Response response = client.performRequest("POST", "/" + index + "/doc/_bulk", singletonMap("refresh", "true"),
111123
new StringEntity(bulk, ContentType.APPLICATION_JSON));
112124

‎x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/geo/GeoSqlSpecTestCase.java

+3-2
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
package org.elasticsearch.xpack.qa.sql.geo;
77

88
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
9+
import org.elasticsearch.client.Request;
910
import org.elasticsearch.xpack.qa.sql.jdbc.LocalH2;
1011
import org.elasticsearch.xpack.qa.sql.jdbc.SpecBaseIntegrationTestCase;
1112
import org.elasticsearch.xpack.sql.jdbc.jdbc.JdbcConfiguration;
@@ -47,8 +48,8 @@ public static List<Object[]> readScriptSpec() throws Exception {
4748
public void setupTestGeoDataIfNeeded() throws Exception {
4849
assumeTrue("Cannot support locales that don't use Hindu-Arabic numerals due to H2",
4950
"42".equals(NumberFormat.getInstance(Locale.getDefault()).format(42)));
50-
if (client().performRequest("HEAD", "/ogc").getStatusLine().getStatusCode() == 404) {
51-
GeoDataLoader.loadDatasetIntoEs(client());
51+
if (client().performRequest(new Request("HEAD", "/ogc")).getStatusLine().getStatusCode() == 404) {
52+
GeoDataLoader.loadOGCDatasetIntoEs(client(), "ogc");
5253
}
5354
}
5455

‎x-pack/qa/sql/src/main/resources/geo/geosql-bulk.json

+33
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
{"index":{"_id": "1"}}
2+
{"region": "Americas", "city": "Mountain View", "location": "37.386483,-122.083843"}
3+
{"index":{"_id": "2"}}
4+
{"region": "Americas", "city": "Chicago", "location": "41.888783,-87.637874"}
5+
{"index":{"_id": "3"}}
6+
{"region": "Americas", "city": "New York", "location": "40.745171,-73.990027"}
7+
{"index":{"_id": "4"}}
8+
{"region": "Americas", "city": "San Francisco", "location": "37.789541,-122.394228"}
9+
{"index":{"_id": "5"}}
10+
{"region": "Americas", "city": "Phoenix", "location": "33.376242,-111.973505"}
11+
{"index":{"_id": "6"}}
12+
{"region": "Europe", "city": "Amsterdam", "location": "52.347557,4.850312"}
13+
{"index":{"_id": "7"}}
14+
{"region": "Europe", "city": "Berlin", "location": "52.486701,13.390889"}
15+
{"index":{"_id": "8"}}
16+
{"region": "Europe", "city": "Munich", "location": "48.146321,11.537505"}
17+
{"index":{"_id": "9"}}
18+
{"region": "Europe", "city": "London", "location": "51.510871,-0.121672"}
19+
{"index":{"_id": "10"}}
20+
{"region": "Europe", "city": "Paris", "location": "48.845538,2.351773"}
21+
{"index":{"_id": "11"}}
22+
{"region": "Asia", "city": "Singapore", "location": "1.295868,103.855535"}
23+
{"index":{"_id": "12"}}
24+
{"region": "Asia", "city": "Hong Kong", "location": "22.281397,114.183925"}
25+
{"index":{"_id": "13"}}
26+
{"region": "Asia", "city": "Seoul", "location": "37.509132,127.060851"}
27+
{"index":{"_id": "14"}}
28+
{"region": "Asia", "city": "Tokyo", "location": "35.669616,139.76402225"}
29+
{"index":{"_id": "15"}}
30+
{"region": "Asia", "city": "Sydney", "location": "-33.863385,151.208629"}
31+
32+
33+

‎x-pack/qa/sql/src/main/resources/geo/geosql.csv-spec

+45
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
//
2+
// Commands on geo test data
3+
//
4+
5+
showTables
6+
SHOW TABLES 'geo';
7+
8+
name:s | type:s
9+
geo |BASE TABLE
10+
;
11+
12+
// DESCRIBE
13+
14+
describe
15+
DESCRIBE "geo";
16+
17+
column:s | type:s
18+
city | VARCHAR
19+
location | OTHER
20+
region | VARCHAR
21+
;
22+
23+
// SELECT ALL
24+
// TODO: For now we just get geopoint formatted as is and we also need to convert it to STRING to work with CSV
25+
26+
selectAllPoints
27+
SELECT city, CAST(location AS STRING) location, region FROM "geo" ORDER BY "city";
28+
29+
city:s | location:s | region:s
30+
Amsterdam |52.347557,4.850312 |Europe
31+
Berlin |52.486701,13.390889 |Europe
32+
Chicago |41.888783,-87.637874 |Americas
33+
Hong Kong |22.281397,114.183925 |Asia
34+
London |51.510871,-0.121672 |Europe
35+
Mountain View |37.386483,-122.083843 |Americas
36+
Munich |48.146321,11.537505 |Europe
37+
New York |40.745171,-73.990027 |Americas
38+
Paris |48.845538,2.351773 |Europe
39+
Phoenix |33.376242,-111.973505 |Americas
40+
San Francisco |37.789541,-122.394228 |Americas
41+
Seoul |37.509132,127.060851 |Asia
42+
Singapore |1.295868,103.855535 |Asia
43+
Sydney |-33.863385,151.208629 |Asia
44+
Tokyo |35.669616,139.76402225|Asia
45+
;

‎x-pack/qa/sql/src/main/resources/geo/geosql.json

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
{
2+
"settings": {
3+
"number_of_shards": 1
4+
},
5+
"mappings": {
6+
"doc": {
7+
"properties": {
8+
"region": {
9+
"type": "keyword"
10+
},
11+
"city": {
12+
"type": "keyword"
13+
},
14+
"location": {
15+
"type": "geo_point"
16+
}
17+
}
18+
}
19+
}
20+
}

0 commit comments

Comments
 (0)