Skip to content

Commit 1157c11

Browse files
imotovimotov
committed
Sql add support for geo shape type (#4204)
Adds initial very minimal support for geo_shapes to SQL. For now, geoshapes are converted into String instead of Geometry objects on the JDBC side and no effort to parse the result is performed. Relates #4080
1 parent 54122d8 commit 1157c11

File tree

24 files changed

+1286
-30
lines changed

24 files changed

+1286
-30
lines changed

‎x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/jdbc/JdbcResultSet.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -367,9 +367,9 @@ private <T> T convert(int columnIndex, Class<T> type) throws SQLException {
367367
}
368368
}
369369

370-
JDBCType columnType = cursor.columns().get(columnIndex - 1).type;
371-
372-
return TypeConverter.convert(val, columnType, type);
370+
ColumnInfo columnInfo = cursor.columns().get(columnIndex - 1);
371+
372+
return TypeConverter.convert(val, columnInfo.type, columnInfo.esType, type);
373373
}
374374

375375
@Override

‎x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/jdbc/TypeConverter.java

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -99,12 +99,12 @@ private static <T> T dateTimeConvert(Long millis, Calendar c, Function<Calendar,
9999
* Converts object val from columnType to type
100100
*/
101101
@SuppressWarnings("unchecked")
102-
static <T> T convert(Object val, JDBCType columnType, Class<T> type) throws SQLException {
102+
static <T> T convert(Object val, JDBCType columnType, String esType, Class<T> type) throws SQLException {
103103
if (type == null) {
104-
return (T) convert(val, columnType);
104+
return (T) convert(val, columnType, esType);
105105
}
106106
if (type == String.class) {
107-
return (T) asString(convert(val, columnType));
107+
return (T) asString(convert(val, columnType, esType));
108108
}
109109
if (type == Boolean.class) {
110110
return (T) asBoolean(val, columnType);
@@ -185,7 +185,7 @@ public static String classNameOf(JDBCType jdbcType) throws JdbcSQLException {
185185
* <p>
186186
* The returned types needs to correspond to ES-portion of classes returned by {@link TypeConverter#classNameOf}
187187
*/
188-
static Object convert(Object v, JDBCType columnType) throws SQLException {
188+
static Object convert(Object v, JDBCType columnType, String esType) throws SQLException {
189189
switch (columnType) {
190190
case NULL:
191191
return null;
@@ -207,9 +207,20 @@ static Object convert(Object v, JDBCType columnType) throws SQLException {
207207
return floatValue(v); // Float might be represented as string for infinity and NaN values
208208
case TIMESTAMP:
209209
return ((Number) v).longValue();
210+
case OTHER:
211+
return convertOtherType(esType, v);
210212
default:
211213
throw new SQLException("Unexpected column type [" + columnType.getName() + "]");
214+
}
215+
}
212216

217+
static Object convertOtherType(String esType, Object v) throws SQLException {
218+
switch (esType) {
219+
case "geo_shape":
220+
// TODO: convert this into a geo object
221+
return v;
222+
default:
223+
throw new SQLException("Unsupported es type [" + esType + "]");
213224
}
214225
}
215226

‎x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/net/client/JdbcHttpClient.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,8 @@ private InfoResponse fetchServerInfo() throws SQLException {
8686
*/
8787
private List<ColumnInfo> toJdbcColumnInfo(List<org.elasticsearch.xpack.sql.plugin.ColumnInfo> columns) {
8888
return columns.stream().map(columnInfo ->
89-
new ColumnInfo(columnInfo.name(), columnInfo.jdbcType(), EMPTY, EMPTY, EMPTY, EMPTY, columnInfo.displaySize())
89+
new ColumnInfo(columnInfo.name(), columnInfo.jdbcType(), EMPTY, EMPTY, EMPTY, EMPTY, columnInfo.displaySize(),
90+
columnInfo.esType())
9091
).collect(Collectors.toList());
9192
}
9293
}

‎x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/net/protocol/ColumnInfo.java

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@
55
*/
66
package org.elasticsearch.xpack.sql.jdbc.net.protocol;
77

8+
import org.elasticsearch.xpack.sql.type.DataType;
9+
810
import java.sql.JDBCType;
911
import java.util.Objects;
1012

@@ -16,8 +18,14 @@ public class ColumnInfo {
1618
public final String name;
1719
public final int displaySize;
1820
public final JDBCType type;
21+
public final String esType;
1922

2023
public ColumnInfo(String name, JDBCType type, String table, String catalog, String schema, String label, int displaySize) {
24+
this(name, type, table, catalog, schema, label, displaySize, DataType.fromJdbcType(type).esType);
25+
}
26+
27+
public ColumnInfo(String name, JDBCType type, String table, String catalog, String schema, String label, int displaySize,
28+
String esType) {
2129
if (name == null) {
2230
throw new IllegalArgumentException("[name] must not be null");
2331
}
@@ -36,13 +44,17 @@ public ColumnInfo(String name, JDBCType type, String table, String catalog, Stri
3644
if (label == null) {
3745
throw new IllegalArgumentException("[label] must not be null");
3846
}
47+
if (esType == null) {
48+
throw new IllegalArgumentException("[esType] must not be null");
49+
}
3950
this.name = name;
4051
this.type = type;
4152
this.table = table;
4253
this.catalog = catalog;
4354
this.schema = schema;
4455
this.label = label;
4556
this.displaySize = displaySize;
57+
this.esType = esType;
4658
}
4759

4860
public int displaySize() {
@@ -81,11 +93,12 @@ public boolean equals(Object obj) {
8193
&& catalog.equals(other.catalog)
8294
&& schema.equals(other.schema)
8395
&& label.equals(other.label)
84-
&& displaySize == other.displaySize;
96+
&& displaySize == other.displaySize
97+
&& esType.equals(other.esType);
8598
}
8699

87100
@Override
88101
public int hashCode() {
89-
return Objects.hash(name, type, table, catalog, schema, label, displaySize);
102+
return Objects.hash(name, type, table, catalog, schema, label, displaySize, esType);
90103
}
91104
}

‎x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/jdbc/TypeConverterTests.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import org.elasticsearch.test.ESTestCase;
1313
import org.elasticsearch.xpack.sql.plugin.AbstractSqlRequest;
1414
import org.elasticsearch.xpack.sql.plugin.SqlQueryResponse;
15+
import org.elasticsearch.xpack.sql.type.DataType;
1516
import org.joda.time.DateTime;
1617

1718
import java.sql.JDBCType;
@@ -55,7 +56,7 @@ private Object convertAsNative(Object value, JDBCType type) throws Exception {
5556
builder.endObject();
5657
builder.close();
5758
Object copy = XContentHelper.convertToMap(BytesReference.bytes(builder), false, builder.contentType()).v2().get("value");
58-
return TypeConverter.convert(copy, type);
59+
return TypeConverter.convert(copy, type, DataType.fromJdbcType(type).esType);
5960
}
6061

6162
}

‎x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/type/DataType.java

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,14 +37,16 @@ public enum DataType {
3737
OBJECT( JDBCType.STRUCT, null, -1, 0, 0),
3838
NESTED( JDBCType.STRUCT, null, -1, 0, 0),
3939
BINARY( JDBCType.VARBINARY, byte[].class, -1, Integer.MAX_VALUE, 0),
40-
DATE( JDBCType.TIMESTAMP, Timestamp.class, Long.BYTES, 19, 20);
40+
DATE( JDBCType.TIMESTAMP, Timestamp.class, Long.BYTES, 19, 20),
41+
// TODO: This should map to some Geography class instead of String
42+
GEO_SHAPE( JDBCType.OTHER, String.class, Integer.MAX_VALUE, Integer.MAX_VALUE, 0, false, false, false);
4143
// @formatter:on
4244

4345
private static final Map<JDBCType, DataType> jdbcToEs;
4446

4547
static {
4648
jdbcToEs = Arrays.stream(DataType.values())
47-
.filter(dataType -> dataType != TEXT && dataType != NESTED && dataType != SCALED_FLOAT) // Remove duplicates
49+
.filter(dataType -> dataType != TEXT && dataType != NESTED && dataType != SCALED_FLOAT && dataType != GEO_SHAPE)
4850
.collect(Collectors.toMap(dataType -> dataType.jdbcType, dataType -> dataType));
4951
}
5052

‎x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysParserTests.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -58,11 +58,11 @@ public void testSysTypes() throws Exception {
5858
Command cmd = sql("SYS TYPES").v1();
5959

6060
List<String> names = asList("BYTE", "SHORT", "INTEGER", "LONG", "HALF_FLOAT", "SCALED_FLOAT", "FLOAT", "DOUBLE", "KEYWORD", "TEXT",
61-
"DATE", "BINARY", "NULL", "UNSUPPORTED", "OBJECT", "NESTED", "BOOLEAN");
61+
"DATE", "BINARY", "NULL", "UNSUPPORTED", "GEO_SHAPE", "OBJECT", "NESTED", "BOOLEAN");
6262

6363
cmd.execute(null, ActionListener.wrap(r -> {
6464
assertEquals(19, r.columnCount());
65-
assertEquals(17, r.size());
65+
assertEquals(18, r.size());
6666
assertFalse(r.schema().types().contains(DataType.NULL));
6767
// test numeric as signed
6868
assertFalse(r.column(9, Boolean.class));

‎x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/TypesTests.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515

1616
import static java.util.Collections.emptyMap;
1717
import static org.elasticsearch.xpack.sql.type.DataType.DATE;
18+
import static org.elasticsearch.xpack.sql.type.DataType.GEO_SHAPE;
1819
import static org.elasticsearch.xpack.sql.type.DataType.INTEGER;
1920
import static org.elasticsearch.xpack.sql.type.DataType.KEYWORD;
2021
import static org.elasticsearch.xpack.sql.type.DataType.NESTED;
@@ -38,12 +39,13 @@ public void testEmptyMap() {
3839

3940
public void testBasicMapping() {
4041
Map<String, EsField> mapping = loadMapping("mapping-basic.json");
41-
assertThat(mapping.size(), is(6));
42+
assertThat(mapping.size(), is(7));
4243
assertThat(mapping.get("emp_no").getDataType(), is(INTEGER));
4344
assertThat(mapping.get("first_name"), instanceOf(TextEsField.class));
4445
assertThat(mapping.get("last_name").getDataType(), is(TEXT));
4546
assertThat(mapping.get("gender").getDataType(), is(KEYWORD));
4647
assertThat(mapping.get("salary").getDataType(), is(INTEGER));
48+
assertThat(mapping.get("site").getDataType(), is(GEO_SHAPE));
4749
}
4850

4951
public void testDefaultStringMapping() {

‎x-pack/plugin/sql/src/test/resources/mapping-basic.json

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,9 @@
1717
},
1818
"salary" : {
1919
"type" : "integer"
20+
},
21+
"site": {
22+
"type": "geo_shape"
2023
}
2124
}
2225
}

‎x-pack/qa/sql/build.gradle

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ dependencies {
1515
// CLI testing dependencies
1616
compile project(path: xpackModule('sql:sql-cli'), configuration: 'nodeps')
1717
compile "org.jline:jline:3.6.0"
18+
compile "org.orbisgis:h2gis-ext:1.3.2"
1819
}
1920

2021
/* disable unit tests because these are all integration tests used
@@ -69,6 +70,9 @@ thirdPartyAudit.excludes = [
6970
'org.fusesource.jansi.internal.Kernel32$SMALL_RECT',
7071
'org.fusesource.jansi.internal.Kernel32',
7172
'org.fusesource.jansi.internal.WindowsSupport',
73+
// TODO: Temporary solution until core removes JTS as a dependency
74+
'org.h2gis.functions.factory.H2GISFunctions',
75+
'org.h2gis.network.functions.NetworkFunctions',
7276
'org.mozilla.universalchardet.UniversalDetector',
7377
]
7478

@@ -85,9 +89,16 @@ subprojects {
8589
testCompile "org.elasticsearch.test:framework:${version}"
8690

8791
// JDBC testing dependencies
88-
testRuntime "net.sourceforge.csvjdbc:csvjdbc:1.0.34"
89-
testRuntime "com.h2database:h2:1.4.197"
9092
testRuntime xpackProject('plugin:sql:jdbc')
93+
testRuntime("net.sourceforge.csvjdbc:csvjdbc:1.0.34") {
94+
transitive = false
95+
}
96+
97+
testRuntime("com.h2database:h2:1.4.196") {
98+
transitive = false
99+
}
100+
101+
testRuntime "org.orbisgis:h2gis-ext:1.3.2"
91102

92103
// TODO check if needed
93104
testRuntime("org.antlr:antlr4-runtime:4.5.3") {
@@ -99,6 +110,16 @@ subprojects {
99110
testRuntime "org.jline:jline:3.6.0"
100111
}
101112

113+
// TODO: Temporary solution until core removes JTS as a dependency
114+
configurations {
115+
testRuntime {
116+
exclude group: 'org.locationtech.jts', module: 'jts-core'
117+
}
118+
testCompile {
119+
exclude group: 'org.locationtech.jts', module: 'jts-core'
120+
}
121+
}
122+
102123
if (project.name != 'security') {
103124
// The security project just configures its subprojects
104125
apply plugin: 'elasticsearch.rest-test'

‎x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/GeoJdbcCsvSpecIT.java

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
7+
package org.elasticsearch.xpack.qa.sql.nosecurity;
8+
9+
import org.elasticsearch.xpack.qa.sql.geo.GeoCsvSpecTestCase;
10+
import org.elasticsearch.xpack.qa.sql.jdbc.CsvTestUtils.CsvTestCase;
11+
12+
public class GeoJdbcCsvSpecIT extends GeoCsvSpecTestCase {
13+
public GeoJdbcCsvSpecIT(String fileName, String groupName, String testName, Integer lineNumber, CsvTestCase testCase) {
14+
super(fileName, groupName, testName, lineNumber, testCase);
15+
}
16+
}

‎x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/GeoJdbcSqlSpecIT.java

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
7+
package org.elasticsearch.xpack.qa.sql.nosecurity;
8+
9+
import org.elasticsearch.xpack.qa.sql.geo.GeoSqlSpecTestCase;
10+
11+
public class GeoJdbcSqlSpecIT extends GeoSqlSpecTestCase {
12+
public GeoJdbcSqlSpecIT(String fileName, String groupName, String testName, Integer lineNumber, String query) {
13+
super(fileName, groupName, testName, lineNumber, query);
14+
}
15+
}

‎x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/geo/GeoCsvSpecTestCase.java

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
7+
package org.elasticsearch.xpack.qa.sql.geo;
8+
9+
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
10+
import org.elasticsearch.xpack.qa.sql.jdbc.CsvTestUtils.CsvTestCase;
11+
import org.elasticsearch.xpack.qa.sql.jdbc.SpecBaseIntegrationTestCase;
12+
import org.elasticsearch.xpack.sql.jdbc.jdbc.JdbcConfiguration;
13+
import org.junit.Before;
14+
15+
import java.sql.Connection;
16+
import java.sql.ResultSet;
17+
import java.util.ArrayList;
18+
import java.util.List;
19+
import java.util.Properties;
20+
21+
import static org.elasticsearch.xpack.qa.sql.jdbc.CsvTestUtils.csvConnection;
22+
import static org.elasticsearch.xpack.qa.sql.jdbc.CsvTestUtils.executeCsvQuery;
23+
import static org.elasticsearch.xpack.qa.sql.jdbc.CsvTestUtils.specParser;
24+
25+
/**
26+
* Tests comparing sql queries executed against our jdbc client
27+
* with hard coded result sets.
28+
*/
29+
public abstract class GeoCsvSpecTestCase extends SpecBaseIntegrationTestCase {
30+
private final CsvTestCase testCase;
31+
32+
@ParametersFactory(argumentFormatting = PARAM_FORMATTING)
33+
public static List<Object[]> readScriptSpec() throws Exception {
34+
Parser parser = specParser();
35+
List<Object[]> tests = new ArrayList<>();
36+
tests.addAll(readScriptSpec("/ogc/ogc.csv-spec", parser));
37+
return tests;
38+
}
39+
40+
public GeoCsvSpecTestCase(String fileName, String groupName, String testName, Integer lineNumber, CsvTestCase testCase) {
41+
super(fileName, groupName, testName, lineNumber);
42+
this.testCase = testCase;
43+
}
44+
45+
46+
@Before
47+
public void setupTestGeoDataIfNeeded() throws Exception {
48+
if (client().performRequest("HEAD", "/ogc").getStatusLine().getStatusCode() == 404) {
49+
GeoDataLoader.loadDatasetIntoEs(client());
50+
}
51+
}
52+
53+
@Override
54+
protected final void doTest() throws Throwable {
55+
try (Connection csv = csvConnection(testCase.expectedResults);
56+
Connection es = esJdbc()) {
57+
58+
// pass the testName as table for debugging purposes (in case the underlying reader is missing)
59+
ResultSet expected = executeCsvQuery(csv, testName);
60+
ResultSet elasticResults = executeJdbcQuery(es, testCase.query);
61+
assertResults(expected, elasticResults);
62+
}
63+
}
64+
65+
// make sure ES uses UTC (otherwise JDBC driver picks up the JVM timezone per spec/convention)
66+
@Override
67+
protected Properties connectionProperties() {
68+
Properties connectionProperties = new Properties();
69+
connectionProperties.setProperty(JdbcConfiguration.TIME_ZONE, "UTC");
70+
return connectionProperties;
71+
}
72+
73+
}

0 commit comments

Comments
 (0)