Skip to content

Commit f707fa9

Browse files
authored
SQL: Introduce SQL DATE data type (#37693)
* SQL: Introduce SQL DATE data type Support ANSI SQL's DATE type by introducing a runtime-only ES SQL date type. Closes: #37340
1 parent b6317ed commit f707fa9

File tree

38 files changed

+657
-151
lines changed

38 files changed

+657
-151
lines changed

docs/reference/sql/functions/grouping.asciidoc

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,3 +76,9 @@ Instead one can rewrite the query to move the expression on the histogram _insid
7676
----
7777
include-tagged::{sql-specs}/docs.csv-spec[histogramDateTimeExpression]
7878
----
79+
80+
[IMPORTANT]
81+
When the histogram in SQL is applied on **DATE** type instead of **DATETIME**, the interval specified is truncated to
82+
the multiple of a day. E.g.: for `HISTOGRAM(CAST(birth_date AS DATE), INTERVAL '2 3:04' DAY TO MINUTE)` the interval
83+
actually used will be `INTERVAL '2' DAY`. If the interval specified is less than 1 day, e.g.:
84+
`HISTOGRAM(CAST(birth_date AS DATE), INTERVAL '20' HOUR)` then the interval used will be `INTERVAL '1' DAY`.

docs/reference/sql/language/data-types.asciidoc

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,6 @@
55

66
beta[]
77

8-
Most of {es} <<mapping-types, data types>> are available in {es-sql}, as indicated below.
9-
As one can see, all of {es} <<mapping-types, data types>> are mapped to the data type with the same
10-
name in {es-sql}, with the exception of **date** data type which is mapped to **datetime** in {es-sql}:
118

129
[cols="^,^m,^,^"]
1310

@@ -46,13 +43,22 @@ s|SQL precision
4643

4744
|===
4845

46+
[NOTE]
47+
Most of {es} <<mapping-types, data types>> are available in {es-sql}, as indicated above.
48+
As one can see, all of {es} <<mapping-types, data types>> are mapped to the data type with the same
49+
name in {es-sql}, with the exception of **date** data type which is mapped to **datetime** in {es-sql}.
50+
This is to avoid confusion with the ANSI SQL **DATE** (date only) type, which is also supported by {es-sql}
51+
in queries (with the use of <<sql-functions-type-conversion-cast>>/<<sql-functions-type-conversion-convert>>),
52+
but doesn't correspond to an actual mapping in {es} (see the <<es-sql-only-types, `table`>> below).
4953

5054
Obviously, not all types in {es} have an equivalent in SQL and vice-versa hence why, {es-sql}
5155
uses the data type _particularities_ of the former over the latter as ultimately {es} is the backing store.
5256

5357
In addition to the types above, {es-sql} also supports at _runtime_ SQL-specific types that do not have an equivalent in {es}.
5458
Such types cannot be loaded from {es} (as it does not know about them) however can be used inside {es-sql} in queries or their results.
5559

60+
[[es-sql-only-types]]
61+
5662
The table below indicates these types:
5763

5864
[cols="^m,^"]
@@ -62,6 +68,7 @@ s|SQL type
6268
s|SQL precision
6369

6470

71+
| date | 24
6572
| interval_year | 7
6673
| interval_month | 7
6774
| interval_day | 23

x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/EsType.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ public enum EsType implements SQLType {
2828
OBJECT( Types.STRUCT),
2929
NESTED( Types.STRUCT),
3030
BINARY( Types.VARBINARY),
31+
DATE( Types.DATE),
3132
DATETIME( Types.TIMESTAMP),
3233
IP( Types.VARCHAR),
3334
INTERVAL_YEAR( ExtraTypes.INTERVAL_YEAR),

x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcDateUtils.java

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -41,10 +41,9 @@ final class JdbcDateUtils {
4141
.appendFraction(MILLI_OF_SECOND, 3, 3, true)
4242
.appendOffsetId()
4343
.toFormatter(Locale.ROOT);
44-
44+
4545
static long asMillisSinceEpoch(String date) {
46-
ZonedDateTime zdt = ISO_WITH_MILLIS.parse(date, ZonedDateTime::from);
47-
return zdt.toInstant().toEpochMilli();
46+
return ISO_WITH_MILLIS.parse(date, ZonedDateTime::from).toInstant().toEpochMilli();
4847
}
4948

5049
static Date asDate(String date) {
@@ -71,7 +70,7 @@ static <R> R asDateTimeField(Object value, Function<String, R> asDateTimeMethod,
7170
}
7271
}
7372

74-
private static long utcMillisRemoveTime(long l) {
73+
static long utcMillisRemoveTime(long l) {
7574
return l - (l % DAY_IN_MILLIS);
7675
}
7776

x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcResultSet.java

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,9 @@
3333
import java.util.function.Function;
3434

3535
import static java.lang.String.format;
36+
import static org.elasticsearch.xpack.sql.jdbc.JdbcDateUtils.asDateTimeField;
37+
import static org.elasticsearch.xpack.sql.jdbc.JdbcDateUtils.asMillisSinceEpoch;
38+
import static org.elasticsearch.xpack.sql.jdbc.JdbcDateUtils.utcMillisRemoveTime;
3639

3740
class JdbcResultSet implements ResultSet, JdbcWrapper {
3841

@@ -252,8 +255,11 @@ private Long dateTime(int columnIndex) throws SQLException {
252255
if (val == null) {
253256
return null;
254257
}
255-
return JdbcDateUtils.asDateTimeField(val, JdbcDateUtils::asMillisSinceEpoch, Function.identity());
256-
};
258+
return asDateTimeField(val, JdbcDateUtils::asMillisSinceEpoch, Function.identity());
259+
}
260+
if (EsType.DATE == type) {
261+
return utcMillisRemoveTime(asMillisSinceEpoch(val.toString()));
262+
}
257263
return val == null ? null : (Long) val;
258264
} catch (ClassCastException cce) {
259265
throw new SQLException(

x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeConverter.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -213,6 +213,8 @@ static Object convert(Object v, EsType columnType, String typeString) throws SQL
213213
return doubleValue(v); // Double might be represented as string for infinity and NaN values
214214
case FLOAT:
215215
return floatValue(v); // Float might be represented as string for infinity and NaN values
216+
case DATE:
217+
return JdbcDateUtils.asDateTimeField(v, JdbcDateUtils::asDate, Date::new);
216218
case DATETIME:
217219
return JdbcDateUtils.asDateTimeField(v, JdbcDateUtils::asTimestamp, Timestamp::new);
218220
case INTERVAL_YEAR:

x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/CsvSpecTestCase.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ public static List<Object[]> readScriptSpec() throws Exception {
3636
tests.addAll(readScriptSpec("/fulltext.csv-spec", parser));
3737
tests.addAll(readScriptSpec("/agg.csv-spec", parser));
3838
tests.addAll(readScriptSpec("/columns.csv-spec", parser));
39+
tests.addAll(readScriptSpec("/date.csv-spec", parser));
3940
tests.addAll(readScriptSpec("/datetime.csv-spec", parser));
4041
tests.addAll(readScriptSpec("/alias.csv-spec", parser));
4142
tests.addAll(readScriptSpec("/null.csv-spec", parser));

x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcAssert.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,7 @@ public static void assertResultSetMetadata(ResultSet expected, ResultSet actual,
139139
if (expectedType == Types.TIMESTAMP_WITH_TIMEZONE) {
140140
expectedType = Types.TIMESTAMP;
141141
}
142+
142143
// since csv doesn't support real, we use float instead.....
143144
if (expectedType == Types.FLOAT && expected instanceof CsvResultSet) {
144145
expectedType = Types.REAL;
@@ -204,6 +205,9 @@ private static void doAssertResultSetData(ResultSet expected, ResultSet actual,
204205
// fix for CSV which returns the shortName not fully-qualified name
205206
if (!columnClassName.contains(".")) {
206207
switch (columnClassName) {
208+
case "Date":
209+
columnClassName = "java.sql.Date";
210+
break;
207211
case "Timestamp":
208212
columnClassName = "java.sql.Timestamp";
209213
break;
Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
//
2+
// Date
3+
//
4+
5+
dateExtractDateParts
6+
SELECT
7+
DAY(CAST(birth_date AS DATE)) d,
8+
DAY_OF_MONTH(CAST(birth_date AS DATE)) dm,
9+
DAY_OF_WEEK(CAST(birth_date AS DATE)) dw,
10+
DAY_OF_YEAR(CAST(birth_date AS DATE)) dy,
11+
ISO_DAY_OF_WEEK(CAST(birth_date AS DATE)) iso_dw,
12+
WEEK(CAST(birth_date AS DATE)) w,
13+
IW(CAST(birth_date AS DATE)) iso_w,
14+
QUARTER(CAST(birth_date AS DATE)) q,
15+
YEAR(CAST(birth_date AS DATE)) y,
16+
birth_date, last_name l FROM "test_emp" WHERE emp_no < 10010 ORDER BY emp_no;
17+
18+
d:i | dm:i | dw:i | dy:i | iso_dw:i | w:i |iso_w:i | q:i | y:i | birth_date:ts | l:s
19+
2 |2 |4 |245 |3 |36 |35 |3 |1953 |1953-09-02T00:00:00Z |Facello
20+
2 |2 |3 |154 |2 |23 |22 |2 |1964 |1964-06-02T00:00:00Z |Simmel
21+
3 |3 |5 |337 |4 |49 |49 |4 |1959 |1959-12-03T00:00:00Z |Bamford
22+
1 |1 |7 |121 |6 |18 |18 |2 |1954 |1954-05-01T00:00:00Z |Koblick
23+
21 |21 |6 |21 |5 |4 |3 |1 |1955 |1955-01-21T00:00:00Z |Maliniak
24+
20 |20 |2 |110 |1 |17 |16 |2 |1953 |1953-04-20T00:00:00Z |Preusig
25+
23 |23 |5 |143 |4 |21 |21 |2 |1957 |1957-05-23T00:00:00Z |Zielinski
26+
19 |19 |4 |50 |3 |8 |8 |1 |1958 |1958-02-19T00:00:00Z |Kalloufi
27+
19 |19 |7 |110 |6 |16 |16 |2 |1952 |1952-04-19T00:00:00Z |Peac
28+
;
29+
30+
31+
dateExtractTimePartsTimeSecond
32+
SELECT
33+
SECOND(CAST(birth_date AS DATE)) d,
34+
MINUTE(CAST(birth_date AS DATE)) m,
35+
HOUR(CAST(birth_date AS DATE)) h
36+
FROM "test_emp" WHERE emp_no < 10010 ORDER BY emp_no;
37+
38+
d:i | m:i | h:i
39+
0 |0 |0
40+
0 |0 |0
41+
0 |0 |0
42+
0 |0 |0
43+
0 |0 |0
44+
0 |0 |0
45+
0 |0 |0
46+
0 |0 |0
47+
0 |0 |0
48+
;
49+
50+
dateAsFilter
51+
SELECT birth_date, last_name FROM "test_emp" WHERE birth_date <= CAST('1955-01-21' AS DATE) ORDER BY emp_no LIMIT 5;
52+
53+
birth_date:ts | last_name:s
54+
1953-09-02T00:00:00Z |Facello
55+
1954-05-01T00:00:00Z |Koblick
56+
1955-01-21T00:00:00Z |Maliniak
57+
1953-04-20T00:00:00Z |Preusig
58+
1952-04-19T00:00:00Z |Peac
59+
;
60+
61+
dateAndFunctionAsGroupingKey
62+
SELECT MONTH(CAST(birth_date AS DATE)) AS m, CAST(SUM(emp_no) AS INT) s FROM test_emp GROUP BY m ORDER BY m LIMIT 5;
63+
64+
m:i | s:i
65+
null |100445
66+
1 |60288
67+
2 |80388
68+
3 |20164
69+
4 |80401
70+
;
71+
72+
dateAndInterval
73+
SELECT YEAR(CAST('2019-01-21' AS DATE) + INTERVAL '1-2' YEAR TO MONTH) AS y, MONTH(INTERVAL '1-2' YEAR TO MONTH + CAST('2019-01-21' AS DATE)) AS m;
74+
75+
y:i | m:i
76+
2020 | 3
77+
;

x-pack/plugin/sql/qa/src/main/resources/datetime.sql-spec

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,9 @@
66
// Time NOT IMPLEMENTED in H2 on TIMESTAMP WITH TIME ZONE - hence why these are moved to CSV
77
//
88

9-
// WEEK_OF_YEAR moved to CSV tests, because H2 builds its Calendar with the local Locale, we consider ROOT as the default Locale
10-
// This has implications on the results, which could change given specific locales where the rules for determining the start of a year are different.
9+
// WEEK_OF_YEAR moved to CSV tests, because H2 builds its Calendar with the local Locale,
10+
// we consider ROOT as the default Locale. This has implications on the results, which could
11+
// change given specific locales where the rules for determining the start of a year are different.
1112

1213
//
1314
// DateTime
@@ -31,10 +32,10 @@ SELECT MONTHNAME(CAST('2018-09-03' AS TIMESTAMP)) month FROM "test_emp" limit 1;
3132
dayNameFromStringDateTime
3233
SELECT DAYNAME(CAST('2018-09-03' AS TIMESTAMP)) day FROM "test_emp" limit 1;
3334

34-
quarterSelect
35+
dateTimeQuarter
3536
SELECT QUARTER(hire_date) q, hire_date FROM test_emp ORDER BY hire_date LIMIT 15;
3637

37-
dayOfWeek
38+
dateTimeDayOfWeek
3839
SELECT DAY_OF_WEEK(birth_date) day, birth_date FROM test_emp ORDER BY DAY_OF_WEEK(birth_date);
3940

4041
//

x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/CompositeKeyExtractor.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@ public Object extract(Bucket bucket) {
9595
if (object == null) {
9696
return object;
9797
} else if (object instanceof Long) {
98-
object = DateUtils.of(((Long) object).longValue(), zoneId);
98+
object = DateUtils.asDateTime(((Long) object).longValue(), zoneId);
9999
} else {
100100
throw new SqlIllegalArgumentException("Invalid date key returned: {}", object);
101101
}
@@ -129,4 +129,4 @@ public boolean equals(Object obj) {
129129
public String toString() {
130130
return "|" + key + "|";
131131
}
132-
}
132+
}

x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractor.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -130,11 +130,11 @@ private Object unwrapMultiValue(Object values) {
130130
}
131131
if (dataType == DataType.DATETIME) {
132132
if (values instanceof String) {
133-
return DateUtils.of(Long.parseLong(values.toString()));
133+
return DateUtils.asDateTime(Long.parseLong(values.toString()));
134134
}
135135
// returned by nested types...
136136
if (values instanceof DateTime) {
137-
return DateUtils.of((DateTime) values);
137+
return DateUtils.asDateTime((DateTime) values);
138138
}
139139
}
140140
if (values instanceof Long || values instanceof Double || values instanceof String || values instanceof Boolean) {

x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/Expressions.java

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
*/
66
package org.elasticsearch.xpack.sql.expression;
77

8-
import org.elasticsearch.common.Strings;
98
import org.elasticsearch.xpack.sql.SqlIllegalArgumentException;
109
import org.elasticsearch.xpack.sql.expression.Expression.TypeResolution;
1110
import org.elasticsearch.xpack.sql.expression.gen.pipeline.Pipe;
@@ -16,11 +15,13 @@
1615
import java.util.Collection;
1716
import java.util.List;
1817
import java.util.Locale;
18+
import java.util.StringJoiner;
1919
import java.util.function.Predicate;
2020

2121
import static java.lang.String.format;
2222
import static java.util.Collections.emptyList;
2323
import static java.util.Collections.emptyMap;
24+
import static org.elasticsearch.xpack.sql.type.DataType.BOOLEAN;
2425

2526
public final class Expressions {
2627

@@ -155,7 +156,7 @@ public static List<Pipe> pipe(List<Expression> expressions) {
155156
}
156157

157158
public static TypeResolution typeMustBeBoolean(Expression e, String operationName, ParamOrdinal paramOrd) {
158-
return typeMustBe(e, dt -> dt == DataType.BOOLEAN, operationName, paramOrd, "boolean");
159+
return typeMustBe(e, dt -> dt == BOOLEAN, operationName, paramOrd, "boolean");
159160
}
160161

161162
public static TypeResolution typeMustBeInteger(Expression e, String operationName, ParamOrdinal paramOrd) {
@@ -171,11 +172,11 @@ public static TypeResolution typeMustBeString(Expression e, String operationName
171172
}
172173

173174
public static TypeResolution typeMustBeDate(Expression e, String operationName, ParamOrdinal paramOrd) {
174-
return typeMustBe(e, dt -> dt == DataType.DATETIME, operationName, paramOrd, "date");
175+
return typeMustBe(e, DataType::isDateBased, operationName, paramOrd, "date", "datetime");
175176
}
176177

177178
public static TypeResolution typeMustBeNumericOrDate(Expression e, String operationName, ParamOrdinal paramOrd) {
178-
return typeMustBe(e, dt -> dt.isNumeric() || dt == DataType.DATETIME, operationName, paramOrd, "numeric", "date");
179+
return typeMustBe(e, dt -> dt.isNumeric() || dt.isDateBased(), operationName, paramOrd, "date", "datetime", "numeric");
179180
}
180181

181182
public static TypeResolution typeMustBe(Expression e,
@@ -188,8 +189,20 @@ public static TypeResolution typeMustBe(Expression e,
188189
new TypeResolution(format(Locale.ROOT, "[%s]%s argument must be [%s], found value [%s] type [%s]",
189190
operationName,
190191
paramOrd == null || paramOrd == ParamOrdinal.DEFAULT ? "" : " " + paramOrd.name().toLowerCase(Locale.ROOT),
191-
Strings.arrayToDelimitedString(acceptedTypes, " or "),
192+
acceptedTypesForErrorMsg(acceptedTypes),
192193
Expressions.name(e),
193194
e.dataType().esType));
194195
}
196+
197+
private static String acceptedTypesForErrorMsg(String... acceptedTypes) {
198+
StringJoiner sj = new StringJoiner(", ");
199+
for (int i = 0; i < acceptedTypes.length - 1; i++) {
200+
sj.add(acceptedTypes[i]);
201+
}
202+
if (acceptedTypes.length > 1) {
203+
return sj.toString() + " or " + acceptedTypes[acceptedTypes.length - 1];
204+
} else {
205+
return acceptedTypes[0];
206+
}
207+
}
195208
}

x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/Max.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,4 +47,4 @@ public String innerName() {
4747
protected TypeResolution resolveType() {
4848
return Expressions.typeMustBeNumericOrDate(field(), sourceText(), ParamOrdinal.DEFAULT);
4949
}
50-
}
50+
}

x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/grouping/Histogram.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ protected TypeResolution resolveType() {
4242
TypeResolution resolution = Expressions.typeMustBeNumericOrDate(field(), "HISTOGRAM", ParamOrdinal.FIRST);
4343
if (resolution == TypeResolution.TYPE_RESOLVED) {
4444
// interval must be Literal interval
45-
if (field().dataType() == DataType.DATETIME) {
45+
if (field().dataType().isDateBased()) {
4646
resolution = Expressions.typeMustBe(interval, DataTypes::isInterval, "(Date) HISTOGRAM", ParamOrdinal.SECOND, "interval");
4747
} else {
4848
resolution = Expressions.typeMustBeNumeric(interval, "(Numeric) HISTOGRAM", ParamOrdinal.SECOND);

x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/BaseDateTimeFunction.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,4 +74,4 @@ public boolean equals(Object obj) {
7474
public int hashCode() {
7575
return Objects.hash(field(), zoneId());
7676
}
77-
}
77+
}

x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/whitelist/InternalSqlScriptUtils.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -357,11 +357,11 @@ private static Object asDateTime(Object dateTime, boolean lenient) {
357357
return ((JodaCompatibleZonedDateTime) dateTime).getZonedDateTime();
358358
}
359359
if (dateTime instanceof ZonedDateTime) {
360-
return (ZonedDateTime) dateTime;
360+
return dateTime;
361361
}
362362
if (false == lenient) {
363363
if (dateTime instanceof Number) {
364-
return DateUtils.of(((Number) dateTime).longValue());
364+
return DateUtils.asDateTime(((Number) dateTime).longValue());
365365
}
366366

367367
throw new SqlIllegalArgumentException("Invalid date encountered [{}]", dateTime);

x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/gen/script/ScriptWeaver.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,7 @@ default ScriptTemplate scriptWithScalar(ScalarFunctionAttribute scalar) {
7979

8080
default ScriptTemplate scriptWithAggregate(AggregateFunctionAttribute aggregate) {
8181
String template = "{}";
82-
if (aggregate.dataType() == DataType.DATETIME) {
82+
if (aggregate.dataType().isDateBased()) {
8383
template = "{sql}.asDateTime({})";
8484
}
8585
return new ScriptTemplate(processScript(template),
@@ -89,7 +89,7 @@ default ScriptTemplate scriptWithAggregate(AggregateFunctionAttribute aggregate)
8989

9090
default ScriptTemplate scriptWithGrouping(GroupingFunctionAttribute grouping) {
9191
String template = "{}";
92-
if (grouping.dataType() == DataType.DATETIME) {
92+
if (grouping.dataType().isDateBased()) {
9393
template = "{sql}.asDateTime({})";
9494
}
9595
return new ScriptTemplate(processScript(template),

0 commit comments

Comments
 (0)