Skip to content

Commit 6b60085

Browse files
matrivmatriv
committed
SQL: Make parsing of date more lenient (#52137)
Make the parsing of date more lenient - as an escaped literal: `{d '2020-02-10[[T| ]10:20[:30][.123456789][tz]]'}` - cast a string to a date: `CAST(2020-02-10[[T| ]10:20[:30][.123456789][tz]]' AS DATE)` Closes: #49379 (cherry picked from commit 5863b27)
1 parent 47255c4 commit 6b60085

File tree

4 files changed

+87
-19
lines changed

4 files changed

+87
-19
lines changed

‎x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -130,9 +130,9 @@
130130
import static java.util.Collections.emptyList;
131131
import static java.util.Collections.singletonList;
132132
import static org.elasticsearch.xpack.ql.type.DataTypeConverter.converterFor;
133-
import static org.elasticsearch.xpack.sql.util.DateUtils.asDateOnly;
134133
import static org.elasticsearch.xpack.sql.util.DateUtils.asTimeOnly;
135-
import static org.elasticsearch.xpack.sql.util.DateUtils.ofEscapedLiteral;
134+
import static org.elasticsearch.xpack.sql.util.DateUtils.dateOfEscapedLiteral;
135+
import static org.elasticsearch.xpack.sql.util.DateUtils.dateTimeOfEscapedLiteral;
136136

137137
abstract class ExpressionBuilder extends IdentifierBuilder {
138138

@@ -761,9 +761,9 @@ private SqlTypedParamValue param(TerminalNode node) {
761761
public Literal visitDateEscapedLiteral(DateEscapedLiteralContext ctx) {
762762
String string = string(ctx.string());
763763
Source source = source(ctx);
764-
// parse yyyy-MM-dd
764+
// parse yyyy-MM-dd (time optional but is set to 00:00:00.000 because of the conversion to DATE
765765
try {
766-
return new Literal(source, asDateOnly(string), SqlDataTypes.DATE);
766+
return new Literal(source, dateOfEscapedLiteral(string), SqlDataTypes.DATE);
767767
} catch(DateTimeParseException ex) {
768768
throw new ParsingException(source, "Invalid date received; {}", ex.getMessage());
769769
}
@@ -789,7 +789,7 @@ public Literal visitTimestampEscapedLiteral(TimestampEscapedLiteralContext ctx)
789789
Source source = source(ctx);
790790
// parse yyyy-mm-dd hh:mm:ss(.f...)
791791
try {
792-
return new Literal(source, ofEscapedLiteral(string), DataTypes.DATETIME);
792+
return new Literal(source, dateTimeOfEscapedLiteral(string), DataTypes.DATETIME);
793793
} catch (DateTimeParseException ex) {
794794
throw new ParsingException(source, "Invalid timestamp received; {}", ex.getMessage());
795795
}

‎x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/util/DateUtils.java

Lines changed: 50 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -32,16 +32,40 @@ public final class DateUtils {
3232
public static final LocalDate EPOCH = LocalDate.of(1970, 1, 1);
3333
public static final long DAY_IN_MILLIS = 60 * 60 * 24 * 1000L;
3434

35-
private static final DateTimeFormatter DATE_TIME_ESCAPED_LITERAL_FORMATTER_WHITESPACE = new DateTimeFormatterBuilder()
35+
private static final DateTimeFormatter DATE_TIME_FORMATTER_WHITESPACE = new DateTimeFormatterBuilder()
3636
.append(ISO_LOCAL_DATE)
3737
.appendLiteral(' ')
3838
.append(ISO_LOCAL_TIME)
3939
.toFormatter().withZone(UTC);
40-
private static final DateTimeFormatter DATE_TIME_ESCAPED_LITERAL_FORMATTER_T_LITERAL = new DateTimeFormatterBuilder()
40+
private static final DateTimeFormatter DATE_TIME_FORMATTER_T_LITERAL = new DateTimeFormatterBuilder()
4141
.append(ISO_LOCAL_DATE)
4242
.appendLiteral('T')
4343
.append(ISO_LOCAL_TIME)
4444
.toFormatter().withZone(UTC);
45+
private static final DateTimeFormatter DATE_OPTIONAL_TIME_FORMATTER_WHITESPACE = new DateTimeFormatterBuilder()
46+
.append(ISO_LOCAL_DATE)
47+
.optionalStart()
48+
.appendLiteral(' ')
49+
.append(ISO_LOCAL_TIME)
50+
.toFormatter().withZone(UTC);
51+
private static final DateTimeFormatter DATE_OPTIONAL_TIME_FORMATTER_T_LITERAL = new DateTimeFormatterBuilder()
52+
.append(ISO_LOCAL_DATE)
53+
.optionalStart()
54+
.appendLiteral('T')
55+
.append(ISO_LOCAL_TIME)
56+
.toFormatter().withZone(UTC);
57+
private static final DateTimeFormatter ISO_LOCAL_DATE_OPTIONAL_TIME_FORMATTER_WHITESPACE = new DateTimeFormatterBuilder()
58+
.append(DATE_OPTIONAL_TIME_FORMATTER_WHITESPACE)
59+
.optionalStart()
60+
.appendZoneOrOffsetId()
61+
.optionalEnd()
62+
.toFormatter().withZone(UTC);
63+
private static final DateTimeFormatter ISO_LOCAL_DATE_OPTIONAL_TIME_FORMATTER_T_LITERAL = new DateTimeFormatterBuilder()
64+
.append(DATE_OPTIONAL_TIME_FORMATTER_T_LITERAL)
65+
.optionalStart()
66+
.appendZoneOrOffsetId()
67+
.optionalEnd()
68+
.toFormatter().withZone(UTC);
4569

4670
private static final DateFormatter UTC_DATE_TIME_FORMATTER = DateFormatter.forPattern("date_optional_time").withZone(UTC);
4771
private static final int DEFAULT_PRECISION_FOR_CURRENT_FUNCTIONS = 3;
@@ -91,7 +115,17 @@ public static ZonedDateTime asDateTime(long millis, ZoneId id) {
91115
* Parses the given string into a Date (SQL DATE type) using UTC as a default timezone.
92116
*/
93117
public static ZonedDateTime asDateOnly(String dateFormat) {
94-
return LocalDate.parse(dateFormat, ISO_LOCAL_DATE).atStartOfDay(UTC);
118+
int separatorIdx = dateFormat.indexOf('-');
119+
if (separatorIdx == 0) { // negative year
120+
separatorIdx = dateFormat.indexOf('-', 1);
121+
}
122+
separatorIdx = dateFormat.indexOf('-', separatorIdx + 1) + 3;
123+
// Avoid index out of bounds - it will lead to DateTimeParseException anyways
124+
if (separatorIdx >= dateFormat.length() || dateFormat.charAt(separatorIdx) == 'T') {
125+
return LocalDate.parse(dateFormat, ISO_LOCAL_DATE_OPTIONAL_TIME_FORMATTER_T_LITERAL).atStartOfDay(UTC);
126+
} else {
127+
return LocalDate.parse(dateFormat, ISO_LOCAL_DATE_OPTIONAL_TIME_FORMATTER_WHITESPACE).atStartOfDay(UTC);
128+
}
95129
}
96130

97131
public static ZonedDateTime asDateOnly(ZonedDateTime zdt) {
@@ -109,13 +143,23 @@ public static ZonedDateTime asDateTime(String dateFormat) {
109143
return DateFormatters.from(UTC_DATE_TIME_FORMATTER.parse(dateFormat)).withZoneSameInstant(UTC);
110144
}
111145

112-
public static ZonedDateTime ofEscapedLiteral(String dateFormat) {
146+
public static ZonedDateTime dateOfEscapedLiteral(String dateFormat) {
147+
int separatorIdx = dateFormat.lastIndexOf('-') + 3;
148+
// Avoid index out of bounds - it will lead to DateTimeParseException anyways
149+
if (separatorIdx >= dateFormat.length() || dateFormat.charAt(separatorIdx) == 'T') {
150+
return LocalDate.parse(dateFormat, DATE_OPTIONAL_TIME_FORMATTER_T_LITERAL).atStartOfDay(UTC);
151+
} else {
152+
return LocalDate.parse(dateFormat, DATE_TIME_FORMATTER_WHITESPACE).atStartOfDay(UTC);
153+
}
154+
}
155+
156+
public static ZonedDateTime dateTimeOfEscapedLiteral(String dateFormat) {
113157
int separatorIdx = dateFormat.lastIndexOf('-') + 3;
114158
// Avoid index out of bounds - it will lead to DateTimeParseException anyways
115159
if (separatorIdx >= dateFormat.length() || dateFormat.charAt(separatorIdx) == 'T') {
116-
return ZonedDateTime.parse(dateFormat, DATE_TIME_ESCAPED_LITERAL_FORMATTER_T_LITERAL.withZone(UTC));
160+
return ZonedDateTime.parse(dateFormat, DATE_TIME_FORMATTER_T_LITERAL.withZone(UTC));
117161
} else {
118-
return ZonedDateTime.parse(dateFormat, DATE_TIME_ESCAPED_LITERAL_FORMATTER_WHITESPACE.withZone(UTC));
162+
return ZonedDateTime.parse(dateFormat, DATE_TIME_FORMATTER_WHITESPACE.withZone(UTC));
119163
}
120164
}
121165

‎x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/EscapedFunctionsTests.java

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,13 @@ private String buildDate() {
8181
return sb.toString();
8282
}
8383

84+
private String buildTime() {
85+
if (randomBoolean()) {
86+
return (randomBoolean() ? "T" : " ") + "11:22" + buildSecsAndFractional();
87+
}
88+
return "";
89+
}
90+
8491
private String buildSecsAndFractional() {
8592
if (randomBoolean()) {
8693
return ":55" + randomFrom("", ".1", ".12", ".123", ".1234", ".12345", ".123456",
@@ -212,7 +219,7 @@ public void testFunctionWithFunctionWithArgAndParams() {
212219
}
213220

214221
public void testDateLiteral() {
215-
Literal l = dateLiteral(buildDate());
222+
Literal l = dateLiteral(buildDate() + buildTime());
216223
assertThat(l.dataType(), is(DATE));
217224
}
218225

‎x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/SqlDataTypeConverterTests.java

Lines changed: 24 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -172,19 +172,37 @@ public void testConversionToDate() {
172172
Converter conversion = converterFor(KEYWORD, to);
173173
assertNull(conversion.convert(null));
174174

175-
assertEquals(date(0L), conversion.convert("1970-01-01"));
176-
assertEquals(date(1483228800000L), conversion.convert("2017-01-01"));
177-
assertEquals(date(-1672531200000L), conversion.convert("1917-01-01"));
178-
assertEquals(date(18000000L), conversion.convert("1970-01-01"));
175+
assertEquals(date(1581292800000L), conversion.convert("2020-02-10T10:20"));
176+
assertEquals(date(-125908819200000L), conversion.convert("-2020-02-10T10:20:30.123"));
177+
assertEquals(date(1581292800000L), conversion.convert("2020-02-10T10:20:30.123456789"));
179178

180-
// double check back and forth conversion
179+
assertEquals(date(1581292800000L), conversion.convert("2020-02-10 10:20"));
180+
assertEquals(date(-125908819200000L), conversion.convert("-2020-02-10 10:20:30.123"));
181+
assertEquals(date(1581292800000L), conversion.convert("2020-02-10 10:20:30.123456789"));
182+
183+
assertEquals(date(1581292800000L), conversion.convert("2020-02-10T10:20+05:00"));
184+
assertEquals(date(-125908819200000L), conversion.convert("-2020-02-10T10:20:30.123-06:00"));
185+
assertEquals(date(1581292800000L), conversion.convert("2020-02-10T10:20:30.123456789+03:00"));
186+
187+
assertEquals(date(1581292800000L), conversion.convert("2020-02-10 10:20+05:00"));
188+
assertEquals(date(-125908819200000L), conversion.convert("-2020-02-10 10:20:30.123-06:00"));
189+
assertEquals(date(1581292800000L), conversion.convert("2020-02-10 10:20:30.123456789+03:00"));
181190

191+
// double check back and forth conversion
182192
ZonedDateTime zdt = org.elasticsearch.common.time.DateUtils.nowWithMillisResolution();
183193
Converter forward = converterFor(DATE, KEYWORD);
184194
Converter back = converterFor(KEYWORD, DATE);
185195
assertEquals(asDateOnly(zdt), back.convert(forward.convert(zdt)));
186196
Exception e = expectThrows(QlIllegalArgumentException.class, () -> conversion.convert("0xff"));
187197
assertEquals("cannot cast [0xff] to [date]: Text '0xff' could not be parsed at index 0", e.getMessage());
198+
e = expectThrows(QlIllegalArgumentException.class, () -> conversion.convert("2020-02-"));
199+
assertEquals("cannot cast [2020-02-] to [date]: Text '2020-02-' could not be parsed at index 8", e.getMessage());
200+
e = expectThrows(QlIllegalArgumentException.class, () -> conversion.convert("2020-"));
201+
assertEquals("cannot cast [2020-] to [date]: Text '2020-' could not be parsed at index 5", e.getMessage());
202+
e = expectThrows(QlIllegalArgumentException.class, () -> conversion.convert("-2020-02-"));
203+
assertEquals("cannot cast [-2020-02-] to [date]: Text '-2020-02-' could not be parsed at index 9", e.getMessage());
204+
e = expectThrows(QlIllegalArgumentException.class, () -> conversion.convert("-2020-"));
205+
assertEquals("cannot cast [-2020-] to [date]: Text '-2020-' could not be parsed at index 6", e.getMessage());
188206
}
189207
}
190208

@@ -285,7 +303,6 @@ public void testConversionToDateTime() {
285303
assertEquals(dateTime(18000000L), conversion.convert("1970-01-01T00:00:00-05:00"));
286304

287305
// double check back and forth conversion
288-
289306
ZonedDateTime dt = org.elasticsearch.common.time.DateUtils.nowWithMillisResolution();
290307
Converter forward = converterFor(DATETIME, KEYWORD);
291308
Converter back = converterFor(KEYWORD, DATETIME);
@@ -692,4 +709,4 @@ static ZonedDateTime date(long millisSinceEpoch) {
692709
static OffsetTime time(long millisSinceEpoch) {
693710
return DateUtils.asTimeOnly(millisSinceEpoch);
694711
}
695-
}
712+
}

0 commit comments

Comments
 (0)