SQL: Implement FORMAT function (#55454) (#62701)

Implement FORMAT according to the SQL Server spec: https://docs.microsoft.com/en-us/sql/t-sql/functions/format-transact-sql?view=sql-server-ver15#ExampleD by translating to the java.time patterns used in DATETIME_FORMAT.

Closes: #54965

Co-authored-by: Marios Trivyzas <[email protected]>
Co-authored-by: Bogdan Pintea <[email protected]>
Co-authored-by: Andrei Stefan <[email protected]>
(cherry picked from commit da511f4)

Author: 3 people

docs/reference/sql/functions/date-time.asciidoc

@@ -579,9 +579,9 @@ include-tagged::{sql-specs}/docs/docs.csv-spec[timeParse2]
 
 [NOTE]
 ====
-If timezone is not specified in the time string expression and the parsing pattern, 
+If timezone is not specified in the time string expression and the parsing pattern,
 the resulting `time` will have the offset of the time zone specified by the user through the
- <<sql-rest-fields-timezone,`time_zone`>>/<<jdbc-cfg-timezone,`timezone`>> REST/driver 
+ <<sql-rest-fields-timezone,`time_zone`>>/<<jdbc-cfg-timezone,`timezone`>> REST/driver
 parameters at the Unix epoch date (`1970-01-01`) with no conversion applied.
 
 [source, sql]
@@ -765,6 +765,59 @@ include-tagged::{sql-specs}/docs/docs.csv-spec[truncateIntervalHour]
 include-tagged::{sql-specs}/docs/docs.csv-spec[truncateIntervalDay]
 --------------------------------------------------
 
+[[sql-functions-datetime-format]]
+==== `FORMAT`
+
+.Synopsis:
+[source, sql]
+--------------------------------------------------
+FORMAT(
+    date_exp/datetime_exp/time_exp, <1>
+    string_exp) <2>
+--------------------------------------------------
+
+*Input*:
+
+<1> date/datetime/time expression
+<2> format pattern
+
+*Output*: string
+
+*Description*: Returns the date/datetime/time as a string using the
+https://docs.microsoft.com/en-us/sql/t-sql/functions/format-transact-sql#arguments[format] specified in the 2nd argument. The formatting
+pattern used is the one from
+https://docs.microsoft.com/en-us/dotnet/standard/base-types/custom-date-and-time-format-strings[Microsoft SQL Server Format Specification].
+If any of the two arguments is `null` or the pattern is an empty string `null` is returned.
+
+[NOTE]
+If the 1st argument is of type `time`, then pattern specified by the 2nd argument cannot contain date related units
+(e.g. 'dd', 'MM', 'YYYY', etc.). If it contains such units an error is returned.
+
+*Special Cases*
+
+- Format specifier `F` will be working similar to format specifier `f`.
+It will return the fractional part of seconds, and the number of digits will be same as of the number of `Fs` provided as input (up to 9 digits).
+Result will contain `0` appended in the end to match with number of `F` provided.
+e.g.: for a time part `10:20:30.1234` and pattern `HH:mm:ss.FFFFFF`, the output string of the function would be: `10:20:30.123400`.
+- Format Specifier `y` will return year-of-era instead of one/two low-order digits.
+eg.: For year `2009`, `y` will be returning `2009` instead of `9`. For year `43`, `y` format specifier will return `43`.
+- Special characters like `"` , `\` and `%` will be returned as it is without any change. eg.: formatting date `17-sep-2020` with `%M` will return `%9`
+
+[source, sql]
+--------------------------------------------------
+include-tagged::{sql-specs}/docs/docs.csv-spec[formatDate]
+--------------------------------------------------
+
+[source, sql]
+--------------------------------------------------
+include-tagged::{sql-specs}/docs/docs.csv-spec[formatDateTime]
+--------------------------------------------------
+
+[source, sql]
+--------------------------------------------------
+include-tagged::{sql-specs}/docs/docs.csv-spec[formatTime]
+--------------------------------------------------
+
 [[sql-functions-datetime-day]]
 ==== `DAY_OF_MONTH/DOM/DAY`
 

docs/reference/sql/functions/index.asciidoc

@@ -58,6 +58,7 @@
 ** <<sql-functions-datetime-dateparse>>
 ** <<sql-functions-datetime-datetimeformat>>
 ** <<sql-functions-datetime-datetimeparse>>
+** <<sql-functions-datetime-format>>
 ** <<sql-functions-datetime-timeparse>>
 ** <<sql-functions-datetime-part>>
 ** <<sql-functions-datetime-trunc>>

x-pack/plugin/sql/qa/server/src/main/resources/command.csv-spec

@@ -66,6 +66,7 @@ DAY_OF_YEAR      |SCALAR
 DOM              |SCALAR
 DOW              |SCALAR         
 DOY              |SCALAR
+FORMAT           |SCALAR
 HOUR             |SCALAR
 HOUR_OF_DAY      |SCALAR         
 IDOW             |SCALAR

x-pack/plugin/sql/qa/server/src/main/resources/datetime.csv-spec

@@ -1017,6 +1017,120 @@ F       | 1997-05-19 00:00:00.000Z
 M       | 1996-11-05 00:00:00.000Z
 ;
 
+selectFormat
+schema::format_date:s|format_datetime:s|format_time:s
+SELECT FORMAT('2020-04-05T11:22:33.123Z'::date, 'dd/MM/YYYY HH:mm:ss.fff') AS format_date,
+FORMAT('2020-04-05T11:22:33.123Z'::datetime, 'dd/MM/YYYY HH:mm:ss.ff') AS format_datetime,
+FORMAT('11:22:33.123456789Z'::time, 'HH:mm:ss.ff') AS format_time;
+
+       format_date      |    format_datetime     |   format_time
+------------------------+------------------------+----------------
+05/04/2020 00:00:00.000 | 05/04/2020 11:22:33.12 | 11:22:33.12
+;
+
+selectFormatWithLength
+schema::format_datetime:s|length:i
+SELECT FORMAT('2020-04-05T11:22:33.123Z'::datetime, 'dd/MM/YYYY HH:mm:ss.ff') AS format_datetime,
+LENGTH(FORMAT('2020-04-05T11:22:33.123Z'::datetime, 'dd/MM/YYYY HH:mm:ss.ff')) AS length;
+
+    format_datetime     | length
+------------------------+----------------
+ 05/04/2020 11:22:33.12 | 22
+;
+
+selectFormatWithField
+schema::birth_date:ts|format_birth_date1:s|format_birth_date2:s|emp_no:i
+SELECT birth_date, FORMAT(birth_date, 'MM/dd/YYYY') AS format_birth_date1, FORMAT(birth_date, concat(gender, 'M/dd')) AS format_birth_date2, emp_no
+FROM test_emp WHERE gender = 'M' AND emp_no BETWEEN 10037 AND 10052 ORDER BY emp_no;
+
+       birth_date        | format_birth_date1 | format_birth_date2 | emp_no
+-------------------------+--------------------+--------------------+----------
+1963-07-22 00:00:00.000Z | 07/22/1963         | 07/22              | 10037
+1960-07-20 00:00:00.000Z | 07/20/1960         | 07/20              | 10038
+1959-10-01 00:00:00.000Z | 10/01/1959         | 10/01              | 10039
+null                     | null               | null               | 10043
+null                     | null               | null               | 10045
+null                     | null               | null               | 10046
+null                     | null               | null               | 10047
+null                     | null               | null               | 10048
+1958-05-21 00:00:00.000Z | 05/21/1958         | 05/21              | 10050
+1953-07-28 00:00:00.000Z | 07/28/1953         | 07/28              | 10051
+1961-02-26 00:00:00.000Z | 02/26/1961         | 02/26              | 10052
+;
+
+formatWhere
+schema::birth_date:ts|format_birth_date:s|emp_no:i
+SELECT birth_date, FORMAT(birth_date, 'MM') AS format_birth_date, emp_no FROM test_emp
+WHERE FORMAT(birth_date, 'MM')::integer > 10 ORDER BY emp_no LIMIT 10;
+
+       birth_date        | format_birth_date | emp_no
+-------------------------+-------------------+----------
+1959-12-03 00:00:00.000Z | 12                | 10003
+1953-11-07 00:00:00.000Z | 11                | 10011
+1952-12-24 00:00:00.000Z | 12                | 10020
+1963-11-26 00:00:00.000Z | 11                | 10028
+1956-12-13 00:00:00.000Z | 12                | 10029
+1956-11-14 00:00:00.000Z | 11                | 10033
+1962-12-29 00:00:00.000Z | 12                | 10034
+1961-11-02 00:00:00.000Z | 11                | 10062
+1952-11-13 00:00:00.000Z | 11                | 10066
+1962-11-26 00:00:00.000Z | 11                | 10068
+;
+
+formatOrderBy
+schema::birth_date:ts|format_birth_date:s
+SELECT birth_date, FORMAT(birth_date, 'MM/dd/YYYY') AS format_birth_date FROM test_emp ORDER BY 2 DESC NULLS LAST LIMIT 10;
+
+       birth_date        | format_birth_date
+-------------------------+---------------
+1962-12-29 00:00:00.000Z | 12/29/1962
+1959-12-25 00:00:00.000Z | 12/25/1959
+1952-12-24 00:00:00.000Z | 12/24/1952
+1960-12-17 00:00:00.000Z | 12/17/1960
+1956-12-13 00:00:00.000Z | 12/13/1956
+1959-12-03 00:00:00.000Z | 12/03/1959
+1957-12-03 00:00:00.000Z | 12/03/1957
+1963-11-26 00:00:00.000Z | 11/26/1963
+1962-11-26 00:00:00.000Z | 11/26/1962
+1962-11-19 00:00:00.000Z | 11/19/1962
+;
+
+formatGroupBy
+schema::count:l|format_birth_date:s
+SELECT count(*) AS count, FORMAT(birth_date, 'MM') AS format_birth_date FROM test_emp GROUP BY format_birth_date ORDER BY 1 DESC, 2 DESC;
+
+ count | format_birth_date
+-------+---------------
+10     | 09
+10     | 05
+10     | null
+9      | 10
+9      | 07
+8      | 11
+8      | 04
+8      | 02
+7      | 12
+7      | 06
+6      | 08
+6      | 01
+2      | 03
+;
+
+formatHaving
+schema::max:ts|format_birth_date:s
+SELECT MAX(birth_date) AS max, FORMAT(birth_date, 'MM') AS format_birth_date FROM test_emp GROUP BY format_birth_date
+HAVING FORMAT(MAX(birth_date), 'dd')::integer > 20  ORDER BY 1 DESC;
+
+          max            | format_birth_date
+-------------------------+---------------
+1963-11-26 00:00:00.000Z | 11
+1963-07-22 00:00:00.000Z | 07
+1963-03-21 00:00:00.000Z | 03
+1962-12-29 00:00:00.000Z | 12
+1961-05-30 00:00:00.000Z | 05
+1961-02-26 00:00:00.000Z | 02
+;
+
 //
 // Aggregate
 //

x-pack/plugin/sql/qa/server/src/main/resources/docs/docs.csv-spec

@@ -262,6 +262,7 @@ DAY_OF_YEAR      |SCALAR
 DOM              |SCALAR
 DOW              |SCALAR         
 DOY              |SCALAR
+FORMAT           |SCALAR
 HOUR             |SCALAR
 HOUR_OF_DAY      |SCALAR         
 IDOW             |SCALAR
@@ -3088,6 +3089,36 @@ SELECT DATE_TRUNC('days', INTERVAL '19 15:24:19' DAY TO SECONDS) AS day;
 // end::truncateIntervalDay
 ;
 
+formatDate
+// tag::formatDate
+SELECT FORMAT(CAST('2020-04-05' AS DATE), 'dd/MM/YYYY') AS "date";
+
+      date
+------------------
+05/04/2020
+// end::formatDate
+;
+
+formatDateTime
+// tag::formatDateTime
+SELECT FORMAT(CAST('2020-04-05T11:22:33.987654' AS DATETIME), 'dd/MM/YYYY HH:mm:ss.ff') AS "datetime";
+
+      datetime
+------------------
+05/04/2020 11:22:33.98
+// end::formatDateTime
+;
+
+formatTime
+// tag::formatTime
+SELECT FORMAT(CAST('11:22:33.987' AS TIME), 'HH mm ss.f') AS "time";
+
+      time
+------------------
+11 22 33.9
+// end::formatTime
+;
+
 constantDayOfWeek
 // tag::dayOfWeek
 SELECT DAY_OF_WEEK(CAST('2018-02-19T10:23:27Z' AS TIMESTAMP)) AS day;

x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/SqlFunctionRegistry.java

@@ -43,6 +43,7 @@
 import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DayOfMonth;
 import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DayOfWeek;
 import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DayOfYear;
+import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.Format;
 import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.HourOfDay;
 import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.IsoDayOfWeek;
 import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.IsoWeekOfYear;
@@ -182,6 +183,7 @@ private static FunctionDefinition[][] functions() {
                 def(DateTimeFormat.class, DateTimeFormat::new, "DATETIME_FORMAT"),
                 def(DateTimeParse.class, DateTimeParse::new, "DATETIME_PARSE"),
                 def(DateTrunc.class, DateTrunc::new, "DATETRUNC", "DATE_TRUNC"),
+                def(Format.class, Format::new, "FORMAT"),
                 def(HourOfDay.class, HourOfDay::new, "HOUR_OF_DAY", "HOUR"),
                 def(IsoDayOfWeek.class, IsoDayOfWeek::new, "ISO_DAY_OF_WEEK", "ISODAYOFWEEK", "ISODOW", "IDOW"),
                 def(IsoWeekOfYear.class, IsoWeekOfYear::new, "ISO_WEEK_OF_YEAR", "ISOWEEKOFYEAR", "ISOWEEK", "IWOY", "IW"),

x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/BaseDateTimeFormatFunction.java

@@ -0,0 +1,65 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.sql.expression.function.scalar.datetime;
+
+import org.elasticsearch.xpack.ql.expression.Expression;
+import org.elasticsearch.xpack.ql.expression.Expressions;
+import org.elasticsearch.xpack.ql.expression.gen.pipeline.Pipe;
+import org.elasticsearch.xpack.ql.tree.NodeInfo;
+import org.elasticsearch.xpack.ql.tree.NodeInfo.NodeCtor3;
+import org.elasticsearch.xpack.ql.tree.Source;
+import org.elasticsearch.xpack.ql.type.DataType;
+import org.elasticsearch.xpack.ql.type.DataTypes;
+
+import java.time.ZoneId;
+
+import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isString;
+import static org.elasticsearch.xpack.sql.expression.SqlTypeResolutions.isDateOrTime;
+import static org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateTimeFormatProcessor.Formatter;
+
+public abstract class BaseDateTimeFormatFunction extends BinaryDateTimeFunction {
+    public BaseDateTimeFormatFunction(Source source, Expression timestamp, Expression pattern, ZoneId zoneId) {
+        super(source, timestamp, pattern, zoneId);
+    }
+
+    @Override
+    public DataType dataType() {
+        return DataTypes.KEYWORD;
+    }
+
+    @Override
+    protected TypeResolution resolveType() {
+        TypeResolution resolution = isDateOrTime(left(), sourceText(), Expressions.ParamOrdinal.FIRST);
+        if (resolution.unresolved()) {
+            return resolution;
+        }
+        resolution = isString(right(), sourceText(), Expressions.ParamOrdinal.SECOND);
+        if (resolution.unresolved()) {
+            return resolution;
+        }
+        return TypeResolution.TYPE_RESOLVED;
+    }
+
+    @Override
+    protected NodeInfo<? extends Expression> info() {
+        return NodeInfo.create(this, ctor(), left(), right(), zoneId());
+    }
+
+    @Override
+    public Object fold() {
+        return formatter().format(left().fold(), right().fold(), zoneId());
+    }
+
+    @Override
+    protected Pipe createPipe(Pipe timestamp, Pipe pattern, ZoneId zoneId) {
+        return new DateTimeFormatPipe(source(), this, timestamp, pattern, zoneId, formatter());
+    }
+
+    protected abstract Formatter formatter();
+
+    protected abstract NodeCtor3<Expression, Expression, ZoneId, BaseDateTimeFormatFunction> ctor();
+}