[ESQL] Date nanos type (#110205)

Resolves #109987

Add initial support for the date nanos data type. At this point, almost no functions are supported, including casting. This just covers loading and returning the values. Like millisecond dates, nanosecond dates are internally modeled as long values, so we don't need a new block type to support them.

This has very patchwork function support. Ideally, I don't think I would have added any function support yet, but the five MV functions you see here declare that they accept any non-spatial type, and will error tests if not wired up for new types. There are other functions, like Values, which also claim to support all non-spatial types, but don't currently enforce that in testing, so I didn't add them yet. Finally, there are functions like == which should work for all types, but are implemented as a specific list. I've left those for a follow up ticket as well.

Author: not-napoleon

docs/reference/esql/functions/kibana/definition/mv_count.json

@@ -7,9 +7,11 @@
 
 package org.elasticsearch.xpack.esql.core.plugin;
 
+import org.elasticsearch.common.util.FeatureFlag;
 import org.elasticsearch.plugins.ExtensiblePlugin;
 import org.elasticsearch.plugins.Plugin;
 
 public class EsqlCorePlugin extends Plugin implements ExtensiblePlugin {
+    public static final FeatureFlag DATE_NANOS_FEATURE_FLAG = new FeatureFlag("esql_date_nanos");
 
 }

docs/reference/esql/functions/kibana/definition/mv_first.json

@@ -12,6 +12,7 @@
 import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.mapper.SourceFieldMapper;
 import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper;
+import org.elasticsearch.xpack.esql.core.plugin.EsqlCorePlugin;
 
 import java.io.IOException;
 import java.math.BigInteger;
@@ -100,7 +101,10 @@ public enum DataType {
     KEYWORD(builder().esType("keyword").unknownSize().docValues()),
     TEXT(builder().esType("text").unknownSize()),
     DATETIME(builder().esType("date").typeName("DATETIME").estimatedSize(Long.BYTES).docValues()),
-    // IP addresses, both IPv4 and IPv6, are encoded using 16 bytes.
+    DATE_NANOS(builder().esType("date_nanos").estimatedSize(Long.BYTES).docValues()),
+    /**
+     * IP addresses, both IPv4 and IPv6, are encoded using 16 bytes.
+     */
     IP(builder().esType("ip").estimatedSize(16).docValues()),
     // 8.15.2-SNAPSHOT is 15 bytes, most are shorter, some can be longer
     VERSION(builder().esType("version").estimatedSize(15).docValues()),
@@ -237,6 +241,9 @@ public static DataType fromTypeName(String name) {
 
     public static DataType fromEs(String name) {
         DataType type = ES_TO_TYPE.get(name);
+        if (type == DATE_NANOS && EsqlCorePlugin.DATE_NANOS_FEATURE_FLAG.isEnabled() == false) {
+            type = UNSUPPORTED;
+        }
         return type != null ? type : UNSUPPORTED;
     }
 
@@ -338,18 +345,34 @@ public static boolean areCompatible(DataType left, DataType right) {
      * Supported types that can be contained in a block.
      */
     public static boolean isRepresentable(DataType t) {
-        return t != OBJECT
-            && t != UNSUPPORTED
-            && t != DATE_PERIOD
-            && t != TIME_DURATION
-            && t != BYTE
-            && t != SHORT
-            && t != FLOAT
-            && t != SCALED_FLOAT
-            && t != SOURCE
-            && t != HALF_FLOAT
-            && t != PARTIAL_AGG
-            && t.isCounter() == false;
+        if (EsqlCorePlugin.DATE_NANOS_FEATURE_FLAG.isEnabled()) {
+            return t != OBJECT
+                && t != UNSUPPORTED
+                && t != DATE_PERIOD
+                && t != TIME_DURATION
+                && t != BYTE
+                && t != SHORT
+                && t != FLOAT
+                && t != SCALED_FLOAT
+                && t != SOURCE
+                && t != HALF_FLOAT
+                && t != PARTIAL_AGG
+                && t.isCounter() == false;
+        } else {
+            return t != OBJECT
+                && t != UNSUPPORTED
+                && t != DATE_PERIOD
+                && t != DATE_NANOS
+                && t != TIME_DURATION
+                && t != BYTE
+                && t != SHORT
+                && t != FLOAT
+                && t != SCALED_FLOAT
+                && t != SOURCE
+                && t != HALF_FLOAT
+                && t != PARTIAL_AGG
+                && t.isCounter() == false;
+        }
     }
 
     public static boolean isSpatialPoint(DataType t) {

docs/reference/esql/functions/kibana/definition/mv_last.json

@@ -8,6 +8,7 @@
 package org.elasticsearch.xpack.esql;
 
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.time.DateFormatter;
 import org.elasticsearch.compute.data.Page;
 import org.elasticsearch.logging.Logger;
 import org.elasticsearch.search.DocValueFormat;
@@ -127,6 +128,7 @@ private static void assertMetadata(
 
                 if (blockType == Type.LONG
                     && (expectedType == Type.DATETIME
+                        || expectedType == Type.DATE_NANOS
                         || expectedType == Type.GEO_POINT
                         || expectedType == Type.CARTESIAN_POINT
                         || expectedType == UNSIGNED_LONG)) {
@@ -214,6 +216,12 @@ public static void assertData(
                         // convert the long from CSV back to its STRING form
                         if (expectedType == Type.DATETIME) {
                             expectedValue = rebuildExpected(expectedValue, Long.class, x -> UTC_DATE_TIME_FORMATTER.formatMillis((long) x));
+                        } else if (expectedType == Type.DATE_NANOS) {
+                            expectedValue = rebuildExpected(
+                                expectedValue,
+                                Long.class,
+                                x -> DateFormatter.forPattern("strict_date_optional_time_nanos").formatNanos((long) x)
+                            );
                         } else if (expectedType == Type.GEO_POINT) {
                             expectedValue = rebuildExpected(expectedValue, BytesRef.class, x -> GEO.wkbToWkt((BytesRef) x));
                         } else if (expectedType == Type.CARTESIAN_POINT) {

docs/reference/esql/functions/kibana/definition/mv_max.json

@@ -37,6 +37,7 @@
 import java.io.StringReader;
 import java.math.BigDecimal;
 import java.net.URL;
+import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Comparator;
@@ -54,6 +55,7 @@
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.reader;
 import static org.elasticsearch.xpack.esql.core.SpecReader.shouldSkipLine;
 import static org.elasticsearch.xpack.esql.core.type.DataTypeConverter.safeToUnsignedLong;
+import static org.elasticsearch.xpack.esql.core.util.DateUtils.ISO_DATE_WITH_NANOS;
 import static org.elasticsearch.xpack.esql.core.util.DateUtils.UTC_DATE_TIME_FORMATTER;
 import static org.elasticsearch.xpack.esql.core.util.NumericUtils.asLongUnsigned;
 import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.CARTESIAN;
@@ -444,6 +446,13 @@ public enum Type {
             (l, r) -> l instanceof Long maybeIP ? maybeIP.compareTo((Long) r) : l.toString().compareTo(r.toString()),
             Long.class
         ),
+        DATE_NANOS(x -> {
+            if (x == null) {
+                return null;
+            }
+            Instant parsed = DateFormatters.from(ISO_DATE_WITH_NANOS.parse(x)).toInstant();
+            return parsed.getEpochSecond() * 1_000_000_000 + parsed.getNano();
+        }, (l, r) -> l instanceof Long maybeIP ? maybeIP.compareTo((Long) r) : l.toString().compareTo(r.toString()), Long.class),
         BOOLEAN(Booleans::parseBoolean, Boolean.class),
         GEO_POINT(x -> x == null ? null : GEO.wktToWkb(x), BytesRef.class),
         CARTESIAN_POINT(x -> x == null ? null : CARTESIAN.wktToWkb(x), BytesRef.class),

docs/reference/esql/functions/kibana/definition/mv_min.json

@@ -76,6 +76,7 @@ public class CsvTestsDataLoader {
     private static final TestsDataset AIRPORTS = new TestsDataset("airports", "mapping-airports.json", "airports.csv");
     private static final TestsDataset AIRPORTS_MP = new TestsDataset("airports_mp", "mapping-airports.json", "airports_mp.csv");
     private static final TestsDataset AIRPORTS_WEB = new TestsDataset("airports_web", "mapping-airports_web.json", "airports_web.csv");
+    private static final TestsDataset DATE_NANOS = new TestsDataset("date_nanos", "mapping-date_nanos.json", "date_nanos.csv");
     private static final TestsDataset COUNTRIES_BBOX = new TestsDataset(
         "countries_bbox",
         "mapping-countries_bbox.json",
@@ -122,6 +123,7 @@ public class CsvTestsDataLoader {
         Map.entry(COUNTRIES_BBOX_WEB.indexName, COUNTRIES_BBOX_WEB),
         Map.entry(AIRPORT_CITY_BOUNDARIES.indexName, AIRPORT_CITY_BOUNDARIES),
         Map.entry(CARTESIAN_MULTIPOLYGONS.indexName, CARTESIAN_MULTIPOLYGONS),
+        Map.entry(DATE_NANOS.indexName, DATE_NANOS),
         Map.entry(K8S.indexName, K8S),
         Map.entry(DISTANCES.indexName, DISTANCES),
         Map.entry(ADDRESSES.indexName, ADDRESSES)

docs/reference/esql/functions/types/mv_count.asciidoc

@@ -630,6 +630,7 @@ public static Literal randomLiteral(DataType type) {
             case UNSIGNED_LONG, LONG, COUNTER_LONG -> randomLong();
             case DATE_PERIOD -> Period.of(randomIntBetween(-1000, 1000), randomIntBetween(-13, 13), randomIntBetween(-32, 32));
             case DATETIME -> randomMillisUpToYear9999();
+            case DATE_NANOS -> randomLong();
             case DOUBLE, SCALED_FLOAT, COUNTER_DOUBLE -> randomDouble();
             case FLOAT -> randomFloat();
             case HALF_FLOAT -> HalfFloatPoint.sortableShortToHalfFloat(HalfFloatPoint.halfFloatToSortableShort(randomFloat()));

docs/reference/esql/functions/types/mv_first.asciidoc

@@ -0,0 +1,9 @@
+millis:date,nanos:date_nanos
+2023-10-23T13:55:01.543Z,2023-10-23T13:55:01.543123456Z
+2023-10-23T13:53:55.832Z,2023-10-23T13:53:55.832987654Z
+2023-10-23T13:52:55.015Z,2023-10-23T13:52:55.015787878Z
+2023-10-23T13:51:54.732Z,2023-10-23T13:51:54.732102837Z
+2023-10-23T13:33:34.937Z,2023-10-23T13:33:34.937193000Z
+2023-10-23T12:27:28.948Z,2023-10-23T12:27:28.948000000Z
+2023-10-23T12:15:03.360Z,2023-10-23T12:15:03.360103847Z
+1999-10-23T12:15:03.360Z,[2023-03-23T12:15:03.360103847Z, 2023-02-23T13:33:34.937193000Z, 2023-01-23T13:55:01.543123456Z]

docs/reference/esql/functions/types/mv_last.asciidoc

@@ -0,0 +1,54 @@
+simple
+required_capability: date_nanos_type
+
+FROM date_nanos | SORT millis DESC | KEEP millis, nanos | LIMIT 1;
+
+millis:date              | nanos:date_nanos
+2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z
+;
+
+mv_max on date nanos
+required_capability: date_nanos_type
+
+FROM date_nanos | SORT millis ASC | EVAL nanos = MV_MAX(nanos) | KEEP nanos | LIMIT 1;
+
+
+nanos:date_nanos
+2023-03-23T12:15:03.360103847Z
+;
+
+mv_min on date nanos
+required_capability: date_nanos_type
+
+FROM date_nanos | SORT millis ASC | EVAL nanos = MV_MIN(nanos) | KEEP nanos | LIMIT 1;
+
+nanos:date_nanos
+2023-01-23T13:55:01.543123456Z
+;
+
+mv_count on date nanos
+required_capability: date_nanos_type
+
+FROM date_nanos | SORT millis ASC | EVAL ct = MV_COUNT(nanos) | KEEP ct | LIMIT 1;
+
+ct:integer
+3
+;
+
+mv_first on date nanos
+required_capability: date_nanos_type
+
+FROM date_nanos | SORT millis ASC | EVAL nanos = MV_FIRST(nanos) | KEEP nanos | LIMIT 1;
+
+nanos:date_nanos
+2023-01-23T13:55:01.543123456Z
+;
+
+mv_last on date nanos
+required_capability: date_nanos_type
+
+FROM date_nanos | SORT millis asc | EVAL nanos = MV_LAST(nanos) | KEEP nanos | LIMIT 1;
+
+nanos:date_nanos
+2023-03-23T12:15:03.360103847Z
+;

docs/reference/esql/functions/types/mv_max.asciidoc

@@ -0,0 +1,10 @@
+{
+    "properties": {
+        "millis": {
+            "type": "date"
+        },
+        "nanos": {
+            "type": "date_nanos"
+        }
+    }
+}