Add nanosecond field mapper

docs/reference/mapping/types.asciidoc

@@ -7,12 +7,13 @@ document:
 [float]
 === Core datatypes
 
-string::        <<text,`text`>> and <<keyword,`keyword`>>
-<<number>>::    `long`, `integer`, `short`, `byte`, `double`, `float`, `half_float`, `scaled_float`
-<<date>>::      `date`
-<<boolean>>::   `boolean`
-<<binary>>::    `binary`
-<<range>>::     `integer_range`, `float_range`, `long_range`, `double_range`, `date_range`
+string::         <<text,`text`>> and <<keyword,`keyword`>>
+<<number>>::     `long`, `integer`, `short`, `byte`, `double`, `float`, `half_float`, `scaled_float`
+<<date>>::       `date`
+<<date_nanos>>:: `date_nanos`
+<<boolean>>::    `boolean`
+<<binary>>::     `binary`
+<<range>>::      `integer_range`, `float_range`, `long_range`, `double_range`, `date_range`
 
 [float]
 === Complex datatypes
@@ -78,6 +79,8 @@ include::types/boolean.asciidoc[]
 
 include::types/date.asciidoc[]
 
+include::types/date_nanos.asciidoc[]
+
 include::types/geo-point.asciidoc[]
 
 include::types/geo-shape.asciidoc[]
@@ -106,4 +109,4 @@ include::types/rank-features.asciidoc[]
 
 include::types/dense-vector.asciidoc[]
 
-include::types/sparse-vector.asciidoc[]
+include::types/sparse-vector.asciidoc[]

docs/reference/mapping/types/date_nanos.asciidoc

@@ -0,0 +1,99 @@
+[[date_nanos]]
+=== date_nanos datatype
+
+This datatype is an addition to the `date` datatype. However there is an
+important distinction between the two. The existing `date` datatype stores
+dates in millisecond resolution. The `date_nanos` data type stores dates
+in nanosecond resolution, which limits its range of dates from roughly
+1970 to 2262, as dates are still stored as a long representing nanoseconds
+since the epoch.
+
+Queries on nanoseconds are internally converted to range queries on this long
+representation, and the result of aggregations and stored fields is converted
+back to a string depending on the date format that is associated with the field.
+
+Date formats can be customised, but if no `format` is specified then it uses
+the default:
+
+    "strict_date_optional_time||epoch_millis"
+
+This means that it will accept dates with optional timestamps, which conform
+to the formats supported by
+<<strict-date-time,`strict_date_optional_time`>> including up to nine second
+fractionals or milliseconds-since-the-epoch (thus losing precision on the
+nano second part).
+
+For instance:
+
+[source,js]
+--------------------------------------------------
+PUT my_index?include_type_name=true
+{
+  "mappings": {
+    "_doc": {
+      "properties": {
+        "date": {
+          "type": "date_nanos" <1>
+        }
+      }
+    }
+  }
+}
+
+PUT my_index/_doc/1
+{ "date": "2015-01-01" } <2>
+
+PUT my_index/_doc/2
+{ "date": "2015-01-01T12:10:30.123456789Z" } <3>
+
+PUT my_index/_doc/3
+{ "date": 1420070400 } <4>
+
+GET my_index/_search
+{
+  "sort": { "date": "asc"} <5>
+}
+
+GET my_index/_search
+{
+  "script_fields" : {
+    "my_field" : {
+      "script" : {
+        "lang" : "painless",
+        "source" : "doc['date'].date.nanos" <6>
+      }
+    }
+  }
+}
+
+GET my_index/_search
+{
+  "docvalue_fields" : [
+    {
+      "field" : "my_ip_field",
+      "format": "strict_date_time" <7>
+    }
+  ]
+}
+--------------------------------------------------
+// CONSOLE
+
+<1> The `date` field uses the default `format`.
+<2> This document uses a plain date.
+<3> This document includes a time.
+<4> This document uses milliseconds-since-the-epoch.
+<5> Note that the `sort` values that are returned are all in
+nanoseconds-since-the-epoch.
+<6> Access the nanosecond part of the date in a script
+<7> Use doc value fields, which can be formatted in nanosecond
+resolution
+
+You can also specify multiple date formats separated by `||`. The
+same mapping parameters than with the `date` field can be used.
+
+[[date-nanos-limitations]]
+==== Limitations
+
+Aggregations are still on millisecond resolution, even when using a
+`date_nanos` field.
+

rest-api-spec/src/main/resources/rest-api-spec/test/field_caps/10_basic.yml

@@ -14,6 +14,8 @@ setup:
                     type:     double
                   geo:
                     type:     geo_point
+                  date:
+                    type:     date
                   object:
                     type: object
                     properties:
@@ -45,6 +47,8 @@ setup:
                     type:     keyword
                   number:
                     type:     double
+                  date:
+                    type:     date
                   geo:
                     type:     geo_point
                   object:
@@ -77,6 +81,8 @@ setup:
                     type:     keyword
                   number:
                     type:     long
+                  date:
+                    type:     date
                   geo:
                     type:     keyword
                   object:
@@ -104,7 +110,7 @@ setup:
   - do:
       field_caps:
         index: 'test1,test2,test3'
-        fields: [text, keyword, number, geo]
+        fields: [text, keyword, number, date, geo]
 
   - match: {fields.text.text.searchable:                true}
   - match: {fields.text.text.aggregatable:              false}
@@ -126,6 +132,11 @@ setup:
   - match: {fields.number.long.indices:                 ["test3"]}
   - is_false: fields.number.long.non_searchable_indices
   - is_false: fields.number.long.non_aggregatable_indices
+  - match: {fields.date.date.searchable:                true}
+  - match: {fields.date.date.aggregatable:              true}
+  - is_false: fields.date.date.indices
+  - is_false: fields.date.date.non_searchable_indices
+  - is_false: fields.date.date.non_aggregatable_indices
   - match: {fields.geo.geo_point.searchable:            true}
   - match: {fields.geo.geo_point.aggregatable:          true}
   - match: {fields.geo.geo_point.indices:               ["test1", "test2"]}
@@ -137,6 +148,33 @@ setup:
   - is_false: fields.geo.keyword.non_searchable_indices
   - is_false: fields.geo.keyword.on_aggregatable_indices
 ---
+"Get date_nanos field caps":
+  - skip:
+      version: " - 6.99.99"
+      reason: date_nanos field mapping type has been introcued in 7.0
+
+  - do:
+        indices.create:
+          include_type_name: false
+          index: test_nanos
+          body:
+              mappings:
+                properties:
+                  date_nanos:
+                    type:     date_nanos
+
+  - do:
+      field_caps:
+        index: 'test_nanos'
+        fields: [date_nanos]
+
+  - match: {fields.date_nanos.date_nanos.searchable:                true}
+  - match: {fields.date_nanos.date_nanos.aggregatable:              true}
+  - is_false: fields.date_nanos.date_nanos.indices
+  - is_false: fields.date_nanos.date_nanos.non_searchable_indices
+  - is_false: fields.date_nanos.date_nanos.non_aggregatable_indices
+
+---
 "Get leaves field caps":
 
   - do:

rest-api-spec/src/main/resources/rest-api-spec/test/search/240_date_nanos.yml

@@ -0,0 +1,161 @@
+setup:
+  - skip:
+      version: " - 6.99.99"
+      reason:  "Implemented in 7.0"
+
+  - do:
+      indices.create:
+          index: date_ns
+          body:
+            settings:
+              number_of_shards: 3
+              number_of_replicas: 0
+            mappings:
+              properties:
+                date:
+                  type: date_nanos
+                field:
+                  type: long
+
+  - do:
+      indices.create:
+          index: date_ms
+          body:
+            settings:
+              number_of_shards: 3
+              number_of_replicas: 0
+            mappings:
+              properties:
+                date:
+                  type: date
+                field:
+                  type: long
+
+---
+"test sorting against date_nanos only fields":
+
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - '{ "index" : { "_index" : "date_ns", "_id" : "first" } }'
+          # millis [1540815132987] to nanos [1540815132987654321]
+          - '{"date" : "2018-10-29T12:12:12.123456789Z", "field" : 1 }'
+          - '{ "index" : { "_index" : "date_ns", "_id" : "second" } }'
+          # millis [1540815132123] to nanos [1540815132123456789]
+          - '{"date" : "2018-10-29T12:12:12.987654321Z", "field" : 2 }'
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: date_ns*
+        body:
+          sort: [ { "date": "desc" } ]
+
+  - match: { hits.total: 2 }
+  - length: { hits.hits: 2 }
+  - match: { hits.hits.0._id: "second" }
+  - match: { hits.hits.0.sort: [1540815132987654321] }
+  - match: { hits.hits.1._id: "first" }
+  - match: { hits.hits.1.sort: [1540815132123456789] }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: date_ns*
+        body:
+          sort: [ { "date": "asc" } ]
+
+  - match: { hits.total: 2 }
+  - length: { hits.hits: 2 }
+  - match: { hits.hits.0._id: "first" }
+  - match: { hits.hits.0.sort: [1540815132123456789] }
+  - match: { hits.hits.1._id: "second" }
+  - match: { hits.hits.1.sort: [1540815132987654321] }
+
+
+---
+"date_nanos requires dates after 1970 and before 2262":
+
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - '{ "index" : { "_index" : "date_ns", "_id" : "date_ns_1" } }'
+          - '{"date" : "1969-10-28T12:12:12.123456789Z" }'
+          - '{ "index" : { "_index" : "date_ns", "_id" : "date_ns_2" } }'
+          - '{"date" : "2263-10-29T12:12:12.123456789Z" }'
+
+  - match: { errors: true }
+  - match: { items.0.index.status: 400 }
+  - match: { items.0.index.error.type: mapper_parsing_exception }
+  - match: { items.0.index.error.caused_by.reason: "date[1969-10-28T12:12:12.123456789Z] is before the epoch in 1970 and cannot be stored in nanosecond resolution" }
+  - match: { items.1.index.status: 400 }
+  - match: { items.1.index.error.type: mapper_parsing_exception }
+  - match: { items.1.index.error.caused_by.reason: "date[2263-10-29T12:12:12.123456789Z] is after 2262-04-11T23:47:16.854775807 and cannot be stored in nanosecond resolution" }
+
+
+---
+"doc value fields are working as expected across date and date_nanos fields":
+
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - '{ "index" : { "_index" : "date_ns", "_id" : "date_ns_1" } }'
+          - '{"date" : "2018-10-29T12:12:12.123456789Z", "field" : 1 }'
+          - '{ "index" : { "_index" : "date_ms", "_id" : "date_ms_1" } }'
+          - '{"date" : "2018-10-29T12:12:12.987Z" }'
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: date*
+        body:
+          docvalue_fields: [ { "field": "date", "format" : "strict_date_optional_time" }, { "field": "date", "format": "epoch_millis" }, { "field" : "date", "format": "uuuu-MM-dd'T'HH:mm:ss.SSSSSSSSSX" } ]
+          sort: [ { "date": "desc" } ]
+
+  - match: { hits.total: 2 }
+  - length: { hits.hits: 2 }
+  - match: { hits.hits.0._id: "date_ns_1" }
+  - match: { hits.hits.1._id: "date_ms_1" }
+  - match: { hits.hits.0.fields.date: [ "2018-10-29T12:12:12.123Z", "1540815132123.456789", "2018-10-29T12:12:12.123456789Z" ] }
+  - match: { hits.hits.1.fields.date: [ "2018-10-29T12:12:12.987Z", "1540815132987", "2018-10-29T12:12:12.987000000Z" ] }
+
+---
+"date histogram aggregation with date and date_nanos mapping":
+
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - '{ "index" : { "_index" : "date_ns", "_id" : "date_ns_1" } }'
+          - '{"date" : "2018-10-29T12:12:12.123456789Z" }'
+          - '{ "index" : { "_index" : "date_ms", "_id" : "date_ms_1" } }'
+          - '{"date" : "2018-10-29T12:12:12.987Z" }'
+          - '{ "index" : { "_index" : "date_ns", "_id" : "date_ns_2" } }'
+          - '{"date" : "2018-10-30T12:12:12.123456789Z" }'
+          - '{ "index" : { "_index" : "date_ms", "_id" : "date_ms_2" } }'
+          - '{"date" : "2018-10-30T12:12:12.987Z" }'
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: date*
+        body:
+          size: 0
+          aggs:
+            date:
+              date_histogram:
+                field: date
+                interval: 1d
+
+  - match: { hits.total: 4 }
+  - length: { aggregations.date.buckets: 2 }
+  - match: { aggregations.date.buckets.0.key: 1540771200000 }
+  - match: { aggregations.date.buckets.0.key_as_string: "2018-10-29T00:00:00.000Z" }
+  - match: { aggregations.date.buckets.0.doc_count: 2 }
+  - match: { aggregations.date.buckets.1.key: 1540857600000 }
+  - match: { aggregations.date.buckets.1.key_as_string: "2018-10-30T00:00:00.000Z" }
+  - match: { aggregations.date.buckets.1.doc_count: 2 }
+

server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java

@@ -512,7 +512,7 @@ private InternalAggregations reduceAggsIncrementally(List<InternalAggregations>
     }
 
     private static InternalAggregations reduceAggs(List<InternalAggregations> aggregationsList,
-                                            List<SiblingPipelineAggregator> pipelineAggregators, ReduceContext reduceContext) {
+                                               List<SiblingPipelineAggregator> pipelineAggregators, ReduceContext reduceContext) {
         InternalAggregations aggregations = InternalAggregations.reduce(aggregationsList, reduceContext);
         if (pipelineAggregators != null) {
             List<InternalAggregation> newAggs = StreamSupport.stream(aggregations.spliterator(), false)