Add a consistent way to parse dates (#61105)

nik9000 · web-flow · commit 3870b6f1c78a · 2020-08-17T14:49:39.000-04:00
This adds a `parse` method to `date` valued `runtime_script`s that
converts any `Object` into a `long` using the same mechanism that
indexing a `date` uses.

We also use this new method to expand our test that run's core's
integration tests against runtime fields to `date` values fields.
Previously we couldn't do that because we didn't have a consistent way
to parse the field.
diff --git a/x-pack/plugin/runtime-fields/qa/rest/build.gradle b/x-pack/plugin/runtime-fields/qa/rest/build.gradle
@@ -33,6 +33,7 @@ yamlRestTest {
       'search/10_source_filtering/docvalue_fields with explicit format',
       'search.aggregation/200_top_hits_metric/top_hits aggregation with sequence numbers',
       'search.aggregation/200_top_hits_metric/top_hits aggregation with nested documents',
+      'search/140_pre_filter_search_shards/pre_filter_shard_size with shards that have no hit',
       /////// TO FIX ///////
 
       /////// NOT SUPPORTED ///////
diff --git a/x-pack/plugin/runtime-fields/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/rest/CoreTestsWithRuntimeFieldsIT.java b/x-pack/plugin/runtime-fields/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/rest/CoreTestsWithRuntimeFieldsIT.java
@@ -11,6 +11,7 @@
 
 import org.elasticsearch.common.xcontent.XContentLocation;
 import org.elasticsearch.index.mapper.BooleanFieldMapper;
+import org.elasticsearch.index.mapper.DateFieldMapper;
 import org.elasticsearch.index.mapper.IpFieldMapper;
 import org.elasticsearch.index.mapper.KeywordFieldMapper;
 import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType;
@@ -183,8 +184,8 @@ private static String painlessToLoadFromSource(String name, String type) {
     }
 
     private static final Map<String, String> PAINLESS_TO_EMIT = Map.ofEntries(
-        // TODO implement dates against the parser
         Map.entry(BooleanFieldMapper.CONTENT_TYPE, "value(parse(value));"),
+        Map.entry(DateFieldMapper.CONTENT_TYPE, "millis(parse(value.toString()));"),
         Map.entry(
             NumberType.DOUBLE.typeName(),
             "value(value instanceof Number ? ((Number) value).doubleValue() : Double.parseDouble(value.toString()));"
diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DateScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DateScriptFieldScript.java
@@ -7,6 +7,7 @@
 package org.elasticsearch.xpack.runtimefields;
 
 import org.apache.lucene.index.LeafReaderContext;
+import org.elasticsearch.common.time.DateFormatter;
 import org.elasticsearch.painless.spi.Whitelist;
 import org.elasticsearch.painless.spi.WhitelistLoader;
 import org.elasticsearch.script.ScriptContext;
@@ -29,15 +30,18 @@ static List<Whitelist> whitelist() {
     public static final String[] PARAMETERS = {};
 
     public interface Factory extends ScriptFactory {
-        LeafFactory newFactory(Map<String, Object> params, SearchLookup searchLookup);
+        LeafFactory newFactory(Map<String, Object> params, SearchLookup searchLookup, DateFormatter formatter);
     }
 
     public interface LeafFactory {
         DateScriptFieldScript newInstance(LeafReaderContext ctx) throws IOException;
     }
 
-    public DateScriptFieldScript(Map<String, Object> params, SearchLookup searchLookup, LeafReaderContext ctx) {
+    private final DateFormatter formatter;
+
+    public DateScriptFieldScript(Map<String, Object> params, SearchLookup searchLookup, DateFormatter formatter, LeafReaderContext ctx) {
         super(params, searchLookup, ctx);
+        this.formatter = formatter;
     }
 
     public static class Millis {
@@ -67,4 +71,15 @@ public void date(TemporalAccessor v) {
         }
     }
 
+    public static class Parse {
+        private final DateScriptFieldScript script;
+
+        public Parse(DateScriptFieldScript script) {
+            this.script = script;
+        }
+
+        public long parse(Object str) {
+            return script.formatter.parseMillis(str.toString());
+        }
+    }
 }
diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptDateMappedFieldType.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptDateMappedFieldType.java
@@ -85,7 +85,7 @@ public ScriptDateFieldData.Builder fielddataBuilder(String fullyQualifiedIndexNa
     }
 
     private DateScriptFieldScript.LeafFactory leafFactory(SearchLookup lookup) {
-        return scriptFactory.newFactory(script.getParams(), lookup);
+        return scriptFactory.newFactory(script.getParams(), lookup, dateTimeFormatter);
     }
 
     @Override
diff --git a/x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/date_whitelist.txt b/x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/date_whitelist.txt
@@ -12,6 +12,7 @@ class org.elasticsearch.xpack.runtimefields.DateScriptFieldScript @no_import {
 static_import {
     void millis(org.elasticsearch.xpack.runtimefields.DateScriptFieldScript, long) bound_to org.elasticsearch.xpack.runtimefields.DateScriptFieldScript$Millis
     void date(org.elasticsearch.xpack.runtimefields.DateScriptFieldScript, TemporalAccessor) bound_to org.elasticsearch.xpack.runtimefields.DateScriptFieldScript$Date
+    long parse(org.elasticsearch.xpack.runtimefields.DateScriptFieldScript, def) bound_to org.elasticsearch.xpack.runtimefields.DateScriptFieldScript$Parse
 }
 
 # This import is required to make painless happy and it isn't 100% clear why
diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeScriptFieldMapperTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeScriptFieldMapperTests.java
@@ -349,7 +349,12 @@ public void execute() {
                         };
                     }
                     if (context == DateScriptFieldScript.CONTEXT) {
-                        return (DateScriptFieldScript.Factory) (params, lookup) -> ctx -> new DateScriptFieldScript(params, lookup, ctx) {
+                        return (DateScriptFieldScript.Factory) (params, lookup, formatter) -> ctx -> new DateScriptFieldScript(
+                            params,
+                            lookup,
+                            formatter,
+                            ctx
+                        ) {
                             @Override
                             public void execute() {
                                 new DateScriptFieldScript.Millis(this).millis(1595431354874L);
diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptDateMappedFieldTypeTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptDateMappedFieldTypeTests.java
@@ -389,7 +389,7 @@ protected ScriptDateMappedFieldType simpleMappedFieldType() throws IOException {
     }
 
     private ScriptDateMappedFieldType coolFormattedFieldType() throws IOException {
-        return build(simpleMappedFieldType().script, DateFormatter.forPattern("yyyy-MM-dd(-■_■)HH:mm:ss.SSSz"));
+        return build(simpleMappedFieldType().script, DateFormatter.forPattern("yyyy-MM-dd(-■_■)HH:mm:ss.SSSz||epoch_millis"));
     }
 
     @Override
@@ -435,16 +435,17 @@ public <FactoryType> FactoryType compile(
                     private DateScriptFieldScript.Factory factory(String code) {
                         switch (code) {
                             case "read_timestamp":
-                                return (params, lookup) -> (ctx) -> new DateScriptFieldScript(params, lookup, ctx) {
+                                return (params, lookup, formatter) -> ctx -> new DateScriptFieldScript(params, lookup, formatter, ctx) {
                                     @Override
                                     public void execute() {
                                         for (Object timestamp : (List<?>) getSource().get("timestamp")) {
-                                            new DateScriptFieldScript.Millis(this).millis(((Number) timestamp).longValue());
+                                            DateScriptFieldScript.Parse parse = new DateScriptFieldScript.Parse(this);
+                                            new DateScriptFieldScript.Millis(this).millis(parse.parse(timestamp));
                                         }
                                     }
                                 };
                             case "add_days":
-                                return (params, lookup) -> (ctx) -> new DateScriptFieldScript(params, lookup, ctx) {
+                                return (params, lookup, formatter) -> ctx -> new DateScriptFieldScript(params, lookup, formatter, ctx) {
                                     @Override
                                     public void execute() {
                                         for (Object timestamp : (List<?>) getSource().get("timestamp")) {
diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/LongScriptFieldDistanceFeatureQueryTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/LongScriptFieldDistanceFeatureQueryTests.java
@@ -87,7 +87,7 @@ public void testMatches() throws IOException {
             try (DirectoryReader reader = iw.getReader()) {
                 IndexSearcher searcher = newSearcher(reader);
                 CheckedFunction<LeafReaderContext, AbstractLongScriptFieldScript, IOException> leafFactory =
-                    ctx -> new DateScriptFieldScript(Map.of(), new SearchLookup(null, null), ctx) {
+                    ctx -> new DateScriptFieldScript(Map.of(), new SearchLookup(null, null), null, ctx) {
                         @Override
                         public void execute() {
                             for (Object timestamp : (List<?>) getSource().get("timestamp")) {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/40_date.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/40_date.yml
@@ -27,13 +27,13 @@ setup:
                     }
                   params:
                     days: 1
-              # Test fetching from _source
+              # Test fetching from _source and parsing
               tomorrow_from_source:
                 type: runtime_script
                 runtime_type: date
                 script:
                   source: |
-                    Instant instant = Instant.ofEpochMilli(source['timestamp']);
+                    Instant instant = Instant.ofEpochMilli(parse(source['timestamp']));
                     ZonedDateTime dt = ZonedDateTime.ofInstant(instant, ZoneId.of("UTC"));
                     date(dt.plus(1, ChronoUnit.DAYS));
               # Test returning millis
@@ -82,7 +82,7 @@ setup:
           {"index":{}}
           {"timestamp": 1516383694000, "temperature": 200, "voltage": 4.2, "node": "c"}
           {"index":{}}
-          {"timestamp": 1516297294000, "temperature": 202, "voltage": 4.0, "node": "c"}
+          {"timestamp": "2018-01-18T17:41:34.000Z", "temperature": 202, "voltage": 4.0, "node": "c"}
 
 ---
 "get mapping":

Original file line number	Diff line number	Diff line change
`@@ -85,7 +85,7 @@ public ScriptDateFieldData.Builder fielddataBuilder(String fullyQualifiedIndexNa`
`85`	`85`	`}`
`86`	`86`
`87`	`87`	`private DateScriptFieldScript.LeafFactory leafFactory(SearchLookup lookup) {`
`88`		`- return scriptFactory.newFactory(script.getParams(), lookup);`
	`88`	`+ return scriptFactory.newFactory(script.getParams(), lookup, dateTimeFormatter);`
`89`	`89`	`}`
`90`	`90`
`91`	`91`	`@Override`
Original file line number	Diff line number	Diff line change
`@@ -12,6 +12,7 @@ class org.elasticsearch.xpack.runtimefields.DateScriptFieldScript @no_import {`
`12`	`12`	`static_import {`
`13`	`13`	`void millis(org.elasticsearch.xpack.runtimefields.DateScriptFieldScript, long) bound_to org.elasticsearch.xpack.runtimefields.DateScriptFieldScript$Millis`
`14`	`14`	`void date(org.elasticsearch.xpack.runtimefields.DateScriptFieldScript, TemporalAccessor) bound_to org.elasticsearch.xpack.runtimefields.DateScriptFieldScript$Date`
	`15`	`+ long parse(org.elasticsearch.xpack.runtimefields.DateScriptFieldScript, def) bound_to org.elasticsearch.xpack.runtimefields.DateScriptFieldScript$Parse`
`15`	`16`	`}`
`16`	`17`
`17`	`18`	`# This import is required to make painless happy and it isn't 100% clear why`