Extend the date rounding logic to be conditional (elastic#89693)

Date rounding logic should take into account the fields that will be
parsed be a parser. If a parser has a DayOfYear field, the rounding logic
should not try to default DayOfMonth as it will conflict with DayOfYear

However the DateTimeFormatter does not have a public method to return
information of fields that will be parsed. The hacky workaround is
to rely on toString() implementation that will return a field info when
it was defined with textual pattern.

This commits introduced conditional logic for DayOfYear, ClockHourOfAMPM and HourOfAmPM

closes elastic#89096
closes elastic#58986

Author: pgomulka

docs/changelog/89693.yaml

@@ -0,0 +1,7 @@
+pr: 89693
+summary: Extend the date rounding logic to be conditional
+area: Infra/Core
+type: bug
+issues:
+ - 89096
+ - 58986

server/src/main/java/org/elasticsearch/common/time/EpochTime.java

@@ -149,15 +149,15 @@ public long getFrom(TemporalAccessor temporal) {
     static final DateFormatter SECONDS_FORMATTER = new JavaDateFormatter(
         "epoch_second",
         SECONDS_FORMATTER1,
-        builder -> builder.parseDefaulting(ChronoField.NANO_OF_SECOND, 999_999_999L),
+        (builder, parser) -> builder.parseDefaulting(ChronoField.NANO_OF_SECOND, 999_999_999L),
         SECONDS_FORMATTER1,
         SECONDS_FORMATTER2
     );
 
     static final DateFormatter MILLIS_FORMATTER = new JavaDateFormatter(
         "epoch_millis",
         MILLISECONDS_FORMATTER1,
-        builder -> builder.parseDefaulting(EpochTime.NANOS_OF_MILLI, 999_999L),
+        (builder, parser) -> builder.parseDefaulting(EpochTime.NANOS_OF_MILLI, 999_999L),
         MILLISECONDS_FORMATTER1,
         MILLISECONDS_FORMATTER2
     );

server/src/main/java/org/elasticsearch/common/time/JavaDateFormatter.java

@@ -17,46 +17,59 @@
 import java.time.format.DateTimeParseException;
 import java.time.temporal.ChronoField;
 import java.time.temporal.TemporalAccessor;
-import java.time.temporal.TemporalField;
 import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
-import java.util.Map;
 import java.util.Objects;
-import java.util.function.Consumer;
-import java.util.stream.Collectors;
+import java.util.function.BiConsumer;
+import java.util.function.UnaryOperator;
 
 class JavaDateFormatter implements DateFormatter {
-
-    // base fields which should be used for default parsing, when we round up for date math
-    private static final Map<TemporalField, Long> ROUND_UP_BASE_FIELDS = new HashMap<>(6);
-
-    {
-        ROUND_UP_BASE_FIELDS.put(ChronoField.MONTH_OF_YEAR, 1L);
-        ROUND_UP_BASE_FIELDS.put(ChronoField.DAY_OF_MONTH, 1L);
-        ROUND_UP_BASE_FIELDS.put(ChronoField.HOUR_OF_DAY, 23L);
-        ROUND_UP_BASE_FIELDS.put(ChronoField.MINUTE_OF_HOUR, 59L);
-        ROUND_UP_BASE_FIELDS.put(ChronoField.SECOND_OF_MINUTE, 59L);
-        ROUND_UP_BASE_FIELDS.put(ChronoField.NANO_OF_SECOND, 999_999_999L);
-    }
+    /**
+    * A default consumer that allows to round up fields (used for range searches, optional fields missing)
+    * it relies on toString implementation of DateTimeFormatter and ChronoField.
+    * For instance for pattern
+    * the parser would have a toString()
+    * <code>
+    *  Value(MonthOfYear,2)'/'Value(DayOfMonth,2)'/'Value(YearOfEra,4,19,EXCEEDS_PAD)'
+    * 'Value(ClockHourOfAmPm,2)':'Value(MinuteOfHour,2)' 'Text(AmPmOfDay,SHORT)
+    * </code>
+    * and ChronoField.CLOCK_HOUR_OF_AMPM would have toString() ClockHourOfAmPm
+    * this allows the rounding logic to default CLOCK_HOUR_OF_AMPM field instead of HOUR_OF_DAY
+    * without this logic, the rounding would result in a conflict as HOUR_OF_DAY would be missing, but CLOCK_HOUR_OF_AMPM would be provided
+    */
+    private static final BiConsumer<DateTimeFormatterBuilder, DateTimeFormatter> DEFAULT_ROUND_UP = (builder, parser) -> {
+        String parserAsString = parser.toString();
+        if (parserAsString.contains(ChronoField.MONTH_OF_YEAR.toString())) {
+            builder.parseDefaulting(ChronoField.MONTH_OF_YEAR, 1L);
+        }
+        if (parserAsString.contains(ChronoField.DAY_OF_MONTH.toString())) {
+            builder.parseDefaulting(ChronoField.DAY_OF_MONTH, 1L);
+        }
+        if (parserAsString.contains(ChronoField.CLOCK_HOUR_OF_AMPM.toString())) {
+            builder.parseDefaulting(ChronoField.CLOCK_HOUR_OF_AMPM, 11L);
+            builder.parseDefaulting(ChronoField.AMPM_OF_DAY, 1L);
+        } else if (parserAsString.contains(ChronoField.HOUR_OF_AMPM.toString())) {
+            builder.parseDefaulting(ChronoField.HOUR_OF_AMPM, 11L);
+            builder.parseDefaulting(ChronoField.AMPM_OF_DAY, 1L);
+        } else {
+            builder.parseDefaulting(ChronoField.HOUR_OF_DAY, 23L);
+        }
+        builder.parseDefaulting(ChronoField.MINUTE_OF_HOUR, 59L);
+        builder.parseDefaulting(ChronoField.SECOND_OF_MINUTE, 59L);
+        builder.parseDefaulting(ChronoField.NANO_OF_SECOND, 999_999_999L);
+    };
 
     private final String format;
     private final DateTimeFormatter printer;
-    private final List<DateTimeFormatter> parsers;
-    private final JavaDateFormatter roundupParser;
+    private final DateTimeFormatter[] parsers;
+    private final RoundUpFormatter roundupParser;
 
-    static class RoundUpFormatter extends JavaDateFormatter {
+    private static final class RoundUpFormatter extends JavaDateFormatter {
 
-        RoundUpFormatter(String format, List<DateTimeFormatter> roundUpParsers) {
-            super(format, firstFrom(roundUpParsers), null, roundUpParsers);
-        }
-
-        private static DateTimeFormatter firstFrom(List<DateTimeFormatter> roundUpParsers) {
-            return roundUpParsers.get(0);
+        RoundUpFormatter(String format, DateTimeFormatter[] roundUpParsers) {
+            super(format, roundUpParsers[0], (RoundUpFormatter) null, roundUpParsers);
         }
 
         @Override
@@ -67,37 +80,47 @@ JavaDateFormatter getRoundupParser() {
 
     // named formatters use default roundUpParser
     JavaDateFormatter(String format, DateTimeFormatter printer, DateTimeFormatter... parsers) {
-        this(format, printer, builder -> ROUND_UP_BASE_FIELDS.forEach(builder::parseDefaulting), parsers);
+        this(
+            format,
+            printer,
+            // set up base fields which should be used for default parsing, when we round up for date math
+            DEFAULT_ROUND_UP,
+            parsers
+        );
     }
 
     // subclasses override roundUpParser
     JavaDateFormatter(
         String format,
         DateTimeFormatter printer,
-        Consumer<DateTimeFormatterBuilder> roundupParserConsumer,
+        BiConsumer<DateTimeFormatterBuilder, DateTimeFormatter> roundupParserConsumer,
         DateTimeFormatter... parsers
     ) {
         if (printer == null) {
             throw new IllegalArgumentException("printer may not be null");
         }
-        long distinctZones = Arrays.stream(parsers).map(DateTimeFormatter::getZone).distinct().count();
-        if (distinctZones > 1) {
-            throw new IllegalArgumentException("formatters must have the same time zone");
-        }
-        long distinctLocales = Arrays.stream(parsers).map(DateTimeFormatter::getLocale).distinct().count();
-        if (distinctLocales > 1) {
-            throw new IllegalArgumentException("formatters must have the same locale");
-        }
         this.printer = printer;
         this.format = format;
+        this.parsers = parsersArray(printer, parsers);
+        this.roundupParser = createRoundUpParser(format, roundupParserConsumer, locale(), this.parsers);
+    }
 
+    private static DateTimeFormatter[] parsersArray(DateTimeFormatter printer, DateTimeFormatter... parsers) {
         if (parsers.length == 0) {
-            this.parsers = Collections.singletonList(printer);
-        } else {
-            this.parsers = Arrays.asList(parsers);
+            return new DateTimeFormatter[] { printer };
+        }
+        final ZoneId zoneId = parsers[0].getZone();
+        final Locale locale = parsers[0].getLocale();
+        for (int i = 1; i < parsers.length; i++) {
+            final DateTimeFormatter parser = parsers[i];
+            if (Objects.equals(parser.getZone(), zoneId) == false) {
+                throw new IllegalArgumentException("formatters must have the same time zone");
+            }
+            if (Objects.equals(parser.getLocale(), locale) == false) {
+                throw new IllegalArgumentException("formatters must have the same locale");
+            }
         }
-        List<DateTimeFormatter> roundUp = createRoundUpParser(format, roundupParserConsumer);
-        this.roundupParser = new RoundUpFormatter(format, roundUp);
+        return parsers;
     }
 
     /**
@@ -109,16 +132,19 @@ JavaDateFormatter getRoundupParser() {
      * <code>DateFormatters</code>.
      * This means that we need to also have multiple RoundUp parsers.
      */
-    private List<DateTimeFormatter> createRoundUpParser(String format, Consumer<DateTimeFormatterBuilder> roundupParserConsumer) {
+    private static RoundUpFormatter createRoundUpParser(
+        String format,
+        BiConsumer<DateTimeFormatterBuilder, DateTimeFormatter> roundupParserConsumer,
+        Locale locale,
+        DateTimeFormatter[] parsers
+    ) {
         if (format.contains("||") == false) {
-            List<DateTimeFormatter> roundUpParsers = new ArrayList<>();
-            for (DateTimeFormatter parser : this.parsers) {
+            return new RoundUpFormatter(format, mapParsers(parser -> {
                 DateTimeFormatterBuilder builder = new DateTimeFormatterBuilder();
                 builder.append(parser);
-                roundupParserConsumer.accept(builder);
-                roundUpParsers.add(builder.toFormatter(locale()));
-            }
-            return roundUpParsers;
+                roundupParserConsumer.accept(builder, parser);
+                return builder.toFormatter(locale);
+            }, parsers));
         }
         return null;
     }
@@ -136,22 +162,26 @@ public static DateFormatter combined(String input, List<DateFormatter> formatter
             if (printer == null) {
                 printer = javaDateFormatter.getPrinter();
             }
-            parsers.addAll(javaDateFormatter.getParsers());
-            roundUpParsers.addAll(javaDateFormatter.getRoundupParser().getParsers());
+            Collections.addAll(parsers, javaDateFormatter.parsers);
+            Collections.addAll(roundUpParsers, javaDateFormatter.getRoundupParser().parsers);
         }
 
-        return new JavaDateFormatter(input, printer, roundUpParsers, parsers);
+        return new JavaDateFormatter(
+            input,
+            printer,
+            roundUpParsers.toArray(new DateTimeFormatter[0]),
+            parsers.toArray(new DateTimeFormatter[0])
+        );
     }
 
-    private JavaDateFormatter(
-        String format,
-        DateTimeFormatter printer,
-        List<DateTimeFormatter> roundUpParsers,
-        List<DateTimeFormatter> parsers
-    ) {
+    private JavaDateFormatter(String format, DateTimeFormatter printer, DateTimeFormatter[] roundUpParsers, DateTimeFormatter[] parsers) {
+        this(format, printer, new RoundUpFormatter(format, roundUpParsers), parsers);
+    }
+
+    private JavaDateFormatter(String format, DateTimeFormatter printer, RoundUpFormatter roundupParser, DateTimeFormatter[] parsers) {
         this.format = format;
         this.printer = printer;
-        this.roundupParser = roundUpParsers != null ? new RoundUpFormatter(format, roundUpParsers) : null;
+        this.roundupParser = roundupParser;
         this.parsers = parsers;
     }
 
@@ -191,7 +221,7 @@ public TemporalAccessor parse(String input) {
      * @throws DateTimeParseException when unable to parse with any parsers
      */
     private TemporalAccessor doParse(String input) {
-        if (parsers.size() > 1) {
+        if (parsers.length > 1) {
             for (DateTimeFormatter formatter : parsers) {
                 ParsePosition pos = new ParsePosition(0);
                 Object object = formatter.toFormat().parseObject(input, pos);
@@ -201,10 +231,10 @@ private TemporalAccessor doParse(String input) {
             }
             throw new DateTimeParseException("Failed to parse with all enclosed parsers", input, 0);
         }
-        return this.parsers.get(0).parse(input);
+        return this.parsers[0].parse(input);
     }
 
-    private boolean parsingSucceeded(Object object, String input, ParsePosition pos) {
+    private static boolean parsingSucceeded(Object object, String input, ParsePosition pos) {
         return object != null && pos.getIndex() == input.length();
     }
 
@@ -214,12 +244,7 @@ public DateFormatter withZone(ZoneId zoneId) {
         if (zoneId.equals(zone())) {
             return this;
         }
-        List<DateTimeFormatter> parsers = this.parsers.stream().map(p -> p.withZone(zoneId)).collect(Collectors.toList());
-        List<DateTimeFormatter> roundUpParsers = this.roundupParser.getParsers()
-            .stream()
-            .map(p -> p.withZone(zoneId))
-            .collect(Collectors.toList());
-        return new JavaDateFormatter(format, printer.withZone(zoneId), roundUpParsers, parsers);
+        return mapParsers(p -> p.withZone(zoneId));
     }
 
     @Override
@@ -228,12 +253,24 @@ public DateFormatter withLocale(Locale locale) {
         if (locale.equals(locale())) {
             return this;
         }
-        List<DateTimeFormatter> parsers = this.parsers.stream().map(p -> p.withLocale(locale)).collect(Collectors.toList());
-        List<DateTimeFormatter> roundUpParsers = this.roundupParser.getParsers()
-            .stream()
-            .map(p -> p.withLocale(locale))
-            .collect(Collectors.toList());
-        return new JavaDateFormatter(format, printer.withLocale(locale), roundUpParsers, parsers);
+        return mapParsers(p -> p.withLocale(locale));
+    }
+
+    private JavaDateFormatter mapParsers(UnaryOperator<DateTimeFormatter> mapping) {
+        return new JavaDateFormatter(
+            format,
+            mapping.apply(printer),
+            mapParsers(mapping, ((JavaDateFormatter) this.roundupParser).parsers),
+            mapParsers(mapping, this.parsers)
+        );
+    }
+
+    private static DateTimeFormatter[] mapParsers(UnaryOperator<DateTimeFormatter> mapping, DateTimeFormatter[] parsers) {
+        DateTimeFormatter[] res = new DateTimeFormatter[parsers.length];
+        for (int i = 0; i < parsers.length; i++) {
+            res[i] = mapping.apply(parsers[i]);
+        }
+        return res;
     }
 
     @Override
@@ -283,7 +320,4 @@ public String toString() {
         return String.format(Locale.ROOT, "format[%s] locale[%s]", format, locale());
     }
 
-    Collection<DateTimeFormatter> getParsers() {
-        return parsers;
-    }
 }

server/src/test/java/org/elasticsearch/common/time/JavaDateMathParserTests.java

@@ -86,6 +86,38 @@ public void testOverridingLocaleOrZoneAndCompositeRoundUpParser() {
         assertDateEquals(gotMillis, "297276785531", "297276785531");
     }
 
+    public void testWeekBasedDate() {
+        DateFormatter formatter = DateFormatter.forPattern("strict_basic_week_date");// YYYY'W'wwe
+        // first week of 2022 is starting on Monday 3rd Jan
+        assertDateMathEquals(formatter.toDateMathParser(), "2022W0101", "2022-01-03T23:59:59.999Z", 0, true, ZoneOffset.UTC);
+
+        // defaulting missing day of week
+        formatter = DateFormatter.forPattern("YYYY'W'ww[e]");// YYYY'W'wwe
+        // second week of 2022 is starting on Monday 10th Jan
+        assertDateMathEquals(formatter.toDateMathParser(), "2022W02", "2022-01-10T23:59:59.999Z", 0, true, ZoneOffset.UTC);
+    }
+
+    public void testDayOfYear() {
+        DateFormatter formatter = DateFormatter.forPattern("yyyy-DDD'T'HH:mm:ss.SSS");
+        assertDateMathEquals(formatter.toDateMathParser(), "2022-104T14:08:30.293", "2022-04-14T14:08:30.293", 0, true, ZoneOffset.UTC);
+    }
+
+    public void testAMPM() {
+        DateFormatter formatter = DateFormatter.forPattern("MM/dd/yyyy hh:mm a"); // h clock-hour-of-am-pm (1-12)
+        assertDateMathEquals(formatter.toDateMathParser(), "04/30/2020 12:48 AM", "2020-04-30T00:48:59.999Z", 0, true, ZoneOffset.UTC);
+
+        formatter = DateFormatter.forPattern("MM/dd/yyyy KK:mm a"); // K hour-of-am-pm (0-11)
+        assertDateMathEquals(formatter.toDateMathParser(), "04/30/2020 00:48 AM", "2020-04-30T00:48:59.999Z", 0, true, ZoneOffset.UTC);
+    }
+
+    public void testAMPMWithTimeMissing() {
+        DateFormatter formatter = DateFormatter.forPattern("MM/dd/yyyy[ hh:mm a]"); // h clock-hour-of-am-pm (1-12)
+        assertDateMathEquals(formatter.toDateMathParser(), "04/30/2020", "2020-04-30T23:59:59.999Z", 0, true, ZoneOffset.UTC);
+
+        formatter = DateFormatter.forPattern("MM/dd/yyyy[ KK:mm a]"); // K hour-of-am-pm (0-11)
+        assertDateMathEquals(formatter.toDateMathParser(), "04/30/2020", "2020-04-30T23:59:59.999Z", 0, true, ZoneOffset.UTC);
+    }
+
     public void testWeekDates() {
         assumeFalse(
             "won't work in jdk8 " + "because SPI mechanism is not looking at classpath - needs ISOCalendarDataProvider in jre's ext/libs",