Refactor ObjectParser and CompatibleObjectParser to support REST Compatible API (elastic#68808)

In order to support compatible fields when parsing XContent additional information has to be set during ParsedField declaration.
This commit adds a set of RestApiCompatibleVersion on a ParsedField in order to specify on which versions a field is supported. By default ParsedField is allowed to be parsed on both current and previous major versions.

ObjectParser - which is used for constructing objects using 'setters' - has a modified fieldParsersMap to be Map of Maps. with key being RestApiCompatibility. This allows to choose set of field-parsers as specified on a request.
Under RestApiCompatibility.minimumSupported key, there is a map that contains field-parsers for both previous and current versions.
Under RestApiCompatibility.current there will be only current versions field (compatible fields not a present)

ConstructingObjectParser - which is used for constructing objects using 'constructors' - is modified to contain a map of Version To constructorArgInfo , declarations of fields to be set on a constructor depending on a version

relates elastic#51816

Author: pgomulka

libs/core/src/main/java/org/elasticsearch/common/compatibility/RestApiCompatibleVersion.java

@@ -19,8 +19,8 @@ public enum RestApiCompatibleVersion {
     V_8(8),
     V_7(7);
 
-    public byte major;
-    private static RestApiCompatibleVersion CURRENT = V_8;
+    public final byte major;
+    private static final RestApiCompatibleVersion CURRENT = V_8;
 
     RestApiCompatibleVersion(int major) {
         this.major = (byte) major;

libs/x-content/src/main/java/org/elasticsearch/common/ParseField.java

@@ -7,11 +7,15 @@
  */
 package org.elasticsearch.common;
 
+import org.elasticsearch.common.compatibility.RestApiCompatibleVersion;
 import org.elasticsearch.common.xcontent.DeprecationHandler;
 import org.elasticsearch.common.xcontent.XContentLocation;
 
+import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Objects;
 import java.util.Set;
 import java.util.function.Supplier;
@@ -23,21 +27,15 @@
 public class ParseField {
     private final String name;
     private final String[] deprecatedNames;
+    private final Set<RestApiCompatibleVersion> restApiCompatibleVersions = new HashSet<>(2);
     private String allReplacedWith = null;
     private final String[] allNames;
     private boolean fullyDeprecated = false;
 
     private static final String[] EMPTY = new String[0];
 
-    /**
-     * @param name
-     *            the primary name for this field. This will be returned by
-     *            {@link #getPreferredName()}
-     * @param deprecatedNames
-     *            names for this field which are deprecated and will not be
-     *            accepted when strict matching is used.
-     */
-    public ParseField(String name, String... deprecatedNames) {
+
+    private ParseField(String name, Collection<RestApiCompatibleVersion> restApiCompatibleVersions, String[] deprecatedNames) {
         this.name = name;
         if (deprecatedNames == null || deprecatedNames.length == 0) {
             this.deprecatedNames = EMPTY;
@@ -46,12 +44,25 @@ public ParseField(String name, String... deprecatedNames) {
             Collections.addAll(set, deprecatedNames);
             this.deprecatedNames = set.toArray(new String[set.size()]);
         }
+        this.restApiCompatibleVersions.addAll(restApiCompatibleVersions);
+
         Set<String> allNames = new HashSet<>();
         allNames.add(name);
         Collections.addAll(allNames, this.deprecatedNames);
         this.allNames = allNames.toArray(new String[allNames.size()]);
     }
 
+    /**
+     * Creates a field available for lookup for both current and previous REST API compatible versions
+     * @param name            the primary name for this field. This will be returned by
+     *                        {@link #getPreferredName()}
+     * @param deprecatedNames names for this field which are deprecated and will not be
+     *                        accepted when strict matching is used.
+     */
+    public ParseField(String name, String... deprecatedNames) {
+        this(name, List.of(RestApiCompatibleVersion.currentVersion(), RestApiCompatibleVersion.minimumSupported()) ,deprecatedNames);
+    }
+
     /**
      * @return the preferred name used for this field
      */
@@ -78,6 +89,22 @@ public ParseField withDeprecation(String... deprecatedNames) {
         return new ParseField(this.name, deprecatedNames);
     }
 
+
+    /**
+     * Creates a new field with current name and deprecatedNames, but overrides restApiCompatibleVersions
+     * @param restApiCompatibleVersions rest api compatibility versions under which specifies when a lookup will be allowed
+     */
+    public ParseField withRestApiCompatibilityVersions(RestApiCompatibleVersion... restApiCompatibleVersions) {
+        return new ParseField(this.name, Arrays.asList(restApiCompatibleVersions), this.deprecatedNames);
+    }
+
+    /**
+     * @return rest api compatibility versions under which a lookup will be allowed
+     */
+    public Set<RestApiCompatibleVersion> getRestApiCompatibleVersions() {
+        return restApiCompatibleVersions;
+    }
+
     /**
      * Return a new ParseField where all field names are deprecated and replaced
      * with {@code allReplacedWith}.
@@ -169,6 +196,7 @@ public String[] getDeprecatedNames() {
         return deprecatedNames;
     }
 
+
     public static class CommonFields {
         public static final ParseField FIELD = new ParseField("field");
         public static final ParseField FIELDS = new ParseField("fields");

libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ConstructingObjectParser.java

@@ -9,16 +9,21 @@
 package org.elasticsearch.common.xcontent;
 
 import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.compatibility.RestApiCompatibleVersion;
 import org.elasticsearch.common.xcontent.ObjectParser.NamedObjectParser;
 import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
+import java.util.EnumMap;
 import java.util.List;
+import java.util.Map;
 import java.util.function.BiConsumer;
 import java.util.function.BiFunction;
 import java.util.function.Consumer;
 import java.util.function.Function;
+import java.util.stream.Collectors;
 
 /**
  * Like {@link ObjectParser} but works with objects that have constructors whose arguments are mixed in with its other settings. Queries are
@@ -82,7 +87,8 @@ public final class ConstructingObjectParser<Value, Context> extends AbstractObje
     /**
      * List of constructor names used for generating the error message if not all arrive.
      */
-    private final List<ConstructorArgInfo> constructorArgInfos = new ArrayList<>();
+    private final Map<RestApiCompatibleVersion, List<ConstructorArgInfo>> constructorArgInfos =
+        new EnumMap<>(RestApiCompatibleVersion.class);
     private final ObjectParser<Target, Context> objectParser;
     private final BiFunction<Object[], Context, Value> builder;
     /**
@@ -205,8 +211,8 @@ public <T> void declareField(BiConsumer<Value, T> consumer, ContextParser<Contex
              * constructor in the argument list so we don't need to do any fancy
              * or expensive lookups whenever the constructor args come in.
              */
-            int position = addConstructorArg(consumer, parseField);
-            objectParser.declareField((target, v) -> target.constructorArg(position, v), parser, parseField, type);
+            Map<RestApiCompatibleVersion, Integer> positions = addConstructorArg(consumer, parseField);
+            objectParser.declareField((target, v) -> target.constructorArg(positions, v), parser, parseField, type);
         } else {
             numberOfFields += 1;
             objectParser.declareField(queueingConsumer(consumer, parseField), parser, parseField, type);
@@ -234,8 +240,8 @@ public <T> void declareNamedObject(BiConsumer<Value, T> consumer, NamedObjectPar
              * constructor in the argument list so we don't need to do any fancy
              * or expensive lookups whenever the constructor args come in.
              */
-            int position = addConstructorArg(consumer, parseField);
-            objectParser.declareNamedObject((target, v) -> target.constructorArg(position, v), namedObjectParser, parseField);
+            Map<RestApiCompatibleVersion, Integer> positions = addConstructorArg(consumer, parseField);
+            objectParser.declareNamedObject((target, v) -> target.constructorArg(positions, v), namedObjectParser, parseField);
         } else {
             numberOfFields += 1;
             objectParser.declareNamedObject(queueingConsumer(consumer, parseField), namedObjectParser, parseField);
@@ -264,8 +270,8 @@ public <T> void declareNamedObjects(BiConsumer<Value, List<T>> consumer, NamedOb
              * constructor in the argument list so we don't need to do any fancy
              * or expensive lookups whenever the constructor args come in.
              */
-            int position = addConstructorArg(consumer, parseField);
-            objectParser.declareNamedObjects((target, v) -> target.constructorArg(position, v), namedObjectParser, parseField);
+            Map<RestApiCompatibleVersion, Integer> positions = addConstructorArg(consumer, parseField);
+            objectParser.declareNamedObjects((target, v) -> target.constructorArg(positions, v), namedObjectParser, parseField);
         } else {
             numberOfFields += 1;
             objectParser.declareNamedObjects(queueingConsumer(consumer, parseField), namedObjectParser, parseField);
@@ -296,18 +302,21 @@ public <T> void declareNamedObjects(BiConsumer<Value, List<T>> consumer, NamedOb
              * constructor in the argument list so we don't need to do any fancy
              * or expensive lookups whenever the constructor args come in.
              */
-            int position = addConstructorArg(consumer, parseField);
-            objectParser.declareNamedObjects((target, v) -> target.constructorArg(position, v), namedObjectParser,
-                    wrapOrderedModeCallBack(orderedModeCallback), parseField);
+            Map<RestApiCompatibleVersion, Integer> positions = addConstructorArg(consumer, parseField);
+            objectParser.declareNamedObjects((target, v) -> target.constructorArg(positions, v), namedObjectParser,
+                wrapOrderedModeCallBack(orderedModeCallback), parseField);
         } else {
             numberOfFields += 1;
             objectParser.declareNamedObjects(queueingConsumer(consumer, parseField), namedObjectParser,
-                    wrapOrderedModeCallBack(orderedModeCallback), parseField);
+                wrapOrderedModeCallBack(orderedModeCallback), parseField);
         }
     }
 
     int getNumberOfFields() {
-        return this.constructorArgInfos.size();
+        assert this.constructorArgInfos.get(RestApiCompatibleVersion.currentVersion()).size()
+            == this.constructorArgInfos.get(RestApiCompatibleVersion.minimumSupported()).size() :
+            "Constructors must have same number of arguments per all compatible versions";
+        return this.constructorArgInfos.get(RestApiCompatibleVersion.currentVersion()).size();
     }
 
     /**
@@ -324,11 +333,17 @@ private boolean isConstructorArg(BiConsumer<?, ?> consumer) {
      * @param parseField Parse field
      * @return The argument position
      */
-    private int addConstructorArg(BiConsumer<?, ?> consumer, ParseField parseField) {
-        int position = constructorArgInfos.size();
+    private Map<RestApiCompatibleVersion, Integer> addConstructorArg(BiConsumer<?, ?> consumer, ParseField parseField) {
+
         boolean required = consumer == REQUIRED_CONSTRUCTOR_ARG_MARKER;
-        constructorArgInfos.add(new ConstructorArgInfo(parseField, required));
-        return position;
+        for (RestApiCompatibleVersion restApiCompatibleVersion : parseField.getRestApiCompatibleVersions()) {
+
+            constructorArgInfos.computeIfAbsent(restApiCompatibleVersion, (v)-> new ArrayList<>())
+                    .add(new ConstructorArgInfo(parseField, required));
+        }
+
+        //calculate the positions for the arguments
+        return constructorArgInfos.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().size()));
     }
 
     @Override
@@ -398,7 +413,7 @@ private class Target {
         /**
          * Array of constructor args to be passed to the {@link ConstructingObjectParser#builder}.
          */
-        private final Object[] constructorArgs = new Object[constructorArgInfos.size()];
+        private final Object[] constructorArgs;
         /**
          * The parser this class is working against. We store it here so we can fetch it conveniently when queueing fields to lookup the
          * location of each field so that we can give a useful error message when replaying the queue.
@@ -437,15 +452,18 @@ private class Target {
         Target(XContentParser parser, Context context) {
             this.parser = parser;
             this.context = context;
+            this.constructorArgs = new Object[constructorArgInfos
+                .getOrDefault(parser.getRestApiCompatibleVersion(), Collections.emptyList()).size()];
         }
 
         /**
          * Set a constructor argument and build the target object if all constructor arguments have arrived.
          */
-        private void constructorArg(int position, Object value) {
+        private void constructorArg(Map<RestApiCompatibleVersion, Integer> positions, Object value) {
+            int position = positions.get(parser.getRestApiCompatibleVersion()) - 1;
             constructorArgs[position] = value;
             constructorArgsCollected++;
-            if (constructorArgsCollected == constructorArgInfos.size()) {
+            if (constructorArgsCollected == constructorArgInfos.get(parser.getRestApiCompatibleVersion()).size()) {
                 buildTarget();
             }
         }
@@ -480,7 +498,7 @@ private Value finish() {
             StringBuilder message = null;
             for (int i = 0; i < constructorArgs.length; i++) {
                 if (constructorArgs[i] != null) continue;
-                ConstructorArgInfo arg = constructorArgInfos.get(i);
+                ConstructorArgInfo arg = constructorArgInfos.get(parser.getRestApiCompatibleVersion()).get(i);
                 if (false == arg.required) continue;
                 if (message == null) {
                     message = new StringBuilder("Required [").append(arg.field);

libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java

@@ -9,11 +9,13 @@
 
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.compatibility.RestApiCompatibleVersion;
 
 import java.io.IOException;
 import java.lang.reflect.Array;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -88,7 +90,9 @@ private static <Value, Context> UnknownFieldParser<Value, Context> ignoreUnknown
 
     private static <Value, Context> UnknownFieldParser<Value, Context> errorOnUnknown() {
         return (op, f, l, p, v, c) -> {
-            throw new XContentParseException(l, ErrorOnUnknown.IMPLEMENTATION.errorMessage(op.name, f, op.fieldParserMap.keySet()));
+            throw new XContentParseException(l, ErrorOnUnknown.IMPLEMENTATION.errorMessage(op.name, f,
+                op.fieldParserMap.getOrDefault(p.getRestApiCompatibleVersion(), Collections.emptyMap())
+                .keySet()));
         };
     }
 
@@ -137,7 +141,9 @@ private static <Value, Category, Context> UnknownFieldParser<Value, Context> unk
             try {
                 o = parser.namedObject(categoryClass, field, context);
             } catch (NamedObjectNotFoundException e) {
-                Set<String> candidates = new HashSet<>(objectParser.fieldParserMap.keySet());
+                Set<String> candidates = new HashSet<>(objectParser.fieldParserMap
+                    .getOrDefault(parser.getRestApiCompatibleVersion(), Collections.emptyMap())
+                    .keySet());
                 e.getCandidates().forEach(candidates::add);
                 String message = ErrorOnUnknown.IMPLEMENTATION.errorMessage(objectParser.name, field, candidates);
                 throw new XContentParseException(location, message, e);
@@ -146,7 +152,7 @@ private static <Value, Category, Context> UnknownFieldParser<Value, Context> unk
         };
     }
 
-    private final Map<String, FieldParser> fieldParserMap = new HashMap<>();
+    private final Map<RestApiCompatibleVersion, Map<String, FieldParser>> fieldParserMap = new HashMap<>();
     private final String name;
     private final Function<Context, Value> valueBuilder;
     private final UnknownFieldParser<Value, Context> unknownFieldParser;
@@ -277,7 +283,8 @@ public Value parse(XContentParser parser, Value value, Context context) throws I
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
                 currentPosition = parser.getTokenLocation();
-                fieldParser = fieldParserMap.get(currentFieldName);
+                fieldParser = fieldParserMap.getOrDefault(parser.getRestApiCompatibleVersion(), Collections.emptyMap())
+                    .get(currentFieldName);
             } else {
                 if (currentFieldName == null) {
                     throw new XContentParseException(parser.getTokenLocation(), "[" + name  + "] no field found");
@@ -359,8 +366,16 @@ public void declareField(Parser<Value, Context> p, ParseField parseField, ValueT
         }
         FieldParser fieldParser = new FieldParser(p, type.supportedTokens(), parseField, type);
         for (String fieldValue : parseField.getAllNamesIncludedDeprecated()) {
-            fieldParserMap.putIfAbsent(fieldValue, fieldParser);
+            if (parseField.getRestApiCompatibleVersions().contains(RestApiCompatibleVersion.minimumSupported())) {
+                fieldParserMap.putIfAbsent(RestApiCompatibleVersion.minimumSupported(), new HashMap<>());
+                fieldParserMap.get(RestApiCompatibleVersion.minimumSupported()).putIfAbsent(fieldValue, fieldParser);
+            }
+            if (parseField.getRestApiCompatibleVersions().contains(RestApiCompatibleVersion.currentVersion())) {
+                fieldParserMap.putIfAbsent(RestApiCompatibleVersion.currentVersion(), new HashMap<>());
+                fieldParserMap.get(RestApiCompatibleVersion.currentVersion()).putIfAbsent(fieldValue, fieldParser);
+            }
         }
+
     }
 
     @Override
@@ -657,7 +672,7 @@ public EnumSet<XContentParser.Token> supportedTokens() {
     public String toString() {
         return "ObjectParser{" +
                 "name='" + name + '\'' +
-                ", fields=" + fieldParserMap.values() +
+                ", fields=" + fieldParserMap +
                 '}';
     }
 }