Add option to ObjectParser to consume unknown fields

Author: romseygeek

libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ConstructingObjectParser.java

@@ -149,7 +149,6 @@ public ConstructingObjectParser(String name, boolean ignoreUnknownFields, Functi
     public ConstructingObjectParser(String name, boolean ignoreUnknownFields, BiFunction<Object[], Context, Value> builder) {
         objectParser = new ObjectParser<>(name, ignoreUnknownFields, null);
         this.builder = builder;
-
     }
 
     /**

libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java

@@ -78,14 +78,69 @@ public static <Value, ElementValue> BiConsumer<Value, List<ElementValue>> fromLi
         };
     }
 
+    private interface UnknownFieldParser<Value, Context> {
+
+        /**
+         * Called when an unknown field is encountered
+         * @param parserName    the parent ObjectParser name
+         * @param field         the name of the unknown field
+         * @param parser        the parser to build values from
+         */
+        void acceptUnknownField(String parserName, String field, XContentParser parser, Value value, Context context) throws IOException;
+    }
+
+    private static <Value, Context> UnknownFieldParser<Value, Context> ignoreUnknown() {
+      return (n, f, x, v, c) -> x.skipChildren();
+    }
+    
+    private static <Value, Context> UnknownFieldParser<Value, Context> errorOnUnknown() {
+        return (n, f, x, v, c) -> {
+            throw new XContentParseException(x.getTokenLocation(),
+                "[" + n + "] unknown field [" + f + "], parser not found");
+        };
+    }
+
+    /**
+     * Defines how to consume a parsed undefined field
+     */
+    public interface UnknownFieldConsumer<Value> {
+        void accept(Value target, String field, Object value);
+    }
+
+    private static <Value, Context> UnknownFieldParser<Value, Context> consumeUnknownField(UnknownFieldConsumer<Value> consumer) {
+        return (parserName, field, parser, value, context) -> {
+            XContentParser.Token t = parser.currentToken();
+            switch (t) {
+                case VALUE_STRING:
+                    consumer.accept(value, field, parser.text());
+                    break;
+                case VALUE_NUMBER:
+                    consumer.accept(value, field, parser.numberValue());
+                    break;
+                case VALUE_BOOLEAN:
+                    consumer.accept(value, field, parser.booleanValue());
+                    break;
+                case VALUE_NULL:
+                    consumer.accept(value, field, null);
+                    break;
+                case START_OBJECT:
+                    consumer.accept(value, field, parser.map());
+                    break;
+                case START_ARRAY:
+                    consumer.accept(value, field, parser.list());
+                    break;
+                default:
+                    throw new XContentParseException(parser.getTokenLocation(),
+                        "[" + parserName + "] cannot parse field [" + field + "] with value type [" + t + "]");
+            }
+        };
+    }
+
     private final Map<String, FieldParser> fieldParserMap = new HashMap<>();
     private final String name;
     private final Supplier<Value> valueSupplier;
-    /**
-     * Should this parser ignore unknown fields? This should generally be set to true only when parsing responses from external systems,
-     * never when parsing requests from users.
-     */
-    private final boolean ignoreUnknownFields;
+
+    private final UnknownFieldParser<Value, Context> unknownFieldParser;
 
     /**
      * Creates a new ObjectParser instance with a name. This name is used to reference the parser in exceptions and messages.
@@ -95,25 +150,45 @@ public ObjectParser(String name) {
     }
 
     /**
-     * Creates a new ObjectParser instance which a name.
+     * Creates a new ObjectParser instance with a name.
      * @param name the parsers name, used to reference the parser in exceptions and messages.
      * @param valueSupplier a supplier that creates a new Value instance used when the parser is used as an inner object parser.
      */
     public ObjectParser(String name, @Nullable Supplier<Value> valueSupplier) {
-        this(name, false, valueSupplier);
+        this(name, errorOnUnknown(), valueSupplier);
     }
 
     /**
-     * Creates a new ObjectParser instance which a name.
+     * Creates a new ObjectParser instance with a name.
      * @param name the parsers name, used to reference the parser in exceptions and messages.
      * @param ignoreUnknownFields Should this parser ignore unknown fields? This should generally be set to true only when parsing
      *      responses from external systems, never when parsing requests from users.
      * @param valueSupplier a supplier that creates a new Value instance used when the parser is used as an inner object parser.
      */
     public ObjectParser(String name, boolean ignoreUnknownFields, @Nullable Supplier<Value> valueSupplier) {
+        this(name, ignoreUnknownFields ? ignoreUnknown() : errorOnUnknown(), valueSupplier);
+    }
+
+    /**
+     * Creates a new ObjectParser instance with a name.
+     * @param name the parsers name, used to reference the parser in exceptions and messages.
+     * @param unknownFieldConsumer how to consume parsed unknown fields
+     * @param valueSupplier a supplier that creates a new Value instance used when the parser is used as an inner object parser.
+     */
+    public ObjectParser(String name, UnknownFieldConsumer<Value> unknownFieldConsumer, @Nullable Supplier<Value> valueSupplier) {
+        this(name, consumeUnknownField(unknownFieldConsumer), valueSupplier);
+    }
+
+    /**
+     * Creates a new ObjectParser instance with a name.
+     * @param name the parsers name, used to reference the parser in exceptions and messages.
+     * @param unknownFieldParser how to parse unknown fields
+     * @param valueSupplier a supplier that creates a new Value instance used when the parser is used as an inner object parser.
+     */
+    private ObjectParser(String name, UnknownFieldParser<Value, Context> unknownFieldParser, @Nullable Supplier<Value> valueSupplier) {
         this.name = name;
         this.valueSupplier = valueSupplier;
-        this.ignoreUnknownFields = ignoreUnknownFields;
+        this.unknownFieldParser = unknownFieldParser;
     }
 
     /**
@@ -155,14 +230,13 @@ public Value parse(XContentParser parser, Value value, Context context) throws I
         while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
-                fieldParser = getParser(currentFieldName, parser);
+                fieldParser = fieldParserMap.get(currentFieldName);
             } else {
                 if (currentFieldName == null) {
                     throw new XContentParseException(parser.getTokenLocation(), "[" + name  + "] no field found");
                 }
                 if (fieldParser == null) {
-                    assert ignoreUnknownFields : "this should only be possible if configured to ignore known fields";
-                    parser.skipChildren(); // noop if parser points to a value, skips children if parser is start object or start array
+                    unknownFieldParser.acceptUnknownField(name, currentFieldName, parser, value, context);
                 } else {
                     fieldParser.assertSupports(name, parser, currentFieldName);
                     parseSub(parser, fieldParser, currentFieldName, value, context);
@@ -363,15 +437,6 @@ private void parseSub(XContentParser parser, FieldParser fieldParser, String cur
         }
     }
 
-    private FieldParser getParser(String fieldName, XContentParser xContentParser) {
-        FieldParser parser = fieldParserMap.get(fieldName);
-        if (parser == null && false == ignoreUnknownFields) {
-            throw new XContentParseException(xContentParser.getTokenLocation(),
-                    "[" + name + "] unknown field [" + fieldName + "], parser not found");
-        }
-        return parser;
-    }
-
     private class FieldParser {
         private final Parser<Value, Context> parser;
         private final EnumSet<XContentParser.Token> supportedTokens;

libs/x-content/src/test/java/org/elasticsearch/common/xcontent/ObjectParserTests.java

@@ -33,7 +33,9 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.concurrent.atomic.AtomicReference;
 
 import static org.hamcrest.Matchers.containsString;
@@ -203,7 +205,7 @@ public void setTest(int test) {
         {
             XContentParser parser = createParser(JsonXContent.jsonXContent, "{\"not_supported_field\" : \"foo\"}");
             XContentParseException ex = expectThrows(XContentParseException.class, () -> objectParser.parse(parser, s, null));
-            assertEquals(ex.getMessage(), "[1:2] [the_parser] unknown field [not_supported_field], parser not found");
+            assertEquals(ex.getMessage(), "[1:26] [the_parser] unknown field [not_supported_field], parser not found");
         }
     }
 
@@ -733,4 +735,41 @@ public void setFoo(int foo) {
             this.foo = foo;
         }
     }
+
+    private static class ObjectWithArbitraryFields {
+        String name;
+        Map<String, Object> fields = new HashMap<>();
+        void setField(String key, Object value) {
+            fields.put(key, value);
+        }
+        void setName(String name) {
+            this.name = name;
+        }
+    }
+
+    public void testConsumeUnknownFields() throws IOException {
+        XContentParser parser = createParser(JsonXContent.jsonXContent,
+            "{\n"
+                + "  \"test\" : \"foo\",\n"
+                + "  \"test_number\" : 2,\n"
+                + "  \"name\" : \"geoff\",\n"
+                + "  \"test_boolean\" : true,\n"
+                + "  \"test_null\" : null,\n"
+                + "  \"test_array\":  [1,2,3,4],\n"
+                + "  \"test_nested\": { \"field\" : \"value\", \"field2\" : [ \"list1\", \"list2\" ] }\n"
+                + "}");
+        ObjectParser<ObjectWithArbitraryFields, Void> op
+            = new ObjectParser<>("unknown", ObjectWithArbitraryFields::setField, ObjectWithArbitraryFields::new);
+        op.declareString(ObjectWithArbitraryFields::setName, new ParseField("name"));
+
+        ObjectWithArbitraryFields o = op.parse(parser, null);
+        assertEquals("geoff", o.name);
+        assertEquals(6, o.fields.size());
+        assertEquals("foo", o.fields.get("test"));
+        assertEquals(2, o.fields.get("test_number"));
+        assertEquals(true, o.fields.get("test_boolean"));
+        assertNull(o.fields.get("test_null"));
+        assertEquals(List.of(1, 2, 3, 4), o.fields.get("test_array"));
+        assertEquals(Map.of("field", "value", "field2", List.of("list1", "list2")), o.fields.get("test_nested"));
+    }
 }