Add support for dots in field names for metrics usecases

docs/changelog/86166.yaml

@@ -0,0 +1,48 @@
+pr: 86166
+summary: Add support for dots in field names for metrics usecases
+area: Mapping
+type: feature
+issues:
+ - 63530
+highlight:
+  title: Add support for dots in field names for metrics usecases
+  body: |-
+    Metrics data can often be made of several fields with dots in their names,
+    sharing common prefixes, like in the following example:
+
+    ```
+    {
+      "metrics.time" : 10,
+      "metrics.time.min" : 1,
+      "metrics.time.max" : 500
+    }
+    ```
+
+    Such format causes a mapping conflict as the `metrics.time` holds a value,
+    but it also needs to be mapped as an object in order to hold the `min` and
+    `max` leaf fields.
+
+    A new object mapping parameter called `subobjects`, which defaults to `true`,
+    has been introduced to preserve dots in field names. An object with `subobjects`
+    set to `false` can only ever hold leaf sub-fields and no further objects. The
+    following example shows how it can be configured in the mappings for the
+    `metrics` object:
+
+    ```
+    {
+      "mappings": {
+        "properties" : {
+          "metrics" : {
+            "type" : "object",
+            "subobjects" : false
+          }
+        }
+      }
+    }
+    ```
+
+    With this configuration any child of `metrics` will be mapped unchanged,
+    without expanding dots in field names to the corresponding object structure.
+    That makes it possible to store the metrics document above.
+
+  notable: true

docs/reference/mapping/params.asciidoc

@@ -31,6 +31,7 @@ The following mapping parameters are common to some or all field data types:
 * <<properties,`properties`>>
 * <<search-analyzer,`search_analyzer`>>
 * <<similarity,`similarity`>>
+* <<subobjects,`subobjects`>>
 * <<mapping-store,`store`>>
 * <<term-vector,`term_vector`>>
 
@@ -83,4 +84,6 @@ include::params/similarity.asciidoc[]
 
 include::params/store.asciidoc[]
 
+include::params/subobjects.asciidoc[]
+
 include::params/term-vector.asciidoc[]

docs/reference/mapping/params/subobjects.asciidoc

@@ -0,0 +1,107 @@
+[[subobjects]]
+=== `subobjects`
+
+When indexing a document or updating mappings, Elasticsearch accepts fields that contain dots in their names,
+which get expanded to their corresponding object structure. For instance, the  field `metrics.time.max`
+is mapped as a `max` leaf field with a parent `time` object, belonging to its parent `metrics` object.
+
+The described default behaviour is reasonable for most scenarios, but causes problems in certain situations
+where for instance a field `metrics.time` holds a value too, which is common when indexing metrics data.
+A document holding a value for both `metrics.time.max` and `metrics.time` gets rejected given that `time`
+would need to be a leaf field to hold a value as well as an object to hold the `max` sub-field.
+
+The `subobjects` setting, which can be applied only to the top-level mapping definition and
+to <<object,`object`>> fields, disables the ability for an object to hold further subobjects and makes it possible
+to store documents where field names contain dots and share common prefixes. From the example above, if the object
+container `metrics` has `subobjects` set to `false`, it can hold values for both `time` and `time.max` directly
+without the need for any intermediate object, as dots in field names are preserved.
+
+[source,console]
+--------------------------------------------------
+PUT my-index-000001
+{
+  "mappings": {
+    "properties": {
+      "metrics": {
+        "type":  "object",
+        "subobjects": false <1>
+      }
+    }
+  }
+}
+
+PUT my-index-000001/_doc/metric_1
+{
+  "metrics.time" : 100, <2>
+  "metrics.time.min" : 10,
+  "metrics.time.max" : 900
+}
+
+PUT my-index-000001/_doc/metric_2
+{
+  "metrics" : {
+    "time" : 100, <3>
+    "time.min" : 10,
+    "time.max" : 900
+  }
+}
+
+GET my-index-000001/_mapping
+--------------------------------------------------
+
+[source,console-result]
+--------------------------------------------------
+{
+  "my-index-000001" : {
+    "mappings" : {
+      "properties" : {
+        "metrics" : {
+          "subobjects" : false,
+          "properties" : {
+            "time" : {
+              "type" : "long"
+            },
+            "time.min" : { <4>
+              "type" : "long"
+            },
+            "time.max" : {
+              "type" : "long"
+            }
+          }
+        }
+      }
+    }
+  }
+}
+--------------------------------------------------
+
+<1> The `metrics` field cannot hold other objects.
+<2> Sample document holding flat paths
+<3> Sample document holding an object (configured to not hold subobjects) and its leaf sub-fields
+<4> The resulting mapping where dots in field names were preserved
+
+The entire mapping may be configured to not support subobjects as well, in which case the document can
+only ever hold leaf sub-fields:
+
+[source,console]
+--------------------------------------------------
+PUT my-index-000001
+{
+  "mappings": {
+    "subobjects": false <1>
+  }
+}
+
+PUT my-index-000001/_doc/metric_1
+{
+  "time" : "100ms", <2>
+  "time.min" : "10ms",
+  "time.max" : "900ms"
+}
+
+--------------------------------------------------
+
+<1> The entire mapping is configured to not support objects.
+<2> The document does not support objects
+
+The `subobjects` setting for existing fields and the top-level mapping definition cannot be updated.

docs/reference/mapping/types/object.asciidoc

@@ -90,6 +90,12 @@ The following parameters are accepted by `object` fields:
     Whether the JSON value given for the object field should be
     parsed and indexed (`true`, default) or completely ignored (`false`).
 
+<<subobjects,`subobjects`>>::
+
+    Whether the object can hold subobjects (`true`, default) or not (`false`). If not, sub-fields
+    with dots in their names will be treated as leaves instead, otherwise their field names
+    would be expanded to their corresponding object structure.
+
 <<properties,`properties`>>::
 
     The fields within the object, which can be of any

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/index/91_metrics_no_subobjects.yml

@@ -0,0 +1,37 @@
+---
+"Metrics indexing":
+  - skip:
+      version: " - 8.2.99"
+      reason: added in 8.3.0
+
+  - do:
+      indices.put_template:
+        name: test
+        body:
+          index_patterns: test-*
+          mappings:
+            dynamic_templates:
+              - no_subobjects:
+                  match: metrics
+                  mapping:
+                    type: object
+                    subobjects: false
+
+  - do:
+      index:
+        index: test-1
+        id: 1
+        refresh: true
+        body:
+          { metrics.time: 10, metrics.time.max: 100, metrics.time.min: 1 }
+
+  - do:
+      field_caps:
+        index: test-1
+        fields: metrics.time*
+  - match: {fields.metrics\.time.long.searchable: true}
+  - match: {fields.metrics\.time.long.aggregatable: true}
+  - match: {fields.metrics\.time\.max.long.searchable: true}
+  - match: {fields.metrics\.time\.max.long.aggregatable: true}
+  - match: {fields.metrics\.time\.min.long.searchable: true}
+  - match: {fields.metrics\.time\.min.long.aggregatable: true}

server/src/main/java/org/elasticsearch/index/mapper/ContentPath.java

@@ -20,6 +20,8 @@ public final class ContentPath {
 
     private String[] path = new String[10];
 
+    private boolean withinLeafObject = false;
+
     public ContentPath() {
         this(0);
     }
@@ -54,6 +56,14 @@ public void remove() {
         path[index--] = null;
     }
 
+    public void setWithinLeafObject(boolean withinLeafObject) {
+        this.withinLeafObject = withinLeafObject;
+    }
+
+    public boolean isWithinLeafObject() {
+        return withinLeafObject;
+    }
+
     public String pathAsText(String name) {
         sb.setLength(0);
         for (int i = offset; i < index; i++) {

server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java

@@ -445,7 +445,13 @@ private static void parseObject(final DocumentParserContext context, ObjectMappe
         Mapper objectMapper = getMapper(context, mapper, currentFieldName);
         if (objectMapper != null) {
             context.path().add(currentFieldName);
+            if (objectMapper instanceof ObjectMapper objMapper) {
+                if (objMapper.subobjects() == false) {
+                    context.path().setWithinLeafObject(true);
+                }
+            }
             parseObjectOrField(context, objectMapper);
+            context.path().setWithinLeafObject(false);
             context.path().remove();
         } else {
             parseObjectDynamic(context, mapper, currentFieldName);
@@ -474,7 +480,13 @@ private static void parseObjectDynamic(DocumentParserContext context, ObjectMapp
                 throwOnCreateDynamicNestedViaCopyTo(dynamicObjectMapper);
             }
             context.path().add(currentFieldName);
+            if (dynamicObjectMapper instanceof ObjectMapper objectMapper) {
+                if (objectMapper.subobjects() == false) {
+                    context.path().setWithinLeafObject(true);
+                }
+            }
             parseObjectOrField(context, dynamicObjectMapper);
+            context.path().setWithinLeafObject(false);
             context.path().remove();
         }
     }
@@ -789,7 +801,7 @@ protected String contentType() {
 
     private static class NoOpObjectMapper extends ObjectMapper {
         NoOpObjectMapper(String name, String fullPath) {
-            super(name, fullPath, Explicit.IMPLICIT_TRUE, Dynamic.RUNTIME, Collections.emptyMap());
+            super(name, fullPath, Explicit.IMPLICIT_TRUE, Explicit.IMPLICIT_TRUE, Dynamic.RUNTIME, Collections.emptyMap());
         }
     }
 
@@ -815,7 +827,11 @@ private static class InternalDocumentParserContext extends DocumentParserContext
             XContentParser parser
         ) throws IOException {
             super(mappingLookup, indexSettings, indexAnalyzers, parserContext, source);
-            this.parser = DotExpandingXContentParser.expandDots(parser);
+            if (mappingLookup.getMapping().getRoot().subobjects()) {
+                this.parser = DotExpandingXContentParser.expandDots(parser, this.path::isWithinLeafObject);
+            } else {
+                this.parser = parser;
+            }
             this.document = new LuceneDocument();
             this.documents.add(document);
             this.maxAllowedNumNestedDocs = indexSettings().getMappingNestedDocsLimit();

server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java

@@ -314,7 +314,7 @@ public LuceneDocument doc() {
      */
     public final DocumentParserContext createCopyToContext(String copyToField, LuceneDocument doc) throws IOException {
         ContentPath path = new ContentPath(0);
-        XContentParser parser = DotExpandingXContentParser.expandDots(new CopyToParser(copyToField, parser()));
+        XContentParser parser = DotExpandingXContentParser.expandDots(new CopyToParser(copyToField, parser()), path::isWithinLeafObject);
         return new Wrapper(this) {
             @Override
             public ContentPath path() {