Support unmapped fields in search 'fields' option (#65386) (#65705)

Currently, the 'fields' option only supports fetching mapped fields. Since
'fields' is meant to be the central place to retrieve document content, it
should allow for loading unmapped values. This change adds implementation and
tests for this feature.

Closes #63690

Author: Christoph Büscher

docs/reference/search/search-your-data/retrieve-selected-fields.asciidoc

@@ -168,7 +168,93 @@ no dedicated array type, and any field could contain multiple values. The
 a specific order. See the mapping documentation on <<array, arrays>> for more
 background.
 
+[discrete]
+[[retrieve-unmapped-fields]]
+==== Retrieving unmapped fields
+
+By default, the `fields` parameter returns only values of mapped fields. However,
+Elasticsearch allows storing fields in `_source` that are unmapped, for example by
+setting <<dynamic-field-mapping,Dynamic field mapping>> to `false` or by using an
+object field with `enabled: false`, thereby disabling parsing and indexing of its content.
 
+Fields in such an object can be retrieved from `_source` using the `include_unmapped` option
+in the `fields` section:
+
+[source,console]
+----
+PUT my-index-000001
+{
+  "mappings": {
+    "enabled": false <1>
+  }
+}
+
+PUT my-index-000001/_doc/1?refresh=true
+{
+  "user_id": "kimchy",
+  "session_data": {
+     "object": {
+       "some_field": "some_value"
+     }
+   }
+}
+
+POST my-index-000001/_search
+{
+  "fields": [
+    "user_id",
+    {
+      "field": "session_data.object.*",
+      "include_unmapped" : true <2>
+    }
+  ],
+  "_source": false
+}
+----
+
+<1> Disable all mappings.
+<2> Include unmapped fields matching this field pattern.
+
+The response will contain fields results under the  `session_data.object.*` path even if the
+fields are unmapped, but will not contain `user_id` since it is unmapped but the `include_unmapped`
+flag hasn't been set to `true` for that field pattern.
+
+[source,console-result]
+----
+{
+  "took" : 2,
+  "timed_out" : false,
+  "_shards" : {
+    "total" : 1,
+    "successful" : 1,
+    "skipped" : 0,
+    "failed" : 0
+  },
+  "hits" : {
+    "total" : {
+      "value" : 1,
+      "relation" : "eq"
+    },
+    "max_score" : 1.0,
+    "hits" : [
+      {
+        "_index" : "my-index-000001",
+        "_id" : "1",
+        "_score" : 1.0,
+        "_type" : "_doc",
+        "fields" : {
+          "session_data.object.some_field": [
+            "some_value"
+          ]
+        }
+      }
+    ]
+  }
+}
+----
+// TESTRESPONSE[s/"took" : 2/"took": $body.took/]
+// TESTRESPONSE[s/"max_score" : 1.0/"max_score" : $body.hits.max_score/]
+// TESTRESPONSE[s/"_score" : 1.0/"_score" : $body.hits.hits.0._score/]
 
 [discrete]
 [[docvalue-fields]]

rest-api-spec/src/main/resources/rest-api-spec/test/search/330_fetch_fields.yml

@@ -295,3 +295,110 @@ setup:
   - is_true: hits.hits.0._id
   - match: { hits.hits.0.fields.count: [2] }
   - is_false: hits.hits.0.fields.count_without_dv
+---
+Test unmapped field:
+  -  skip:
+        version: ' - 7.10.99'
+        reason: support was introduced in 7.11
+  -  do:
+        indices.create:
+           index: test
+           body:
+              mappings:
+                 dynamic: false
+                 properties:
+                    f1:
+                       type: keyword
+                    f2:
+                       type: object
+                       enabled: false
+                    f3:
+                       type: object
+  -  do:
+        index:
+           index: test
+           id: 1
+           refresh: true
+           body:
+              f1: some text
+              f2:
+                 a: foo
+                 b: bar
+              f3:
+                 c: baz
+              f4: some other text
+  -  do:
+        search:
+           index: test
+           body:
+              fields:
+              - f1
+              - { "field" : "f4", "include_unmapped" : true }
+  -  match:
+        hits.hits.0.fields.f1: 
+        - some text
+  -  match:
+        hits.hits.0.fields.f4:
+        - some other text
+  -  do:
+        search:
+           index: test
+           body:
+              fields:
+              - { "field" : "f*", "include_unmapped" : true }
+  -  match:
+        hits.hits.0.fields.f1: 
+        - some text
+  -  match:
+        hits.hits.0.fields.f2\.a:
+        - foo
+  -  match:
+        hits.hits.0.fields.f2\.b:
+        - bar
+  -  match:
+        hits.hits.0.fields.f3\.c:
+        - baz
+  -  match:
+        hits.hits.0.fields.f4:
+        - some other text
+---
+Test unmapped fields inside disabled objects:
+  -  skip:
+        version: ' - 7.10.99'
+        reason: support was introduced in 7.11
+  -  do:
+        indices.create:
+           index: test
+           body:
+              mappings:
+                 properties:
+                    f1:
+                       type: object
+                       enabled: false
+  -  do:
+        index:
+           index: test
+           id: 1
+           refresh: true
+           body:
+              f1: 
+                 - some text
+                 - a: b
+                 -
+                   - 1
+                   - 2
+                   - 3
+  -  do:
+        search:
+           index: test
+           body:
+              fields: [ { "field" : "*", "include_unmapped" : true } ]
+  -  match:
+        hits.hits.0.fields.f1:
+        - 1
+        - 2
+        - 3
+        - some text
+  -  match:
+        hits.hits.0.fields.f1\.a:
+        - b

server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java

@@ -138,7 +138,7 @@ private SearchSourceBuilder buildExpandSearchSourceBuilder(InnerHitBuilder optio
             }
         }
         if (options.getFetchFields() != null) {
-            options.getFetchFields().forEach(ff -> groupSource.fetchField(ff.field, ff.format));
+            options.getFetchFields().forEach(ff -> groupSource.fetchField(ff));
         }
         if (options.getDocValueFields() != null) {
             options.getDocValueFields().forEach(ff -> groupSource.docValueField(ff.field, ff.format));

server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java

@@ -32,6 +32,7 @@
 import org.elasticsearch.search.builder.PointInTimeBuilder;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.collapse.CollapseBuilder;
+import org.elasticsearch.search.fetch.subphase.FieldAndFormat;
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
 import org.elasticsearch.search.rescore.RescorerBuilder;
 import org.elasticsearch.search.slice.SliceBuilder;
@@ -321,18 +322,18 @@ public SearchRequestBuilder addDocValueField(String name) {
      * @param name The field to load
      */
     public SearchRequestBuilder addFetchField(String name) {
-        sourceBuilder().fetchField(name, null);
+        sourceBuilder().fetchField(new FieldAndFormat(name, null, null));
         return this;
     }
 
     /**
      * Adds a field to load and return. The field must be present in the document _source.
      *
-     * @param name The field to load
-     * @param format an optional format string used when formatting values, for example a date format.
+     * @param fetchField a {@link FieldAndFormat} specifying the field pattern, optional format (for example a date format) and
+     * whether this field pattern sould also include unmapped fields
      */
-    public SearchRequestBuilder addFetchField(String name, String format) {
-        sourceBuilder().fetchField(name, format);
+    public SearchRequestBuilder addFetchField(FieldAndFormat fetchField) {
+        sourceBuilder().fetchField(fetchField);
         return this;
     }
 

server/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java

@@ -424,10 +424,20 @@ public InnerHitBuilder addFetchField(String name) {
      * @param format an optional format string used when formatting values, for example a date format.
      */
     public InnerHitBuilder addFetchField(String name, @Nullable String format) {
+        return addFetchField(name, format, null);
+    }
+
+    /**
+     * Adds a field to load and return as part of the search request.
+     * @param name the field name.
+     * @param format an optional format string used when formatting values, for example a date format.
+     * @param includeUnmapped whether unmapped fields should be returned as well
+     */
+    public InnerHitBuilder addFetchField(String name, @Nullable String format, Boolean includeUnmapped) {
         if (fetchFields == null || fetchFields.isEmpty()) {
             fetchFields = new ArrayList<>();
         }
-        fetchFields.add(new FieldAndFormat(name, format));
+        fetchFields.add(new FieldAndFormat(name, format, includeUnmapped));
         return this;
     }
 

server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregationBuilder.java

@@ -452,22 +452,22 @@ public List<FieldAndFormat> docValueFields() {
     /**
      * Adds a field to load and return as part of the search request.
      */
-    public TopHitsAggregationBuilder fetchField(String field, String format) {
-        if (field == null) {
+    public TopHitsAggregationBuilder fetchField(FieldAndFormat fieldAndFormat) {
+        if (fieldAndFormat == null) {
             throw new IllegalArgumentException("[fields] must not be null: [" + name + "]");
         }
         if (fetchFields == null) {
             fetchFields = new ArrayList<>();
         }
-        fetchFields.add(new FieldAndFormat(field, format));
+        fetchFields.add(fieldAndFormat);
         return this;
     }
 
     /**
      * Adds a field to load and return as part of the search request.
      */
     public TopHitsAggregationBuilder fetchField(String field) {
-        return fetchField(field, null);
+        return fetchField(new FieldAndFormat(field, null, null));
     }
 
     /**
@@ -802,7 +802,7 @@ public static TopHitsAggregationBuilder parse(String aggregationName, XContentPa
                 } else if (SearchSourceBuilder.FETCH_FIELDS_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         FieldAndFormat ff = FieldAndFormat.fromXContent(parser);
-                        factory.fetchField(ff.field, ff.format);
+                        factory.fetchField(ff);
                     }
                 } else if (SearchSourceBuilder.SORT_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
                     List<SortBuilder<?>> sorts = SortBuilder.fromXContent(parser);

server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java

@@ -904,19 +904,18 @@ public List<FieldAndFormat> fetchFields() {
      * Adds a field to load and return as part of the search request.
      */
     public SearchSourceBuilder fetchField(String name) {
-        return fetchField(name, null);
+        return fetchField(new FieldAndFormat(name, null, null));
     }
 
     /**
      * Adds a field to load and return as part of the search request.
-     * @param name the field name.
-     * @param format an optional format string used when formatting values, for example a date format.
+     * @param fetchField defining the field name, optional format and optional inclusion of unmapped fields
      */
-    public SearchSourceBuilder fetchField(String name, @Nullable String format) {
+    public SearchSourceBuilder fetchField(FieldAndFormat fetchField) {
         if (fetchFields == null) {
             fetchFields = new ArrayList<>();
         }
-        fetchFields.add(new FieldAndFormat(name, format));
+        fetchFields.add(fetchField);
         return this;
     }
 

server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesContext.java

@@ -45,7 +45,7 @@ public FetchDocValuesContext(QueryShardContext shardContext, List<FieldAndFormat
             Collection<String> fieldNames = shardContext.simpleMatchToIndexNames(field.field);
             for (String fieldName : fieldNames) {
                 if (shardContext.isFieldMapped(fieldName)) {
-                    fields.add(new FieldAndFormat(fieldName, field.format));
+                    fields.add(new FieldAndFormat(fieldName, field.format, field.includeUnmapped));
                 }
             }
         }

server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhase.java

@@ -54,6 +54,7 @@ public FetchSubPhaseProcessor getProcessor(FetchContext fetchContext) {
         }
 
         FieldFetcher fieldFetcher = FieldFetcher.create(fetchContext.getQueryShardContext(), searchLookup, fetchFieldsContext.fields());
+
         return new FetchSubPhaseProcessor() {
             @Override
             public void setNextReader(LeafReaderContext readerContext) {