Support unmapped fields in search 'fields' option

docs/reference/search/search-your-data/retrieve-selected-fields.asciidoc

@@ -167,7 +167,92 @@ no dedicated array type, and any field could contain multiple values. The
 a specific order. See the mapping documentation on <<array, arrays>> for more
 background.
 
+[discrete]
+[[retrieve-unmapped-fields]]
+==== Retrieving unmapped fields
+
+By default, the `fields` parameter returns only values of mapped fields. However,
+Elasticsearch allows storing fields in `_source` that are unmapped, for example by
+setting <<dynamic-field-mapping,Dynamic field mapping>> to `false` or by using an
+object field with `enabled: false`, thereby disabling parsing and indexing of its content.
+
+Fields in such an object can be retrieved from `_source` using the `include_unmapped` option
+in the `fields` section:
+
+[source,console]
+----
+PUT my-index-000001
+{
+  "mappings": {
+    "enabled": false <1>
+  }
+}
+
+PUT my-index-000001/_doc/1?refresh=true
+{
+  "user_id": "kimchy",
+  "session_data": {
+     "object": {
+       "some_field": "some_value"
+     }
+   }
+}
+
+POST my-index-000001/_search
+{
+  "fields": [
+    "user_id",
+    {
+      "field": "session_data.object.*",
+      "include_unmapped" : true <2>
+    }
+  ],
+  "_source": false
+}
+----
+
+<1> Disable all mappings.
+<2> Include unmapped fields matching this field pattern.
 
+The response will contain fields results under the  `session_data.object.*` path even if the
+fields are unmapped, but will not contain `user_id` since it is unmapped but the `include_unmapped`
+flag hasn't been set to `true` for that field pattern.
+
+[source,console-result]
+----
+{
+  "took" : 2,
+  "timed_out" : false,
+  "_shards" : {
+    "total" : 1,
+    "successful" : 1,
+    "skipped" : 0,
+    "failed" : 0
+  },
+  "hits" : {
+    "total" : {
+      "value" : 1,
+      "relation" : "eq"
+    },
+    "max_score" : 1.0,
+    "hits" : [
+      {
+        "_index" : "my-index-000001",
+        "_id" : "1",
+        "_score" : 1.0,
+        "fields" : {
+          "session_data.object.some_field": [
+            "some_value"
+          ]
+        }
+      }
+    ]
+  }
+}
+----
+// TESTRESPONSE[s/"took" : 2/"took": $body.took/]
+// TESTRESPONSE[s/"max_score" : 1.0/"max_score" : $body.hits.max_score/]
+// TESTRESPONSE[s/"_score" : 1.0/"_score" : $body.hits.hits.0._score/]
 
 [discrete]
 [[docvalue-fields]]

rest-api-spec/src/main/resources/rest-api-spec/test/search/330_fetch_fields.yml

@@ -295,3 +295,110 @@ setup:
   - is_true: hits.hits.0._id
   - match: { hits.hits.0.fields.count: [2] }
   - is_false: hits.hits.0.fields.count_without_dv
+---
+Test unmapped field:
+  -  skip:
+        version: ' - 7.99.99'
+        reason: support isn't yet backported
+  -  do:
+        indices.create:
+           index: test
+           body:
+              mappings:
+                 dynamic: false
+                 properties:
+                    f1:
+                       type: keyword
+                    f2:
+                       type: object
+                       enabled: false
+                    f3:
+                       type: object
+  -  do:
+        index:
+           index: test
+           id: 1
+           refresh: true
+           body:
+              f1: some text
+              f2:
+                 a: foo
+                 b: bar
+              f3:
+                 c: baz
+              f4: some other text
+  -  do:
+        search:
+           index: test
+           body:
+              fields:
+              - f1
+              - { "field" : "f4", "include_unmapped" : true }
+  -  match:
+        hits.hits.0.fields.f1: 
+        - some text
+  -  match:
+        hits.hits.0.fields.f4:
+        - some other text
+  -  do:
+        search:
+           index: test
+           body:
+              fields:
+              - { "field" : "f*", "include_unmapped" : true }
+  -  match:
+        hits.hits.0.fields.f1: 
+        - some text
+  -  match:
+        hits.hits.0.fields.f2\.a:
+        - foo
+  -  match:
+        hits.hits.0.fields.f2\.b:
+        - bar
+  -  match:
+        hits.hits.0.fields.f3\.c:
+        - baz
+  -  match:
+        hits.hits.0.fields.f4:
+        - some other text
+---
+Test unmapped fields inside disabled objects:
+  -  skip:
+        version: ' - 7.99.99'
+        reason: support isn't yet backported
+  -  do:
+        indices.create:
+           index: test
+           body:
+              mappings:
+                 properties:
+                    f1:
+                       type: object
+                       enabled: false
+  -  do:
+        index:
+           index: test
+           id: 1
+           refresh: true
+           body:
+              f1: 
+                 - some text
+                 - a: b
+                 -
+                   - 1
+                   - 2
+                   - 3
+  -  do:
+        search:
+           index: test
+           body:
+              fields: [ { "field" : "*", "include_unmapped" : true } ]
+  -  match:
+        hits.hits.0.fields.f1:
+        - 1
+        - 2
+        - 3
+        - some text
+  -  match:
+        hits.hits.0.fields.f1\.a:
+        - b

server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java

@@ -138,7 +138,7 @@ private SearchSourceBuilder buildExpandSearchSourceBuilder(InnerHitBuilder optio
             }
         }
         if (options.getFetchFields() != null) {
-            options.getFetchFields().forEach(ff -> groupSource.fetchField(ff.field, ff.format));
+            options.getFetchFields().forEach(ff -> groupSource.fetchField(ff));
         }
         if (options.getDocValueFields() != null) {
             options.getDocValueFields().forEach(ff -> groupSource.docValueField(ff.field, ff.format));

server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java

@@ -32,6 +32,7 @@
 import org.elasticsearch.search.builder.PointInTimeBuilder;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.collapse.CollapseBuilder;
+import org.elasticsearch.search.fetch.subphase.FieldAndFormat;
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
 import org.elasticsearch.search.rescore.RescorerBuilder;
 import org.elasticsearch.search.slice.SliceBuilder;
@@ -310,18 +311,18 @@ public SearchRequestBuilder addDocValueField(String name) {
      * @param name The field to load
      */
     public SearchRequestBuilder addFetchField(String name) {
-        sourceBuilder().fetchField(name, null);
+        sourceBuilder().fetchField(new FieldAndFormat(name, null, null));
         return this;
     }
 
     /**
      * Adds a field to load and return. The field must be present in the document _source.
      *
-     * @param name The field to load
-     * @param format an optional format string used when formatting values, for example a date format.
+     * @param fetchField a {@link FieldAndFormat} specifying the field pattern, optional format (for example a date format) and
+     * whether this field pattern sould also include unmapped fields
      */
-    public SearchRequestBuilder addFetchField(String name, String format) {
-        sourceBuilder().fetchField(name, format);
+    public SearchRequestBuilder addFetchField(FieldAndFormat fetchField) {
+        sourceBuilder().fetchField(fetchField);
         return this;
     }
 

server/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java

@@ -395,10 +395,20 @@ public InnerHitBuilder addFetchField(String name) {
      * @param format an optional format string used when formatting values, for example a date format.
      */
     public InnerHitBuilder addFetchField(String name, @Nullable String format) {
+        return addFetchField(name, format, null);
+    }
+
+    /**
+     * Adds a field to load and return as part of the search request.
+     * @param name the field name.
+     * @param format an optional format string used when formatting values, for example a date format.
+     * @param includeUnmapped whether unmapped fields should be returned as well
+     */
+    public InnerHitBuilder addFetchField(String name, @Nullable String format, Boolean includeUnmapped) {
         if (fetchFields == null || fetchFields.isEmpty()) {
             fetchFields = new ArrayList<>();
         }
-        fetchFields.add(new FieldAndFormat(name, format));
+        fetchFields.add(new FieldAndFormat(name, format, includeUnmapped));
         return this;
     }
 

server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregationBuilder.java

@@ -446,22 +446,22 @@ public List<FieldAndFormat> docValueFields() {
     /**
      * Adds a field to load and return as part of the search request.
      */
-    public TopHitsAggregationBuilder fetchField(String field, String format) {
-        if (field == null) {
+    public TopHitsAggregationBuilder fetchField(FieldAndFormat fieldAndFormat) {
+        if (fieldAndFormat == null) {
             throw new IllegalArgumentException("[fields] must not be null: [" + name + "]");
         }
         if (fetchFields == null) {
             fetchFields = new ArrayList<>();
         }
-        fetchFields.add(new FieldAndFormat(field, format));
+        fetchFields.add(fieldAndFormat);
         return this;
     }
 
     /**
      * Adds a field to load and return as part of the search request.
      */
     public TopHitsAggregationBuilder fetchField(String field) {
-        return fetchField(field, null);
+        return fetchField(new FieldAndFormat(field, null, null));
     }
 
     /**
@@ -796,7 +796,7 @@ public static TopHitsAggregationBuilder parse(String aggregationName, XContentPa
                 } else if (SearchSourceBuilder.FETCH_FIELDS_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         FieldAndFormat ff = FieldAndFormat.fromXContent(parser);
-                        factory.fetchField(ff.field, ff.format);
+                        factory.fetchField(ff);
                     }
                 } else if (SearchSourceBuilder.SORT_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
                     List<SortBuilder<?>> sorts = SortBuilder.fromXContent(parser);

server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java

@@ -868,19 +868,18 @@ public List<FieldAndFormat> fetchFields() {
      * Adds a field to load and return as part of the search request.
      */
     public SearchSourceBuilder fetchField(String name) {
-        return fetchField(name, null);
+        return fetchField(new FieldAndFormat(name, null, null));
     }
 
     /**
      * Adds a field to load and return as part of the search request.
-     * @param name the field name.
-     * @param format an optional format string used when formatting values, for example a date format.
+     * @param fetchField defining the field name, optional format and optional inclusion of unmapped fields
      */
-    public SearchSourceBuilder fetchField(String name, @Nullable String format) {
+    public SearchSourceBuilder fetchField(FieldAndFormat fetchField) {
         if (fetchFields == null) {
             fetchFields = new ArrayList<>();
         }
-        fetchFields.add(new FieldAndFormat(name, format));
+        fetchFields.add(fetchField);
         return this;
     }
 

server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesContext.java

@@ -45,7 +45,7 @@ public FetchDocValuesContext(QueryShardContext shardContext, List<FieldAndFormat
             Collection<String> fieldNames = shardContext.simpleMatchToIndexNames(field.field);
             for (String fieldName : fieldNames) {
                 if (shardContext.isFieldMapped(fieldName)) {
-                    fields.add(new FieldAndFormat(fieldName, field.format));
+                    fields.add(new FieldAndFormat(fieldName, field.format, field.includeUnmapped));
                 }
             }
         }

server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhase.java

@@ -54,6 +54,7 @@ public FetchSubPhaseProcessor getProcessor(FetchContext fetchContext) {
         }
 
         FieldFetcher fieldFetcher = FieldFetcher.create(fetchContext.getQueryShardContext(), searchLookup, fetchFieldsContext.fields());
+
         return new FetchSubPhaseProcessor() {
             @Override
             public void setNextReader(LeafReaderContext readerContext) {