From f11a5e8641b9859b40614d501e8d5806a16b6e6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Thu, 25 Mar 2021 18:13:19 +0100 Subject: [PATCH 1/7] Support fetching flattened subfields Currently the `fields` API fetches the root flattened field and returns it in a structured way in the response. In addition this change makes it possible to directly query subfields. However, requesting flattened subfields via wildcard patterns is not possible. Closes #70605 --- .../index/mapper/FieldTypeLookup.java | 4 ++ .../flattened/FlattenedFieldMapper.java | 13 +++-- .../KeyedFlattenedFieldTypeTests.java | 4 +- .../fetch/subphase/FieldFetcherTests.java | 58 +++++++++++++++++++ .../search/lookup/LeafDocLookupTests.java | 5 +- 5 files changed, 74 insertions(+), 10 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java index e256bc9deff40..6c3d07aaeff56 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java @@ -132,6 +132,10 @@ Set sourcePaths(String field) { if (fullNameToFieldType.isEmpty()) { return Set.of(); } + if (dynamicKeyLookup.get(field) != null) { + return Set.of(field); + } + String resolvedField = field; int lastDotIndex = field.lastIndexOf('.'); if (lastDotIndex > 0) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java index 4b72755ab091a..8c0a07f8f4f35 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java @@ -163,17 +163,19 @@ public FlattenedFieldMapper build(ContentPath contentPath) { */ public static final class KeyedFlattenedFieldType extends StringFieldType { private final String key; + private final String rootName; - public KeyedFlattenedFieldType(String name, boolean indexed, boolean hasDocValues, String key, + public KeyedFlattenedFieldType(String name, String rootName, boolean indexed, boolean hasDocValues, String key, boolean splitQueriesOnWhitespace, Map meta) { super(name, indexed, false, hasDocValues, splitQueriesOnWhitespace ? TextSearchInfo.WHITESPACE_MATCH_ONLY : TextSearchInfo.SIMPLE_MATCH_ONLY, meta); this.key = key; + this.rootName = rootName; } - private KeyedFlattenedFieldType(String name, String key, RootFlattenedFieldType ref) { - this(name, ref.isSearchable(), ref.hasDocValues(), key, ref.splitQueriesOnWhitespace, ref.meta()); + private KeyedFlattenedFieldType(String name, String rootName, String key, RootFlattenedFieldType ref) { + this(name, rootName, ref.isSearchable(), ref.hasDocValues(), key, ref.splitQueriesOnWhitespace, ref.meta()); } @Override @@ -259,8 +261,7 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S @Override public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { - // This is an internal field but it can match a field pattern so we return an empty list. - return lookup -> List.of(); + return SourceValueFetcher.identity(rootName + "." + key, context, format); } } @@ -441,7 +442,7 @@ public RootFlattenedFieldType fieldType() { @Override public KeyedFlattenedFieldType keyedFieldType(String key) { - return new KeyedFlattenedFieldType(keyedFieldName(), key, fieldType()); + return new KeyedFlattenedFieldType(keyedFieldName(), name(), key, fieldType()); } public String keyedFieldName() { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedFieldTypeTests.java index d5c8f585705ab..7f14ba5656b20 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedFieldTypeTests.java @@ -31,7 +31,7 @@ public class KeyedFlattenedFieldTypeTests extends FieldTypeTestCase { private static KeyedFlattenedFieldType createFieldType() { - return new KeyedFlattenedFieldType("field", true, true, "key", false, Collections.emptyMap()); + return new KeyedFlattenedFieldType("field", "field", true, true, "key", false, Collections.emptyMap()); } public void testIndexedValueForSearch() { @@ -56,7 +56,7 @@ public void testTermQuery() { expected = AutomatonQueries.caseInsensitiveTermQuery(new Term("field", "key\0value")); assertEquals(expected, ft.termQueryCaseInsensitive("value", null)); - KeyedFlattenedFieldType unsearchable = new KeyedFlattenedFieldType("field", false, true, "key", + KeyedFlattenedFieldType unsearchable = new KeyedFlattenedFieldType("field", "field", false, true, "key", false, Collections.emptyMap()); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> unsearchable.termQuery("field", null)); diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java index 4c1d8b6bb85cf..79e6a39a40007 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java @@ -669,6 +669,64 @@ public void testNestedFields() throws IOException { assertEquals("value4b", eval("inner_nested.0.f4.0", obj1)); } + @SuppressWarnings("unchecked") + public void testFlattenedField() throws IOException { + XContentBuilder mapping = XContentFactory.jsonBuilder().startObject() + .startObject("_doc") + .startObject("properties") + .startObject("flat") + .field("type", "flattened") + .endObject() + .endObject() + .endObject() + .endObject(); + + MapperService mapperService = createMapperService(mapping); + + XContentBuilder source = XContentFactory.jsonBuilder().startObject() + .startObject("flat") + .field("f1", "value1") + .field("f2", 1) + .endObject() + .endObject(); + + // requesting via wildcard should retrieve the root field as a structured map + Map fields = fetchFields(mapperService, source, fieldAndFormatList("*", null, false)); + assertEquals(1, fields.size()); + assertThat(fields.keySet(), containsInAnyOrder("flat")); + Map flattendedValue = (Map) fields.get("flat").getValue(); + assertThat(flattendedValue.keySet(), containsInAnyOrder("f1", "f2")); + assertEquals("value1", flattendedValue.get("f1")); + assertEquals(1, flattendedValue.get("f2")); + + // direct retrieval of subfield is possible + List fieldAndFormatList = new ArrayList<>(); + fieldAndFormatList.add(new FieldAndFormat("flat.f1", null)); + fields = fetchFields(mapperService, source, fieldAndFormatList); + assertEquals(1, fields.size()); + assertThat(fields.keySet(), containsInAnyOrder("flat.f1")); + assertThat(fields.get("flat.f1").getValue(), equalTo("value1")); + + // direct retrieval of root field and subfield is possible + fieldAndFormatList.add(new FieldAndFormat("*", null)); + fields = fetchFields(mapperService, source, fieldAndFormatList); + assertEquals(2, fields.size()); + assertThat(fields.keySet(), containsInAnyOrder("flat", "flat.f1")); + flattendedValue = (Map) fields.get("flat").getValue(); + assertThat(flattendedValue.keySet(), containsInAnyOrder("f1", "f2")); + assertEquals("value1", flattendedValue.get("f1")); + assertEquals(1, flattendedValue.get("f2")); + assertThat(fields.get("flat.f1").getValue(), equalTo("value1")); + + // retrieval of subfield with widlcard is not possible + fields = fetchFields(mapperService, source, fieldAndFormatList("flat.f*", null, false)); + assertEquals(0, fields.size()); + + // retrieval of non-existing subfield returns empty result + fields = fetchFields(mapperService, source, fieldAndFormatList("flat.baz", null, false)); + assertEquals(0, fields.size()); + } + public void testUnmappedFieldsInsideObject() throws IOException { XContentBuilder mapping = XContentFactory.jsonBuilder().startObject() .startObject("_doc") diff --git a/server/src/test/java/org/elasticsearch/search/lookup/LeafDocLookupTests.java b/server/src/test/java/org/elasticsearch/search/lookup/LeafDocLookupTests.java index c3df44c795e30..381fb60b5d665 100644 --- a/server/src/test/java/org/elasticsearch/search/lookup/LeafDocLookupTests.java +++ b/server/src/test/java/org/elasticsearch/search/lookup/LeafDocLookupTests.java @@ -28,6 +28,7 @@ public class LeafDocLookupTests extends ESTestCase { private ScriptDocValues docValues; private LeafDocLookup docLookup; + @Override @Before public void setUp() throws Exception { super.setUp(); @@ -61,9 +62,9 @@ public void testFlattenedField() { IndexFieldData fieldData2 = createFieldData(docValues2); FlattenedFieldMapper.KeyedFlattenedFieldType fieldType1 - = new FlattenedFieldMapper.KeyedFlattenedFieldType("field", true, true, "key1", false, Collections.emptyMap()); + = new FlattenedFieldMapper.KeyedFlattenedFieldType("field", "field", true, true, "key1", false, Collections.emptyMap()); FlattenedFieldMapper.KeyedFlattenedFieldType fieldType2 - = new FlattenedFieldMapper.KeyedFlattenedFieldType( "field", true, true, "key2", false, Collections.emptyMap()); + = new FlattenedFieldMapper.KeyedFlattenedFieldType( "field", "field", true, true, "key2", false, Collections.emptyMap()); Function> fieldDataSupplier = fieldType -> { FlattenedFieldMapper.KeyedFlattenedFieldType keyedFieldType = (FlattenedFieldMapper.KeyedFlattenedFieldType) fieldType; From f556a7d5473ad8233b8744d110b410b87f17cf60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Fri, 26 Mar 2021 16:28:03 +0100 Subject: [PATCH 2/7] iter --- .../search/fetch/subphase/FieldFetcherTests.java | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java index 79e6a39a40007..825d4b9815b4d 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java @@ -671,16 +671,7 @@ public void testNestedFields() throws IOException { @SuppressWarnings("unchecked") public void testFlattenedField() throws IOException { - XContentBuilder mapping = XContentFactory.jsonBuilder().startObject() - .startObject("_doc") - .startObject("properties") - .startObject("flat") - .field("type", "flattened") - .endObject() - .endObject() - .endObject() - .endObject(); - + XContentBuilder mapping = mapping(b -> b.startObject("flat").field("type", "flattened").endObject()); MapperService mapperService = createMapperService(mapping); XContentBuilder source = XContentFactory.jsonBuilder().startObject() From 812a3e2745022d14aae936ce90e1535039683730 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Wed, 14 Apr 2021 12:29:44 +0200 Subject: [PATCH 3/7] iter --- .../flattened/FlattenedFieldMapper.java | 16 +++---- .../index/mapper/FieldTypeLookupTests.java | 4 +- .../FlattenedIndexFieldDataTests.java | 4 +- .../KeyedFlattenedFieldTypeTests.java | 46 ++++++++++++++----- .../fetch/subphase/FieldFetcherTests.java | 18 ++++---- .../search/lookup/LeafDocLookupTests.java | 9 ++-- 6 files changed, 57 insertions(+), 40 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java index cb935a9b1d04f..5cf332652cbc1 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java @@ -167,17 +167,17 @@ public static final class KeyedFlattenedFieldType extends StringFieldType { private final String key; private final String rootName; - public KeyedFlattenedFieldType(String name, String rootName, boolean indexed, boolean hasDocValues, String key, + KeyedFlattenedFieldType(String rootName, boolean indexed, boolean hasDocValues, String key, boolean splitQueriesOnWhitespace, Map meta) { - super(name, indexed, false, hasDocValues, + super(rootName + KEYED_FIELD_SUFFIX, indexed, false, hasDocValues, splitQueriesOnWhitespace ? TextSearchInfo.WHITESPACE_MATCH_ONLY : TextSearchInfo.SIMPLE_MATCH_ONLY, meta); this.key = key; this.rootName = rootName; } - private KeyedFlattenedFieldType(String name, String rootName, String key, RootFlattenedFieldType ref) { - this(name, rootName, ref.isSearchable(), ref.hasDocValues(), key, ref.splitQueriesOnWhitespace, ref.meta()); + private KeyedFlattenedFieldType(String rootName, String key, RootFlattenedFieldType ref) { + this(rootName, ref.isSearchable(), ref.hasDocValues(), key, ref.splitQueriesOnWhitespace, ref.meta()); } @Override @@ -427,7 +427,7 @@ private FlattenedFieldMapper(String simpleName, Builder builder) { super(simpleName, mappedFieldType, Lucene.KEYWORD_ANALYZER, CopyTo.empty()); this.builder = builder; - this.fieldParser = new FlattenedFieldParser(mappedFieldType.name(), keyedFieldName(), + this.fieldParser = new FlattenedFieldParser(mappedFieldType.name(), mappedFieldType.name() + KEYED_FIELD_SUFFIX, mappedFieldType, builder.depthLimit.get(), builder.ignoreAbove.get(), builder.nullValue.get()); } @@ -451,11 +451,7 @@ public RootFlattenedFieldType fieldType() { @Override public KeyedFlattenedFieldType keyedFieldType(String key) { - return new KeyedFlattenedFieldType(keyedFieldName(), name(), key, fieldType()); - } - - public String keyedFieldName() { - return mappedFieldType.name() + KEYED_FIELD_SUFFIX; + return new KeyedFlattenedFieldType(name(), key, fieldType()); } @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java index c0fe2e862816c..9f92d75a921dd 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java @@ -189,7 +189,7 @@ public void testFlattenedLookup() { String searchFieldName = fieldName + "." + objectKey; MappedFieldType searchFieldType = lookup.get(searchFieldName); - assertEquals(mapper.keyedFieldName(), searchFieldType.name()); + assertEquals(mapper.keyedFieldType(objectKey).name(), searchFieldType.name()); assertThat(searchFieldType, Matchers.instanceOf(FlattenedFieldMapper.KeyedFlattenedFieldType.class)); FlattenedFieldMapper.KeyedFlattenedFieldType keyedFieldType = (FlattenedFieldMapper.KeyedFlattenedFieldType) searchFieldType; @@ -210,7 +210,7 @@ public void testFlattenedLookupWithAlias() { String searchFieldName = aliasName + "." + objectKey; MappedFieldType searchFieldType = lookup.get(searchFieldName); - assertEquals(mapper.keyedFieldName(), searchFieldType.name()); + assertEquals(mapper.keyedFieldType(objectKey).name(), searchFieldType.name()); assertThat(searchFieldType, Matchers.instanceOf(FlattenedFieldMapper.KeyedFlattenedFieldType.class)); FlattenedFieldMapper.KeyedFlattenedFieldType keyedFieldType = (FlattenedFieldMapper.KeyedFlattenedFieldType) searchFieldType; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedIndexFieldDataTests.java b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedIndexFieldDataTests.java index 36dc484ad79d8..2bd5c67268417 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedIndexFieldDataTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedIndexFieldDataTests.java @@ -46,12 +46,13 @@ public void testGlobalFieldDataCaching() throws IOException { indexService.mapperService()); FlattenedFieldMapper fieldMapper = new FlattenedFieldMapper.Builder("json").build(new ContentPath(1)); + KeyedFlattenedFieldType fieldType1 = fieldMapper.keyedFieldType("key"); AtomicInteger onCacheCalled = new AtomicInteger(); ifdService.setListener(new IndexFieldDataCache.Listener() { @Override public void onCache(ShardId shardId, String fieldName, Accountable ramUsage) { - assertEquals(fieldMapper.keyedFieldName(), fieldName); + assertEquals(fieldType1.name(), fieldName); onCacheCalled.incrementAndGet(); } }); @@ -71,7 +72,6 @@ public void onCache(ShardId shardId, String fieldName, Accountable ramUsage) { new ShardId("test", "_na_", 1)); // Load global field data for subfield 'key'. - KeyedFlattenedFieldType fieldType1 = fieldMapper.keyedFieldType("key"); IndexFieldData ifd1 = ifdService.getForField(fieldType1, "test", () -> { throw new UnsupportedOperationException("search lookup not available"); }); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedFieldTypeTests.java index 7f14ba5656b20..3498a9de831fb 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedFieldTypeTests.java @@ -20,18 +20,25 @@ import org.elasticsearch.common.lucene.search.AutomatonQueries; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.index.mapper.FieldTypeTestCase; +import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.mapper.flattened.FlattenedFieldMapper.KeyedFlattenedFieldType; +import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.search.lookup.SourceLookup; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Set; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class KeyedFlattenedFieldTypeTests extends FieldTypeTestCase { private static KeyedFlattenedFieldType createFieldType() { - return new KeyedFlattenedFieldType("field", "field", true, true, "key", false, Collections.emptyMap()); + return new KeyedFlattenedFieldType("field", true, true, "key", false, Collections.emptyMap()); } public void testIndexedValueForSearch() { @@ -50,23 +57,22 @@ public void testIndexedValueForSearch() { public void testTermQuery() { KeyedFlattenedFieldType ft = createFieldType(); - Query expected = new TermQuery(new Term("field", "key\0value")); + Query expected = new TermQuery(new Term(ft.name(), "key\0value")); assertEquals(expected, ft.termQuery("value", null)); - expected = AutomatonQueries.caseInsensitiveTermQuery(new Term("field", "key\0value")); + expected = AutomatonQueries.caseInsensitiveTermQuery(new Term(ft.name(), "key\0value")); assertEquals(expected, ft.termQueryCaseInsensitive("value", null)); - KeyedFlattenedFieldType unsearchable = new KeyedFlattenedFieldType("field", "field", false, true, "key", - false, Collections.emptyMap()); + KeyedFlattenedFieldType unsearchable = new KeyedFlattenedFieldType("field", false, true, "key", false, Collections.emptyMap()); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> unsearchable.termQuery("field", null)); - assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage()); + assertEquals("Cannot search on field [" + ft.name() + "] since it is not indexed.", e.getMessage()); } public void testTermsQuery() { KeyedFlattenedFieldType ft = createFieldType(); - Query expected = new TermInSetQuery("field", + Query expected = new TermInSetQuery(ft.name(), new BytesRef("key\0value1"), new BytesRef("key\0value2")); @@ -81,17 +87,17 @@ public void testTermsQuery() { public void testExistsQuery() { KeyedFlattenedFieldType ft = createFieldType(); - Query expected = new PrefixQuery(new Term("field", "key\0")); + Query expected = new PrefixQuery(new Term(ft.name(), "key\0")); assertEquals(expected, ft.existsQuery(null)); } public void testPrefixQuery() { KeyedFlattenedFieldType ft = createFieldType(); - Query expected = new PrefixQuery(new Term("field", "key\0val")); + Query expected = new PrefixQuery(new Term(ft.name(), "key\0val")); assertEquals(expected, ft.prefixQuery("val", MultiTermQuery.CONSTANT_SCORE_REWRITE, false, MOCK_CONTEXT)); - expected = AutomatonQueries.caseInsensitivePrefixQuery(new Term("field", "key\0vAl")); + expected = AutomatonQueries.caseInsensitivePrefixQuery(new Term(ft.name(), "key\0vAl")); assertEquals(expected, ft.prefixQuery("vAl", MultiTermQuery.CONSTANT_SCORE_REWRITE, true, MOCK_CONTEXT)); ElasticsearchException ee = expectThrows(ElasticsearchException.class, @@ -111,12 +117,12 @@ public void testFuzzyQuery() { public void testRangeQuery() { KeyedFlattenedFieldType ft = createFieldType(); - TermRangeQuery expected = new TermRangeQuery("field", + TermRangeQuery expected = new TermRangeQuery(ft.name(), new BytesRef("key\0lower"), new BytesRef("key\0upper"), false, false); assertEquals(expected, ft.rangeQuery("lower", "upper", false, false, MOCK_CONTEXT)); - expected = new TermRangeQuery("field", + expected = new TermRangeQuery(ft.name(), new BytesRef("key\0lower"), new BytesRef("key\0upper"), true, true); assertEquals(expected, ft.rangeQuery("lower", "upper", true, true, MOCK_CONTEXT)); @@ -160,4 +166,20 @@ public void testFetchIsEmpty() throws IOException { assertEquals(List.of(), fetchSourceValue(ft, sourceValue)); assertEquals(List.of(), fetchSourceValue(ft, null)); } + + public void testFetchSourceValue() throws IOException { + KeyedFlattenedFieldType ft = createFieldType(); + Map sourceValue = Map.of("key", "value"); + + SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class); + when(searchExecutionContext.sourcePath("field.key")).thenReturn(Set.of("field.key")); + + ValueFetcher fetcher = ft.valueFetcher(searchExecutionContext, null); + SourceLookup lookup = new SourceLookup(); + lookup.setSource(Collections.singletonMap("field", sourceValue)); + + assertEquals(List.of("value"), fetcher.fetchValues(lookup)); + lookup.setSource(Collections.singletonMap("field", null)); + assertEquals(List.of(), fetcher.fetchValues(lookup)); + } } diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java index f82dc15b70452..3707678c980d4 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java @@ -692,10 +692,10 @@ public void testFlattenedField() throws IOException { Map fields = fetchFields(mapperService, source, fieldAndFormatList("*", null, false)); assertEquals(1, fields.size()); assertThat(fields.keySet(), containsInAnyOrder("flat")); - Map flattendedValue = (Map) fields.get("flat").getValue(); - assertThat(flattendedValue.keySet(), containsInAnyOrder("f1", "f2")); - assertEquals("value1", flattendedValue.get("f1")); - assertEquals(1, flattendedValue.get("f2")); + Map flattenedValue = (Map) fields.get("flat").getValue(); + assertThat(flattenedValue.keySet(), containsInAnyOrder("f1", "f2")); + assertEquals("value1", flattenedValue.get("f1")); + assertEquals(1, flattenedValue.get("f2")); // direct retrieval of subfield is possible List fieldAndFormatList = new ArrayList<>(); @@ -710,13 +710,13 @@ public void testFlattenedField() throws IOException { fields = fetchFields(mapperService, source, fieldAndFormatList); assertEquals(2, fields.size()); assertThat(fields.keySet(), containsInAnyOrder("flat", "flat.f1")); - flattendedValue = (Map) fields.get("flat").getValue(); - assertThat(flattendedValue.keySet(), containsInAnyOrder("f1", "f2")); - assertEquals("value1", flattendedValue.get("f1")); - assertEquals(1, flattendedValue.get("f2")); + flattenedValue = (Map) fields.get("flat").getValue(); + assertThat(flattenedValue.keySet(), containsInAnyOrder("f1", "f2")); + assertEquals("value1", flattenedValue.get("f1")); + assertEquals(1, flattenedValue.get("f2")); assertThat(fields.get("flat.f1").getValue(), equalTo("value1")); - // retrieval of subfield with widlcard is not possible + // retrieval of subfield with wildcard is not possible fields = fetchFields(mapperService, source, fieldAndFormatList("flat.f*", null, false)); assertEquals(0, fields.size()); diff --git a/server/src/test/java/org/elasticsearch/search/lookup/LeafDocLookupTests.java b/server/src/test/java/org/elasticsearch/search/lookup/LeafDocLookupTests.java index 381fb60b5d665..bb972e0380c07 100644 --- a/server/src/test/java/org/elasticsearch/search/lookup/LeafDocLookupTests.java +++ b/server/src/test/java/org/elasticsearch/search/lookup/LeafDocLookupTests.java @@ -10,12 +10,12 @@ import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.LeafFieldData; import org.elasticsearch.index.fielddata.ScriptDocValues; +import org.elasticsearch.index.mapper.ContentPath; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.flattened.FlattenedFieldMapper; import org.elasticsearch.test.ESTestCase; import org.junit.Before; -import java.util.Collections; import java.util.function.Function; import static org.mockito.AdditionalAnswers.returnsFirstArg; @@ -61,10 +61,9 @@ public void testFlattenedField() { ScriptDocValues docValues2 = mock(ScriptDocValues.class); IndexFieldData fieldData2 = createFieldData(docValues2); - FlattenedFieldMapper.KeyedFlattenedFieldType fieldType1 - = new FlattenedFieldMapper.KeyedFlattenedFieldType("field", "field", true, true, "key1", false, Collections.emptyMap()); - FlattenedFieldMapper.KeyedFlattenedFieldType fieldType2 - = new FlattenedFieldMapper.KeyedFlattenedFieldType( "field", "field", true, true, "key2", false, Collections.emptyMap()); + FlattenedFieldMapper fieldMapper = new FlattenedFieldMapper.Builder("field").build(new ContentPath(1)); + FlattenedFieldMapper.KeyedFlattenedFieldType fieldType1 = fieldMapper.keyedFieldType("key1"); + FlattenedFieldMapper.KeyedFlattenedFieldType fieldType2 = fieldMapper.keyedFieldType("key2"); Function> fieldDataSupplier = fieldType -> { FlattenedFieldMapper.KeyedFlattenedFieldType keyedFieldType = (FlattenedFieldMapper.KeyedFlattenedFieldType) fieldType; From 2880759382eed46ef963bf68e195b16b61e500a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Wed, 14 Apr 2021 14:22:55 +0200 Subject: [PATCH 4/7] add yaml test --- .../test/search/340_flattened.yml | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/340_flattened.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/340_flattened.yml index 22394ddcad1d3..16edf895950dd 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/340_flattened.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/340_flattened.yml @@ -180,3 +180,60 @@ setup: - length: { hits.hits: 1 } - length: { hits.hits.0.fields: 1 } - match: { hits.hits.0.fields.flattened: [ { "some_field": "some_value" } ] } + +--- +"Test fetching flattened subfields via fields option": + - do: + indices.create: + index: test + body: + mappings: + properties: + flattened: + type: flattened + + - do: + index: + index: test + id: 1 + body: + flattened: + some_field: some_value + some_fields: + - value1 + - value2 + refresh: true + + - do: + search: + index: test + body: + fields: [ { "field" : "flattened.some_field" } ] + + - length: { hits.hits.0.fields: 1 } + - match: { hits.hits.0.fields.flattened\.some_field: [ "some_value" ] } + + - do: + search: + index: test + body: + fields: [ { "field" : "flattened.some_fields" } ] + + - length: { hits.hits.0.fields: 1 } + - match: { hits.hits.0.fields.flattened\.some_fields: [ "value1", "value2" ] } + + - do: + search: + index: test + body: + fields: [ { "field" : "flattened.some*" } ] + + - is_false: hits.hits.0.fields + + - do: + search: + index: test + body: + fields: [ { "field" : "flattened.non_existing_field" } ] + + - is_false: hits.hits.0.fields From 1c22ad7f99ec0fbb601896cf1e2c408c885ba659 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Wed, 14 Apr 2021 17:06:04 +0200 Subject: [PATCH 5/7] add docs --- .../retrieve-selected-fields.asciidoc | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/docs/reference/search/search-your-data/retrieve-selected-fields.asciidoc b/docs/reference/search/search-your-data/retrieve-selected-fields.asciidoc index 3240e568bdedd..3b949aa2e3ef4 100644 --- a/docs/reference/search/search-your-data/retrieve-selected-fields.asciidoc +++ b/docs/reference/search/search-your-data/retrieve-selected-fields.asciidoc @@ -325,6 +325,77 @@ will return only the users first name but still maintain the structure of the ne However, when the `fields` pattern targets the nested `user` field directly, no values will be returned since the pattern doesn't match any leaf fields. +[discrete] +[[search-fields-flattened]] +==== Handling of flattened fields + +The `fields` response for <> is a second example where the +`_source` content is returned unaltered. The `flattened` field maps an entire +object with potentially many subfields as a single field, so fetching its contents +maintains this structure + +Single subfields, however, can be fetched by specifying them explicitly in the request, but +they will not be returned by any wildcard field pattern: + +[source,console] +-------------------------------------------------- +PUT my-index-000001 +{ + "mappings": { + "properties": { + "flattened_field": { + "type": "flattened" + } + } + } +} + +PUT my-index-000001/_doc/1?refresh=true +{ + "flattened_field" : { + "subfield" : "value" + } +} + +POST my-index-000001/_search +{ + "fields": ["flattened_field.subfield"], + "_source": false +} +-------------------------------------------------- + +[source,console-result] +---- +{ + "took": 2, + "timed_out": false, + "_shards": { + "total": 1, + "successful": 1, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": { + "value": 1, + "relation": "eq" + }, + "max_score": 1.0, + "hits": [{ + "_index": "my-index-000001", + "_id": "1", + "_score": 1.0, + "fields": { + "flattened_field.subfield" : [ "value" ] + } + }] + } +} +---- +// TESTRESPONSE[s/"took": 2/"took": $body.took/] +// TESTRESPONSE[s/"max_score" : 1.0/"max_score" : $body.hits.max_score/] +// TESTRESPONSE[s/"_score" : 1.0/"_score" : $body.hits.hits.0._score/] + [discrete] [[retrieve-unmapped-fields]] ==== Retrieving unmapped fields From 83c09fb2600bf9085dc578d5d7e273c0b948d623 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Wed, 14 Apr 2021 21:34:16 +0200 Subject: [PATCH 6/7] iter --- .../mapping/types/flattened.asciidoc | 70 ++++++++++++++++++ .../retrieve-selected-fields.asciidoc | 71 ------------------- .../flattened/FlattenedFieldMapper.java | 3 + .../KeyedFlattenedFieldTypeTests.java | 3 + 4 files changed, 76 insertions(+), 71 deletions(-) diff --git a/docs/reference/mapping/types/flattened.asciidoc b/docs/reference/mapping/types/flattened.asciidoc index 42e7d4b54d6db..8bf740977010d 100644 --- a/docs/reference/mapping/types/flattened.asciidoc +++ b/docs/reference/mapping/types/flattened.asciidoc @@ -121,6 +121,76 @@ lexicographically. Flattened object fields currently cannot be stored. It is not possible to specify the <> parameter in the mapping. +[[search-fields-flattened]] +==== Retrieving flattened fields + +Field values and concrete subfields can be retrieved using the +<>. content. Since the `flattened` field maps an +entire object with potentially many subfields as a single field, the response contains +the unaltered structure from `_source`. + +Single subfields, however, can be fetched by specifying them explicitly in the request. +This only works for concrete paths, but not using wildcards: + +[source,console] +-------------------------------------------------- +PUT my-index-000001 +{ + "mappings": { + "properties": { + "flattened_field": { + "type": "flattened" + } + } + } +} + +PUT my-index-000001/_doc/1?refresh=true +{ + "flattened_field" : { + "subfield" : "value" + } +} + +POST my-index-000001/_search +{ + "fields": ["flattened_field.subfield"], + "_source": false +} +-------------------------------------------------- + +[source,console-result] +---- +{ + "took": 2, + "timed_out": false, + "_shards": { + "total": 1, + "successful": 1, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": { + "value": 1, + "relation": "eq" + }, + "max_score": 1.0, + "hits": [{ + "_index": "my-index-000001", + "_id": "1", + "_score": 1.0, + "fields": { + "flattened_field.subfield" : [ "value" ] + } + }] + } +} +---- +// TESTRESPONSE[s/"took": 2/"took": $body.took/] +// TESTRESPONSE[s/"max_score" : 1.0/"max_score" : $body.hits.max_score/] +// TESTRESPONSE[s/"_score" : 1.0/"_score" : $body.hits.hits.0._score/] + [[flattened-params]] ==== Parameters for flattened object fields diff --git a/docs/reference/search/search-your-data/retrieve-selected-fields.asciidoc b/docs/reference/search/search-your-data/retrieve-selected-fields.asciidoc index 3b949aa2e3ef4..3240e568bdedd 100644 --- a/docs/reference/search/search-your-data/retrieve-selected-fields.asciidoc +++ b/docs/reference/search/search-your-data/retrieve-selected-fields.asciidoc @@ -325,77 +325,6 @@ will return only the users first name but still maintain the structure of the ne However, when the `fields` pattern targets the nested `user` field directly, no values will be returned since the pattern doesn't match any leaf fields. -[discrete] -[[search-fields-flattened]] -==== Handling of flattened fields - -The `fields` response for <> is a second example where the -`_source` content is returned unaltered. The `flattened` field maps an entire -object with potentially many subfields as a single field, so fetching its contents -maintains this structure - -Single subfields, however, can be fetched by specifying them explicitly in the request, but -they will not be returned by any wildcard field pattern: - -[source,console] --------------------------------------------------- -PUT my-index-000001 -{ - "mappings": { - "properties": { - "flattened_field": { - "type": "flattened" - } - } - } -} - -PUT my-index-000001/_doc/1?refresh=true -{ - "flattened_field" : { - "subfield" : "value" - } -} - -POST my-index-000001/_search -{ - "fields": ["flattened_field.subfield"], - "_source": false -} --------------------------------------------------- - -[source,console-result] ----- -{ - "took": 2, - "timed_out": false, - "_shards": { - "total": 1, - "successful": 1, - "skipped": 0, - "failed": 0 - }, - "hits": { - "total": { - "value": 1, - "relation": "eq" - }, - "max_score": 1.0, - "hits": [{ - "_index": "my-index-000001", - "_id": "1", - "_score": 1.0, - "fields": { - "flattened_field.subfield" : [ "value" ] - } - }] - } -} ----- -// TESTRESPONSE[s/"took": 2/"took": $body.took/] -// TESTRESPONSE[s/"max_score" : 1.0/"max_score" : $body.hits.max_score/] -// TESTRESPONSE[s/"_score" : 1.0/"_score" : $body.hits.hits.0._score/] - [discrete] [[retrieve-unmapped-fields]] ==== Retrieving unmapped fields diff --git a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java index 5cf332652cbc1..751d5e58bcc89 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java @@ -263,6 +263,9 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S @Override public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + rootName + "." + key + "] of type [" + typeName() + "] doesn't support formats."); + } return SourceValueFetcher.identity(rootName + "." + key, context, format); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedFieldTypeTests.java index 3498a9de831fb..a9e7c77f32c00 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedFieldTypeTests.java @@ -181,5 +181,8 @@ public void testFetchSourceValue() throws IOException { assertEquals(List.of("value"), fetcher.fetchValues(lookup)); lookup.setSource(Collections.singletonMap("field", null)); assertEquals(List.of(), fetcher.fetchValues(lookup)); + + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> ft.valueFetcher(searchExecutionContext, "format")); + assertEquals("Field [field.key] of type [flattened] doesn't support formats.", e.getMessage()); } } From 6c86f2a906731e3f8d25de5f771b792c098453af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Thu, 15 Apr 2021 10:54:35 +0200 Subject: [PATCH 7/7] checkstyle --- .../index/mapper/flattened/FlattenedFieldMapper.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java index 751d5e58bcc89..25b5665c5da56 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java @@ -264,7 +264,8 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S @Override public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { if (format != null) { - throw new IllegalArgumentException("Field [" + rootName + "." + key + "] of type [" + typeName() + "] doesn't support formats."); + throw new IllegalArgumentException("Field [" + rootName + "." + key + "] of type [" + typeName() + + "] doesn't support formats."); } return SourceValueFetcher.identity(rootName + "." + key, context, format); }