Skip to content

Commit 9a4d006

Browse files
javannajavanna
authored
REST high-level client: parse back _ignored meta field (#32362)
`GetResult` and `SearchHit` have been adjusted to parse back the `_ignored` meta field whenever it gets printed out. Expanded the existing tests to make sure this is covered. Fixed also a small problem around highlighted fields in `SearchHitTests`.
1 parent 14a4a74 commit 9a4d006

File tree

7 files changed

+103
-73
lines changed

7 files changed

+103
-73
lines changed

‎server/src/main/java/org/elasticsearch/index/get/GetResult.java

Lines changed: 19 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
package org.elasticsearch.index.get;
2121

2222
import org.elasticsearch.ElasticsearchParseException;
23+
import org.elasticsearch.common.Strings;
2324
import org.elasticsearch.common.bytes.BytesReference;
2425
import org.elasticsearch.common.compress.CompressorFactory;
2526
import org.elasticsearch.common.document.DocumentField;
@@ -30,6 +31,7 @@
3031
import org.elasticsearch.common.xcontent.XContentBuilder;
3132
import org.elasticsearch.common.xcontent.XContentHelper;
3233
import org.elasticsearch.common.xcontent.XContentParser;
34+
import org.elasticsearch.index.mapper.IgnoredFieldMapper;
3335
import org.elasticsearch.index.mapper.SourceFieldMapper;
3436
import org.elasticsearch.search.lookup.SourceLookup;
3537

@@ -225,10 +227,13 @@ public XContentBuilder toXContentEmbedded(XContentBuilder builder, Params params
225227
}
226228
}
227229
}
228-
229230
for (DocumentField field : metaFields) {
230-
Object value = field.getValue();
231-
builder.field(field.getName(), value);
231+
// TODO: can we avoid having an exception here?
232+
if (field.getName().equals(IgnoredFieldMapper.NAME)) {
233+
builder.field(field.getName(), field.getValues());
234+
} else {
235+
builder.field(field.getName(), field.<Object>getValue());
236+
}
232237
}
233238

234239
builder.field(FOUND, exists);
@@ -316,7 +321,11 @@ public static GetResult fromXContentEmbedded(XContentParser parser, String index
316321
parser.skipChildren(); // skip potential inner objects for forward compatibility
317322
}
318323
} else if (token == XContentParser.Token.START_ARRAY) {
319-
parser.skipChildren(); // skip potential inner arrays for forward compatibility
324+
if (IgnoredFieldMapper.NAME.equals(currentFieldName)) {
325+
fields.put(currentFieldName, new DocumentField(currentFieldName, parser.list()));
326+
} else {
327+
parser.skipChildren(); // skip potential inner arrays for forward compatibility
328+
}
320329
}
321330
}
322331
return new GetResult(index, type, id, version, found, source, fields);
@@ -400,7 +409,12 @@ public boolean equals(Object o) {
400409

401410
@Override
402411
public int hashCode() {
403-
return Objects.hash(index, type, id, version, exists, fields, sourceAsMap());
412+
return Objects.hash(version, exists, index, type, id, fields, sourceAsMap());
413+
}
414+
415+
@Override
416+
public String toString() {
417+
return Strings.toString(this, true, true);
404418
}
405419
}
406420

‎server/src/main/java/org/elasticsearch/search/SearchHit.java

Lines changed: 23 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -602,16 +602,24 @@ private static void declareMetaDataFields(ObjectParser<Map<String, Object>, Void
602602
for (String metadatafield : MapperService.getAllMetaFields()) {
603603
if (metadatafield.equals(Fields._ID) == false && metadatafield.equals(Fields._INDEX) == false
604604
&& metadatafield.equals(Fields._TYPE) == false) {
605-
parser.declareField((map, field) -> {
606-
@SuppressWarnings("unchecked")
607-
Map<String, DocumentField> fieldMap = (Map<String, DocumentField>) map.computeIfAbsent(Fields.FIELDS,
608-
v -> new HashMap<String, DocumentField>());
609-
fieldMap.put(field.getName(), field);
610-
}, (p, c) -> {
611-
List<Object> values = new ArrayList<>();
612-
values.add(parseFieldsValue(p));
613-
return new DocumentField(metadatafield, values);
614-
}, new ParseField(metadatafield), ValueType.VALUE);
605+
if (metadatafield.equals(IgnoredFieldMapper.NAME)) {
606+
parser.declareObjectArray((map, list) -> {
607+
@SuppressWarnings("unchecked")
608+
Map<String, DocumentField> fieldMap = (Map<String, DocumentField>) map.computeIfAbsent(Fields.FIELDS,
609+
v -> new HashMap<String, DocumentField>());
610+
DocumentField field = new DocumentField(metadatafield, list);
611+
fieldMap.put(field.getName(), field);
612+
}, (p, c) -> parseFieldsValue(p),
613+
new ParseField(metadatafield));
614+
} else {
615+
parser.declareField((map, field) -> {
616+
@SuppressWarnings("unchecked")
617+
Map<String, DocumentField> fieldMap = (Map<String, DocumentField>) map.computeIfAbsent(Fields.FIELDS,
618+
v -> new HashMap<String, DocumentField>());
619+
fieldMap.put(field.getName(), field);
620+
}, (p, c) -> new DocumentField(metadatafield, Collections.singletonList(parseFieldsValue(p))),
621+
new ParseField(metadatafield), ValueType.VALUE);
622+
}
615623
}
616624
}
617625
}
@@ -958,4 +966,9 @@ public int hashCode() {
958966
return Objects.hash(field, offset, child);
959967
}
960968
}
969+
970+
@Override
971+
public String toString() {
972+
return Strings.toString(this, true, true);
973+
}
961974
}

‎server/src/test/java/org/elasticsearch/index/get/DocumentFieldTests.java

Lines changed: 26 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,11 @@
2626
import org.elasticsearch.common.xcontent.ToXContent;
2727
import org.elasticsearch.common.xcontent.XContentParser;
2828
import org.elasticsearch.common.xcontent.XContentType;
29-
import org.elasticsearch.index.mapper.RoutingFieldMapper;
29+
import org.elasticsearch.index.mapper.IdFieldMapper;
30+
import org.elasticsearch.index.mapper.IgnoredFieldMapper;
31+
import org.elasticsearch.index.mapper.IndexFieldMapper;
32+
import org.elasticsearch.index.mapper.MapperService;
33+
import org.elasticsearch.index.mapper.TypeFieldMapper;
3034
import org.elasticsearch.test.ESTestCase;
3135
import org.elasticsearch.test.RandomObjects;
3236

@@ -98,14 +102,28 @@ private static DocumentField mutateDocumentField(DocumentField documentField) {
98102

99103
public static Tuple<DocumentField, DocumentField> randomDocumentField(XContentType xContentType) {
100104
if (randomBoolean()) {
101-
String fieldName = randomFrom(RoutingFieldMapper.NAME);
102-
DocumentField documentField = new DocumentField(fieldName, Collections.singletonList(randomAlphaOfLengthBetween(3, 10)));
105+
String metaField = randomValueOtherThanMany(field -> field.equals(TypeFieldMapper.NAME)
106+
|| field.equals(IndexFieldMapper.NAME) || field.equals(IdFieldMapper.NAME),
107+
() -> randomFrom(MapperService.getAllMetaFields()));
108+
DocumentField documentField;
109+
if (metaField.equals(IgnoredFieldMapper.NAME)) {
110+
int numValues = randomIntBetween(1, 3);
111+
List<Object> ignoredFields = new ArrayList<>(numValues);
112+
for (int i = 0; i < numValues; i++) {
113+
ignoredFields.add(randomAlphaOfLengthBetween(3, 10));
114+
}
115+
documentField = new DocumentField(metaField, ignoredFields);
116+
} else {
117+
//meta fields are single value only, besides _ignored
118+
documentField = new DocumentField(metaField, Collections.singletonList(randomAlphaOfLengthBetween(3, 10)));
119+
}
103120
return Tuple.tuple(documentField, documentField);
121+
} else {
122+
String fieldName = randomAlphaOfLengthBetween(3, 10);
123+
Tuple<List<Object>, List<Object>> tuple = RandomObjects.randomStoredFieldValues(random(), xContentType);
124+
DocumentField input = new DocumentField(fieldName, tuple.v1());
125+
DocumentField expected = new DocumentField(fieldName, tuple.v2());
126+
return Tuple.tuple(input, expected);
104127
}
105-
String fieldName = randomAlphaOfLengthBetween(3, 10);
106-
Tuple<List<Object>, List<Object>> tuple = RandomObjects.randomStoredFieldValues(random(), xContentType);
107-
DocumentField input = new DocumentField(fieldName, tuple.v1());
108-
DocumentField expected = new DocumentField(fieldName, tuple.v2());
109-
return Tuple.tuple(input, expected);
110128
}
111129
}

‎server/src/test/java/org/elasticsearch/index/get/GetResultTests.java

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,6 @@ public void testToAndFromXContentEmbedded() throws Exception {
9090
XContentType xContentType = randomFrom(XContentType.values());
9191
Tuple<GetResult, GetResult> tuple = randomGetResult(xContentType);
9292
GetResult getResult = tuple.v1();
93-
9493
// We don't expect to retrieve the index/type/id of the GetResult because they are not rendered
9594
// by the toXContentEmbedded method.
9695
GetResult expectedGetResult = new GetResult(null, null, null, -1,
@@ -106,7 +105,6 @@ public void testToAndFromXContentEmbedded() throws Exception {
106105
parsedEmbeddedGetResult = GetResult.fromXContentEmbedded(parser);
107106
assertNull(parser.nextToken());
108107
}
109-
110108
assertEquals(expectedGetResult, parsedEmbeddedGetResult);
111109
//print the parsed object out and test that the output is the same as the original output
112110
BytesReference finalBytes = toXContentEmbedded(parsedEmbeddedGetResult, xContentType, humanReadable);
@@ -203,16 +201,17 @@ public static Tuple<GetResult, GetResult> randomGetResult(XContentType xContentT
203201
return Tuple.tuple(getResult, expectedGetResult);
204202
}
205203

206-
private static Tuple<Map<String, DocumentField>,Map<String, DocumentField>> randomDocumentFields(XContentType xContentType) {
204+
public static Tuple<Map<String, DocumentField>,Map<String, DocumentField>> randomDocumentFields(XContentType xContentType) {
207205
int numFields = randomIntBetween(2, 10);
208206
Map<String, DocumentField> fields = new HashMap<>(numFields);
209207
Map<String, DocumentField> expectedFields = new HashMap<>(numFields);
210-
for (int i = 0; i < numFields; i++) {
208+
while (fields.size() < numFields) {
211209
Tuple<DocumentField, DocumentField> tuple = randomDocumentField(xContentType);
212210
DocumentField getField = tuple.v1();
213211
DocumentField expectedGetField = tuple.v2();
214-
fields.put(getField.getName(), getField);
215-
expectedFields.put(expectedGetField.getName(), expectedGetField);
212+
if (fields.putIfAbsent(getField.getName(), getField) == null) {
213+
assertNull(expectedFields.putIfAbsent(expectedGetField.getName(), expectedGetField));
214+
}
216215
}
217216
return Tuple.tuple(fields, expectedFields);
218217
}

‎server/src/test/java/org/elasticsearch/search/SearchHitTests.java

Lines changed: 13 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -19,40 +19,38 @@
1919

2020
package org.elasticsearch.search;
2121

22+
import java.io.IOException;
23+
import java.io.InputStream;
24+
import java.util.ArrayList;
25+
import java.util.Collections;
26+
import java.util.HashMap;
27+
import java.util.List;
28+
import java.util.Map;
29+
import java.util.function.Predicate;
30+
2231
import org.apache.lucene.search.Explanation;
2332
import org.elasticsearch.action.OriginalIndices;
2433
import org.elasticsearch.common.Strings;
2534
import org.elasticsearch.common.bytes.BytesArray;
2635
import org.elasticsearch.common.bytes.BytesReference;
27-
import org.elasticsearch.common.collect.Tuple;
2836
import org.elasticsearch.common.document.DocumentField;
2937
import org.elasticsearch.common.io.stream.BytesStreamOutput;
3038
import org.elasticsearch.common.io.stream.InputStreamStreamInput;
3139
import org.elasticsearch.common.text.Text;
32-
import org.elasticsearch.common.util.set.Sets;
3340
import org.elasticsearch.common.xcontent.ToXContent;
3441
import org.elasticsearch.common.xcontent.XContentBuilder;
3542
import org.elasticsearch.common.xcontent.XContentParser;
3643
import org.elasticsearch.common.xcontent.XContentType;
3744
import org.elasticsearch.common.xcontent.json.JsonXContent;
3845
import org.elasticsearch.index.Index;
46+
import org.elasticsearch.index.get.GetResultTests;
3947
import org.elasticsearch.index.shard.ShardId;
4048
import org.elasticsearch.search.SearchHit.NestedIdentity;
4149
import org.elasticsearch.search.fetch.subphase.highlight.HighlightField;
4250
import org.elasticsearch.search.fetch.subphase.highlight.HighlightFieldTests;
4351
import org.elasticsearch.test.ESTestCase;
4452
import org.elasticsearch.test.RandomObjects;
4553

46-
import java.io.IOException;
47-
import java.io.InputStream;
48-
import java.util.ArrayList;
49-
import java.util.Collections;
50-
import java.util.HashMap;
51-
import java.util.List;
52-
import java.util.Map;
53-
import java.util.Set;
54-
import java.util.function.Predicate;
55-
5654
import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
5755
import static org.elasticsearch.test.XContentTestUtils.insertRandomFields;
5856
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
@@ -63,8 +61,6 @@
6361

6462
public class SearchHitTests extends ESTestCase {
6563

66-
private static Set<String> META_FIELDS = Sets.newHashSet("_uid", "_parent", "_routing", "_size", "_timestamp", "_ttl");
67-
6864
public static SearchHit createTestItem(boolean withOptionalInnerHits) {
6965
int internalId = randomInt();
7066
String uid = randomAlphaOfLength(10);
@@ -75,18 +71,7 @@ public static SearchHit createTestItem(boolean withOptionalInnerHits) {
7571
}
7672
Map<String, DocumentField> fields = new HashMap<>();
7773
if (randomBoolean()) {
78-
int size = randomIntBetween(0, 10);
79-
for (int i = 0; i < size; i++) {
80-
Tuple<List<Object>, List<Object>> values = RandomObjects.randomStoredFieldValues(random(),
81-
XContentType.JSON);
82-
if (randomBoolean()) {
83-
String metaField = randomFrom(META_FIELDS);
84-
fields.put(metaField, new DocumentField(metaField, values.v1()));
85-
} else {
86-
String fieldName = randomAlphaOfLengthBetween(5, 10);
87-
fields.put(fieldName, new DocumentField(fieldName, values.v1()));
88-
}
89-
}
74+
fields = GetResultTests.randomDocumentFields(XContentType.JSON).v1();
9075
}
9176
SearchHit hit = new SearchHit(internalId, uid, type, nestedIdentity, fields);
9277
if (frequently()) {
@@ -109,7 +94,8 @@ public static SearchHit createTestItem(boolean withOptionalInnerHits) {
10994
int size = randomIntBetween(0, 5);
11095
Map<String, HighlightField> highlightFields = new HashMap<>(size);
11196
for (int i = 0; i < size; i++) {
112-
highlightFields.put(randomAlphaOfLength(5), HighlightFieldTests.createTestItem());
97+
HighlightField testItem = HighlightFieldTests.createTestItem();
98+
highlightFields.put(testItem.getName(), testItem);
11399
}
114100
hit.highlightFields(highlightFields);
115101
}

‎server/src/test/java/org/elasticsearch/search/SearchSortValuesTests.java

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -46,13 +46,13 @@ public static SearchSortValues createTestItem() {
4646
List<Supplier<Object>> valueSuppliers = new ArrayList<>();
4747
// this should reflect all values that are allowed to go through the transport layer
4848
valueSuppliers.add(() -> null);
49-
valueSuppliers.add(() -> randomInt());
50-
valueSuppliers.add(() -> randomLong());
51-
valueSuppliers.add(() -> randomDouble());
52-
valueSuppliers.add(() -> randomFloat());
53-
valueSuppliers.add(() -> randomByte());
54-
valueSuppliers.add(() -> randomShort());
55-
valueSuppliers.add(() -> randomBoolean());
49+
valueSuppliers.add(ESTestCase::randomInt);
50+
valueSuppliers.add(ESTestCase::randomLong);
51+
valueSuppliers.add(ESTestCase::randomDouble);
52+
valueSuppliers.add(ESTestCase::randomFloat);
53+
valueSuppliers.add(ESTestCase::randomByte);
54+
valueSuppliers.add(ESTestCase::randomShort);
55+
valueSuppliers.add(ESTestCase::randomBoolean);
5656
valueSuppliers.add(() -> frequently() ? randomAlphaOfLengthBetween(1, 30) : randomRealisticUnicodeOfCodepointLength(30));
5757

5858
int size = randomIntBetween(1, 20);

‎test/framework/src/main/java/org/elasticsearch/test/RandomObjects.java

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@
4848
import java.util.Random;
4949

5050
import static com.carrotsearch.randomizedtesting.generators.RandomNumbers.randomIntBetween;
51-
import static com.carrotsearch.randomizedtesting.generators.RandomStrings.randomAsciiOfLength;
51+
import static com.carrotsearch.randomizedtesting.generators.RandomStrings.randomAsciiLettersOfLength;
5252
import static com.carrotsearch.randomizedtesting.generators.RandomStrings.randomUnicodeOfLengthBetween;
5353
import static java.util.Collections.singleton;
5454
import static org.elasticsearch.cluster.metadata.IndexMetaData.INDEX_UUID_NA_VALUE;
@@ -122,7 +122,7 @@ public static Tuple<List<Object>, List<Object>> randomStoredFieldValues(Random r
122122
expectedParsedValues.add(randomBoolean);
123123
break;
124124
case 7:
125-
String randomString = random.nextBoolean() ? RandomStrings.randomAsciiOfLengthBetween(random, 3, 10 ) :
125+
String randomString = random.nextBoolean() ? RandomStrings.randomAsciiLettersOfLengthBetween(random, 3, 10) :
126126
randomUnicodeOfLengthBetween(random, 3, 10);
127127
originalValues.add(randomString);
128128
expectedParsedValues.add(randomString);
@@ -191,11 +191,11 @@ private static void addFields(Random random, XContentBuilder builder, int minNum
191191
for (int i = 0; i < numFields; i++) {
192192
if (currentDepth < 5 && random.nextInt(100) >= 70) {
193193
if (random.nextBoolean()) {
194-
builder.startObject(RandomStrings.randomAsciiOfLengthBetween(random, 6, 10));
194+
builder.startObject(RandomStrings.randomAsciiLettersOfLengthBetween(random, 6, 10));
195195
addFields(random, builder, minNumFields, currentDepth + 1);
196196
builder.endObject();
197197
} else {
198-
builder.startArray(RandomStrings.randomAsciiOfLengthBetween(random, 6, 10));
198+
builder.startArray(RandomStrings.randomAsciiLettersOfLengthBetween(random, 6, 10));
199199
int numElements = randomIntBetween(random, 1, 5);
200200
boolean object = random.nextBoolean();
201201
int dataType = -1;
@@ -214,7 +214,7 @@ private static void addFields(Random random, XContentBuilder builder, int minNum
214214
builder.endArray();
215215
}
216216
} else {
217-
builder.field(RandomStrings.randomAsciiOfLengthBetween(random, 6, 10),
217+
builder.field(RandomStrings.randomAsciiLettersOfLengthBetween(random, 6, 10),
218218
randomFieldValue(random, randomDataType(random)));
219219
}
220220
}
@@ -227,9 +227,9 @@ private static int randomDataType(Random random) {
227227
private static Object randomFieldValue(Random random, int dataType) {
228228
switch(dataType) {
229229
case 0:
230-
return RandomStrings.randomAsciiOfLengthBetween(random, 3, 10);
230+
return RandomStrings.randomAsciiLettersOfLengthBetween(random, 3, 10);
231231
case 1:
232-
return RandomStrings.randomAsciiOfLengthBetween(random, 3, 10);
232+
return RandomStrings.randomAsciiLettersOfLengthBetween(random, 3, 10);
233233
case 2:
234234
return random.nextLong();
235235
case 3:
@@ -287,10 +287,10 @@ public static Tuple<ShardInfo, ShardInfo> randomShardInfo(Random random, boolean
287287
* @param random Random generator
288288
*/
289289
private static Tuple<Failure, Failure> randomShardInfoFailure(Random random) {
290-
String index = randomAsciiOfLength(random, 5);
291-
String indexUuid = randomAsciiOfLength(random, 5);
290+
String index = randomAsciiLettersOfLength(random, 5);
291+
String indexUuid = randomAsciiLettersOfLength(random, 5);
292292
int shardId = randomIntBetween(random, 1, 10);
293-
String nodeId = randomAsciiOfLength(random, 5);
293+
String nodeId = randomAsciiLettersOfLength(random, 5);
294294
RestStatus status = randomFrom(random, RestStatus.INTERNAL_SERVER_ERROR, RestStatus.FORBIDDEN, RestStatus.NOT_FOUND);
295295
boolean primary = random.nextBoolean();
296296
ShardId shard = new ShardId(index, indexUuid, shardId);

0 commit comments

Comments
 (0)