Skip to content

Commit ebb53e1

Browse files
jtibshiranijtibshirani
committed
Add support for storing JSON fields. (#34942)
1 parent e863219 commit ebb53e1

File tree

4 files changed

+91
-35
lines changed

4 files changed

+91
-35
lines changed

‎server/src/main/java/org/elasticsearch/index/mapper/JsonFieldMapper.java

Lines changed: 37 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
package org.elasticsearch.index.mapper;
2121

2222
import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
23+
import org.apache.lucene.document.StoredField;
2324
import org.apache.lucene.index.IndexOptions;
2425
import org.apache.lucene.index.IndexableField;
2526
import org.apache.lucene.index.Term;
@@ -28,11 +29,14 @@
2829
import org.apache.lucene.search.Query;
2930
import org.apache.lucene.search.TermQuery;
3031
import org.apache.lucene.util.BytesRef;
32+
import org.elasticsearch.common.bytes.BytesReference;
3133
import org.elasticsearch.common.lucene.Lucene;
3234
import org.elasticsearch.common.settings.Settings;
3335
import org.elasticsearch.common.unit.Fuzziness;
3436
import org.elasticsearch.common.xcontent.XContentBuilder;
37+
import org.elasticsearch.common.xcontent.XContentFactory;
3538
import org.elasticsearch.common.xcontent.XContentParser;
39+
import org.elasticsearch.common.xcontent.json.JsonXContent;
3640
import org.elasticsearch.common.xcontent.support.XContentMapValues;
3741
import org.elasticsearch.index.analysis.AnalyzerScope;
3842
import org.elasticsearch.index.analysis.NamedAnalyzer;
@@ -71,6 +75,9 @@
7175
*
7276
* Note that \0 is a reserved separator character, and cannot be used in the keys of the JSON object
7377
* (see {@link JsonFieldParser#SEPARATOR}).
78+
*
79+
* When 'store' is enabled, a single stored field is added containing the entire JSON object in
80+
* pretty-printed format.
7481
*/
7582
public final class JsonFieldMapper extends FieldMapper {
7683

@@ -139,12 +146,6 @@ public Builder copyTo(CopyTo copyTo) {
139146
throw new UnsupportedOperationException("[copy_to] is not supported for [" + CONTENT_TYPE + "] fields.");
140147
}
141148

142-
@Override
143-
public Builder store(boolean store) {
144-
throw new UnsupportedOperationException("[store] is not currently supported for [" +
145-
CONTENT_TYPE + "] fields.");
146-
}
147-
148149
@Override
149150
public JsonFieldMapper build(BuilderContext context) {
150151
setupFieldType(context);
@@ -377,7 +378,8 @@ private JsonFieldMapper(String simpleName,
377378
assert fieldType.indexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) <= 0;
378379

379380
this.ignoreAbove = ignoreAbove;
380-
this.fieldParser = new JsonFieldParser(fieldType.name(), keyedFieldName(), fieldType, ignoreAbove);
381+
this.fieldParser = new JsonFieldParser(fieldType.name(), keyedFieldName(),
382+
ignoreAbove, fieldType.nullValueAsString());
381383
}
382384

383385
@Override
@@ -415,12 +417,36 @@ protected void parseCreateField(ParseContext context, List<IndexableField> field
415417
return;
416418
}
417419

418-
if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) {
419-
fields.addAll(fieldParser.parse(context.parser()));
420-
createFieldNamesField(context, fields);
421-
} else {
420+
if (fieldType.indexOptions() == IndexOptions.NONE && !fieldType.stored()) {
422421
context.parser().skipChildren();
422+
return;
423+
}
424+
425+
BytesRef storedValue = null;
426+
if (fieldType.stored()) {
427+
XContentBuilder builder = XContentFactory.jsonBuilder()
428+
.prettyPrint()
429+
.copyCurrentStructure(context.parser());
430+
storedValue = BytesReference.bytes(builder).toBytesRef();
431+
fields.add(new StoredField(fieldType.name(), storedValue));
423432
}
433+
434+
if (fieldType().indexOptions() != IndexOptions.NONE) {
435+
XContentParser indexedFieldsParser = context.parser();
436+
437+
// If store is enabled, we've already consumed the content to produce the stored field. Here we
438+
// 'reset' the parser, so that we can traverse the content again.
439+
if (storedValue != null) {
440+
indexedFieldsParser = JsonXContent.jsonXContent.createParser(context.parser().getXContentRegistry(),
441+
context.parser().getDeprecationHandler(),
442+
storedValue.bytes);
443+
indexedFieldsParser.nextToken();
444+
}
445+
446+
fields.addAll(fieldParser.parse(indexedFieldsParser));
447+
}
448+
449+
createFieldNamesField(context, fields);
424450
}
425451

426452
@Override

‎server/src/main/java/org/elasticsearch/index/mapper/JsonFieldParser.java

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
package org.elasticsearch.index.mapper;
2121

2222
import org.apache.lucene.document.Field;
23+
import org.apache.lucene.document.StringField;
2324
import org.apache.lucene.index.IndexableField;
2425
import org.apache.lucene.util.BytesRef;
2526
import org.elasticsearch.common.xcontent.XContentParser;
@@ -39,17 +40,17 @@ public class JsonFieldParser {
3940
private final String rootFieldName;
4041
private final String keyedFieldName;
4142

42-
private final MappedFieldType fieldType;
4343
private final int ignoreAbove;
44+
private final String nullValueAsString;
4445

4546
JsonFieldParser(String rootFieldName,
4647
String keyedFieldName,
47-
MappedFieldType fieldType,
48-
int ignoreAbove) {
48+
int ignoreAbove,
49+
String nullValueAsString) {
4950
this.rootFieldName = rootFieldName;
5051
this.keyedFieldName = keyedFieldName;
51-
this.fieldType = fieldType;
5252
this.ignoreAbove = ignoreAbove;
53+
this.nullValueAsString = nullValueAsString;
5354
}
5455

5556
public List<IndexableField> parse(XContentParser parser) throws IOException {
@@ -111,9 +112,8 @@ private void parseFieldValue(XContentParser.Token token,
111112
String value = parser.text();
112113
addField(path, currentName, value, fields);
113114
} else if (token == XContentParser.Token.VALUE_NULL) {
114-
String value = fieldType.nullValueAsString();
115-
if (value != null) {
116-
addField(path, currentName, value, fields);
115+
if (nullValueAsString != null) {
116+
addField(path, currentName, nullValueAsString, fields);
117117
}
118118
} else {
119119
// Note that we throw an exception here just to be safe. We don't actually expect to reach
@@ -137,8 +137,8 @@ private void addField(ContentPath path,
137137
}
138138
String keyedValue = createKeyedValue(key, value);
139139

140-
fields.add(new Field(rootFieldName, new BytesRef(value), fieldType));
141-
fields.add(new Field(keyedFieldName, new BytesRef(keyedValue), fieldType));
140+
fields.add(new StringField(rootFieldName, new BytesRef(value), Field.Store.NO));
141+
fields.add(new StringField(keyedFieldName, new BytesRef(keyedValue), Field.Store.NO));
142142
}
143143

144144
public static String createKeyedValue(String key, String value) {

‎server/src/test/java/org/elasticsearch/index/mapper/JsonFieldMapperTests.java

Lines changed: 39 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import org.elasticsearch.common.compress.CompressedXContent;
2828
import org.elasticsearch.common.xcontent.XContentFactory;
2929
import org.elasticsearch.common.xcontent.XContentType;
30+
import org.elasticsearch.common.xcontent.json.JsonXContent;
3031
import org.elasticsearch.index.IndexService;
3132
import org.elasticsearch.index.mapper.JsonFieldMapper.RootJsonFieldType;
3233
import org.elasticsearch.plugins.Plugin;
@@ -130,16 +131,51 @@ public void testEnableStore() throws Exception {
130131
String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject()
131132
.startObject("type")
132133
.startObject("properties")
133-
.startObject("field")
134+
.startObject("store_and_index")
134135
.field("type", "json")
135136
.field("store", true)
136137
.endObject()
138+
.startObject("store_only")
139+
.field("type", "json")
140+
.field("index", false)
141+
.field("store", true)
142+
.endObject()
137143
.endObject()
138144
.endObject()
139145
.endObject());
140146

141-
expectThrows(UnsupportedOperationException.class, () ->
142-
parser.parse("type", new CompressedXContent(mapping)));
147+
DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
148+
assertEquals(mapping, mapper.mappingSource().toString());
149+
150+
BytesReference doc = BytesReference.bytes(XContentFactory.jsonBuilder().startObject()
151+
.startObject("store_only")
152+
.field("key", "value")
153+
.endObject()
154+
.startObject("store_and_index")
155+
.field("key", "value")
156+
.endObject()
157+
.endObject());
158+
ParsedDocument parsedDoc = mapper.parse(new SourceToParse("test", "type", "1", doc, XContentType.JSON));
159+
160+
// We make sure to pretty-print here, since the field is always stored in pretty-printed format.
161+
BytesReference storedValue = BytesReference.bytes(JsonXContent.contentBuilder()
162+
.prettyPrint()
163+
.startObject()
164+
.field("key", "value")
165+
.endObject());
166+
167+
IndexableField[] storeOnly = parsedDoc.rootDoc().getFields("store_only");
168+
assertEquals(1, storeOnly.length);
169+
170+
assertTrue(storeOnly[0].fieldType().stored());
171+
assertEquals(storedValue.toBytesRef(), storeOnly[0].binaryValue());
172+
173+
IndexableField[] storeAndIndex = parsedDoc.rootDoc().getFields("store_and_index");
174+
assertEquals(2, storeAndIndex.length);
175+
176+
assertTrue(storeAndIndex[0].fieldType().stored());
177+
assertEquals(storedValue.toBytesRef(), storeAndIndex[0].binaryValue());
178+
assertFalse(storeAndIndex[1].fieldType().stored());
143179
}
144180

145181
public void testIndexOptions() throws IOException {

‎server/src/test/java/org/elasticsearch/index/mapper/JsonFieldParserTests.java

Lines changed: 6 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -41,10 +41,7 @@ public class JsonFieldParserTests extends ESTestCase {
4141
@Before
4242
public void setUp() throws Exception {
4343
super.setUp();
44-
45-
MappedFieldType fieldType = new RootJsonFieldType();
46-
fieldType.setName("field");
47-
parser = new JsonFieldParser("field", "field._keyed", fieldType, Integer.MAX_VALUE);
44+
parser = new JsonFieldParser("field", "field._keyed", Integer.MAX_VALUE, null);
4845
}
4946

5047
public void testTextValues() throws Exception {
@@ -222,9 +219,9 @@ public void testIgnoreAbove() throws Exception {
222219

223220
RootJsonFieldType fieldType = new RootJsonFieldType();
224221
fieldType.setName("field");
225-
JsonFieldParser ignoreAboveParser = new JsonFieldParser("field", "field._keyed", fieldType, 10);
222+
JsonFieldParser parserWithIgnoreAbove = new JsonFieldParser("field", "field._keyed", 10, null);
226223

227-
List<IndexableField> fields = ignoreAboveParser.parse(xContentParser);
224+
List<IndexableField> fields = parserWithIgnoreAbove.parse(xContentParser);
228225
assertEquals(0, fields.size());
229226
}
230227

@@ -236,13 +233,10 @@ public void testNullValues() throws Exception {
236233
assertEquals(0, fields.size());
237234

238235
xContentParser = createXContentParser(input);
236+
JsonFieldParser parserWithNullValue = new JsonFieldParser("field", "field._keyed",
237+
Integer.MAX_VALUE, "placeholder");
239238

240-
RootJsonFieldType fieldType = new RootJsonFieldType();
241-
fieldType.setName("field");
242-
fieldType.setNullValue("placeholder");
243-
JsonFieldParser nullValueParser = new JsonFieldParser("field", "field._keyed", fieldType, Integer.MAX_VALUE);
244-
245-
fields = nullValueParser.parse(xContentParser);
239+
fields = parserWithNullValue.parse(xContentParser);
246240
assertEquals(2, fields.size());
247241

248242
IndexableField field = fields.get(0);

0 commit comments

Comments
 (0)