Ensure that the indexed leaf values are not stored.

Author: jtibshirani

server/src/main/java/org/elasticsearch/index/mapper/JsonFieldMapper.java

@@ -42,7 +42,6 @@
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.query.QueryShardContext;
 
-import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.util.Iterator;
 import java.util.List;
@@ -379,7 +378,8 @@ private JsonFieldMapper(String simpleName,
         assert fieldType.indexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) <= 0;
 
         this.ignoreAbove = ignoreAbove;
-        this.fieldParser = new JsonFieldParser(fieldType.name(), keyedFieldName(), fieldType, ignoreAbove);
+        this.fieldParser = new JsonFieldParser(fieldType.name(), keyedFieldName(),
+            ignoreAbove, fieldType.nullValueAsString());
     }
 
     @Override

server/src/main/java/org/elasticsearch/index/mapper/JsonFieldParser.java

@@ -20,6 +20,7 @@
 package org.elasticsearch.index.mapper;
 
 import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StringField;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -39,17 +40,17 @@ public class JsonFieldParser {
     private final String rootFieldName;
     private final String keyedFieldName;
 
-    private final MappedFieldType fieldType;
     private final int ignoreAbove;
+    private final String nullValueAsString;
 
     JsonFieldParser(String rootFieldName,
                     String keyedFieldName,
-                    MappedFieldType fieldType,
-                    int ignoreAbove) {
+                    int ignoreAbove,
+                    String nullValueAsString) {
         this.rootFieldName = rootFieldName;
         this.keyedFieldName = keyedFieldName;
-        this.fieldType = fieldType;
         this.ignoreAbove = ignoreAbove;
+        this.nullValueAsString = nullValueAsString;
     }
 
     public List<IndexableField> parse(XContentParser parser) throws IOException {
@@ -111,9 +112,8 @@ private void parseFieldValue(XContentParser.Token token,
             String value = parser.text();
             addField(path, currentName, value, fields);
         } else if (token == XContentParser.Token.VALUE_NULL) {
-            String value = fieldType.nullValueAsString();
-            if (value != null) {
-                addField(path, currentName, value, fields);
+            if (nullValueAsString != null) {
+                addField(path, currentName, nullValueAsString, fields);
             }
         } else {
             // Note that we throw an exception here just to be safe. We don't actually expect to reach
@@ -137,8 +137,8 @@ private void addField(ContentPath path,
         }
         String keyedValue = createKeyedValue(key, value);
 
-        fields.add(new Field(rootFieldName, new BytesRef(value), fieldType));
-        fields.add(new Field(keyedFieldName, new BytesRef(keyedValue), fieldType));
+        fields.add(new StringField(rootFieldName, new BytesRef(value), Field.Store.NO));
+        fields.add(new StringField(keyedFieldName, new BytesRef(keyedValue), Field.Store.NO));
     }
 
     public static String createKeyedValue(String key, String value) {

server/src/test/java/org/elasticsearch/index/mapper/JsonFieldMapperTests.java

@@ -131,7 +131,11 @@ public void testEnableStore() throws Exception {
         String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject()
             .startObject("type")
                 .startObject("properties")
-                    .startObject("field")
+                    .startObject("store_and_index")
+                        .field("type", "json")
+                        .field("store", true)
+                    .endObject()
+                    .startObject("store_only")
                         .field("type", "json")
                         .field("index", false)
                         .field("store", true)
@@ -144,24 +148,34 @@ public void testEnableStore() throws Exception {
         assertEquals(mapping, mapper.mappingSource().toString());
 
         BytesReference doc = BytesReference.bytes(XContentFactory.jsonBuilder().startObject()
-                .startObject("field")
+                .startObject("store_only")
+                    .field("key", "value")
+                .endObject()
+                .startObject("store_and_index")
                     .field("key", "value")
                 .endObject()
             .endObject());
-
         ParsedDocument parsedDoc = mapper.parse(SourceToParse.source("test", "type", "1", doc, XContentType.JSON));
 
-        IndexableField[] fields = parsedDoc.rootDoc().getFields("field");
-        assertEquals(1, fields.length);
-        assertTrue(fields[0].fieldType().stored());
-
         // We make sure to pretty-print here, since the field is always stored in pretty-printed format.
         BytesReference storedValue = BytesReference.bytes(JsonXContent.contentBuilder()
             .prettyPrint()
             .startObject()
-                .field("key", "value")
+            .field("key", "value")
             .endObject());
-        assertEquals(storedValue.toBytesRef(), fields[0].binaryValue());
+
+        IndexableField[] storeOnly = parsedDoc.rootDoc().getFields("store_only");
+        assertEquals(1, storeOnly.length);
+
+        assertTrue(storeOnly[0].fieldType().stored());
+        assertEquals(storedValue.toBytesRef(), storeOnly[0].binaryValue());
+
+        IndexableField[] storeAndIndex = parsedDoc.rootDoc().getFields("store_and_index");
+        assertEquals(2, storeAndIndex.length);
+
+        assertTrue(storeAndIndex[0].fieldType().stored());
+        assertEquals(storedValue.toBytesRef(), storeAndIndex[0].binaryValue());
+        assertFalse(storeAndIndex[1].fieldType().stored());
     }
 
     public void testIndexOptions() throws IOException {

server/src/test/java/org/elasticsearch/index/mapper/JsonFieldParserTests.java

@@ -41,10 +41,7 @@ public class JsonFieldParserTests extends ESTestCase {
     @Before
     public void setUp() throws Exception {
         super.setUp();
-
-        MappedFieldType fieldType = new RootJsonFieldType();
-        fieldType.setName("field");
-        parser = new JsonFieldParser("field", "field._keyed", fieldType, Integer.MAX_VALUE);
+        parser = new JsonFieldParser("field", "field._keyed", Integer.MAX_VALUE, null);
     }
 
     public void testTextValues() throws Exception {
@@ -222,9 +219,9 @@ public void testIgnoreAbove() throws Exception {
 
         RootJsonFieldType fieldType = new RootJsonFieldType();
         fieldType.setName("field");
-        JsonFieldParser ignoreAboveParser = new JsonFieldParser("field", "field._keyed", fieldType, 10);
+        JsonFieldParser parserWithIgnoreAbove = new JsonFieldParser("field", "field._keyed", 10, null);
 
-        List<IndexableField> fields = ignoreAboveParser.parse(xContentParser);
+        List<IndexableField> fields = parserWithIgnoreAbove.parse(xContentParser);
         assertEquals(0, fields.size());
     }
 
@@ -236,13 +233,10 @@ public void testNullValues() throws Exception {
         assertEquals(0, fields.size());
 
         xContentParser = createXContentParser(input);
+        JsonFieldParser parserWithNullValue = new JsonFieldParser("field", "field._keyed",
+            Integer.MAX_VALUE, "placeholder");
 
-        RootJsonFieldType fieldType = new RootJsonFieldType();
-        fieldType.setName("field");
-        fieldType.setNullValue("placeholder");
-        JsonFieldParser nullValueParser = new JsonFieldParser("field", "field._keyed", fieldType, Integer.MAX_VALUE);
-
-        fields = nullValueParser.parse(xContentParser);
+        fields = parserWithNullValue.parse(xContentParser);
         assertEquals(2, fields.size());
 
         IndexableField field = fields.get(0);