allow to also store the all field

Author: kimchy

.idea/libraries/lucene.xml

@@ -47,6 +47,7 @@ dependencies {
     compile 'org.apache.lucene:lucene-core:3.0.1'
     compile 'org.apache.lucene:lucene-analyzers:3.0.1'
     compile 'org.apache.lucene:lucene-queries:3.0.1'
+    compile 'org.apache.lucene:lucene-fast-vector-highlighter:3.0.1'
 
     compile('jgroups:jgroups:2.9.0.GA') { transitive = false }
     compile('org.jboss.netty:netty:3.1.5.GA') { transitive = false }

modules/elasticsearch/build.gradle

@@ -20,13 +20,16 @@
 package org.elasticsearch.index.mapper.json;
 
 import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.Fieldable;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.mapper.AllFieldMapper;
+import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.MergeMappingException;
 import org.elasticsearch.util.json.JsonBuilder;
 import org.elasticsearch.util.lucene.Lucene;
+import org.elasticsearch.util.lucene.all.AllAnalyzer;
 
 import java.io.IOException;
 
@@ -61,6 +64,10 @@ public Builder enabled(boolean enabled) {
             return this;
         }
 
+        @Override public Builder store(Field.Store store) {
+            return super.store(store);
+        }
+
         @Override public Builder termVector(Field.TermVector termVector) {
             return super.termVector(termVector);
         }
@@ -74,21 +81,23 @@ public Builder enabled(boolean enabled) {
         }
 
         @Override public JsonAllFieldMapper build(BuilderContext context) {
-            return new JsonAllFieldMapper(name, termVector, omitNorms, omitTermFreqAndPositions,
+            return new JsonAllFieldMapper(name, store, termVector, omitNorms, omitTermFreqAndPositions,
                     indexAnalyzer, searchAnalyzer, enabled);
         }
     }
 
 
     private boolean enabled;
 
+    private AllAnalyzer allAnalyzer;
+
     public JsonAllFieldMapper() {
-        this(Defaults.NAME, Defaults.TERM_VECTOR, Defaults.OMIT_NORMS, Defaults.OMIT_TERM_FREQ_AND_POSITIONS, null, null, Defaults.ENABLED);
+        this(Defaults.NAME, Defaults.STORE, Defaults.TERM_VECTOR, Defaults.OMIT_NORMS, Defaults.OMIT_TERM_FREQ_AND_POSITIONS, null, null, Defaults.ENABLED);
     }
 
-    protected JsonAllFieldMapper(String name, Field.TermVector termVector, boolean omitNorms, boolean omitTermFreqAndPositions,
+    protected JsonAllFieldMapper(String name, Field.Store store, Field.TermVector termVector, boolean omitNorms, boolean omitTermFreqAndPositions,
                                  NamedAnalyzer indexAnalyzer, NamedAnalyzer searchAnalyzer, boolean enabled) {
-        super(new Names(name, name, name, name), Field.Index.ANALYZED, Field.Store.NO, termVector, 1.0f, omitNorms, omitTermFreqAndPositions,
+        super(new Names(name, name, name, name), Field.Index.ANALYZED, store, termVector, 1.0f, omitNorms, omitTermFreqAndPositions,
                 indexAnalyzer, searchAnalyzer);
         this.enabled = enabled;
     }
@@ -101,14 +110,30 @@ public boolean enabled() {
         if (!enabled) {
             return null;
         }
-        Analyzer analyzer = indexAnalyzer();
+        // reset the entries
+        jsonContext.allEntries().reset();
+
+        Analyzer analyzer = findAnalyzer(jsonContext.docMapper());
+        TokenStream tokenStream = allTokenStream(names.indexName(), jsonContext.allEntries(), analyzer);
+        if (stored()) {
+            // TODO when its possible to pass char[] to field, we can optimize
+            Field field = new Field(names.indexName(), jsonContext.allEntries().buildText(), store, index, termVector);
+            field.setTokenStream(tokenStream);
+            return field;
+        } else {
+            return new Field(names.indexName(), tokenStream, termVector);
+        }
+    }
+
+    private Analyzer findAnalyzer(DocumentMapper docMapper) {
+        Analyzer analyzer = indexAnalyzer;
         if (analyzer == null) {
-            analyzer = jsonContext.docMapper().indexAnalyzer();
+            analyzer = docMapper.indexAnalyzer();
             if (analyzer == null) {
                 analyzer = Lucene.STANDARD_ANALYZER;
             }
         }
-        return new Field(names.indexName(), allTokenStream(names.indexName(), jsonContext.allEntries().finishTexts(), analyzer), termVector);
+        return analyzer;
     }
 
     @Override public Void value(Fieldable field) {
@@ -138,6 +163,7 @@ public boolean enabled() {
     @Override public void toJson(JsonBuilder builder, Params params) throws IOException {
         builder.startObject(JSON_TYPE);
         builder.field("enabled", enabled);
+        builder.field("store", store.name().toLowerCase());
         builder.field("termVector", termVector.name().toLowerCase());
         if (indexAnalyzer != null && !indexAnalyzer.name().startsWith("_")) {
             builder.field("indexAnalyzer", indexAnalyzer.name());

modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonAllFieldMapper.java

@@ -24,6 +24,7 @@
 import org.codehaus.jackson.JsonFactory;
 import org.codehaus.jackson.JsonParser;
 import org.codehaus.jackson.JsonToken;
+import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.mapper.*;
 import org.elasticsearch.util.Nullable;
 import org.elasticsearch.util.Preconditions;
@@ -58,9 +59,9 @@ public static class Builder {
 
         private JsonAllFieldMapper allFieldMapper = new JsonAllFieldMapper();
 
-        private Analyzer indexAnalyzer;
+        private NamedAnalyzer indexAnalyzer;
 
-        private Analyzer searchAnalyzer;
+        private NamedAnalyzer searchAnalyzer;
 
         private final JsonObjectMapper rootObjectMapper;
 
@@ -107,7 +108,7 @@ public Builder mappingSource(String mappingSource) {
             return this;
         }
 
-        public Builder indexAnalyzer(Analyzer indexAnalyzer) {
+        public Builder indexAnalyzer(NamedAnalyzer indexAnalyzer) {
             this.indexAnalyzer = indexAnalyzer;
             return this;
         }
@@ -116,7 +117,7 @@ public boolean hasIndexAnalyzer() {
             return indexAnalyzer != null;
         }
 
-        public Builder searchAnalyzer(Analyzer searchAnalyzer) {
+        public Builder searchAnalyzer(NamedAnalyzer searchAnalyzer) {
             this.searchAnalyzer = searchAnalyzer;
             return this;
         }
@@ -393,6 +394,6 @@ void addFieldMapper(FieldMapper fieldMapper) {
     }
 
     @Override public void toJson(JsonBuilder builder, Params params) throws IOException {
-        rootObjectMapper.toJson(builder, params);
+        rootObjectMapper.toJson(builder, params, allFieldMapper);
     }
 }

modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonDocumentMapper.java

@@ -411,6 +411,10 @@ private void serializeValue(JsonParseContext jsonContext, String currentFieldNam
     }
 
     @Override public void toJson(JsonBuilder builder, Params params) throws IOException {
+        toJson(builder, params, null);
+    }
+
+    public void toJson(JsonBuilder builder, Params params, JsonMapper... additionalMappers) throws IOException {
         builder.startObject(name);
         builder.field("type", JSON_TYPE);
         builder.field("dynamic", dynamic);
@@ -434,6 +438,11 @@ private void serializeValue(JsonParseContext jsonContext, String currentFieldNam
                 mapper.toJson(builder, params);
             }
         }
+        if (additionalMappers != null) {
+            for (JsonMapper mapper : additionalMappers) {
+                mapper.toJson(builder, params);
+            }
+        }
 
         if (!mappers.isEmpty()) {
             builder.startObject("properties");

modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/json/JsonObjectMapper.java

@@ -0,0 +1,28 @@
+/*
+ * Licensed to Elastic Search and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Elastic Search licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.util.io;
+
+import java.io.Reader;
+
+/**
+ * @author kimchy (shay.banon)
+ */
+public abstract class CharSequenceReader extends Reader implements CharSequence {
+}

modules/elasticsearch/src/main/java/org/elasticsearch/util/io/CharSequenceReader.java

@@ -28,11 +28,32 @@
 /**
  * A similar class to {@link java.io.CharArrayWriter} allowing to get the underlying <tt>char[]</tt> buffer.
  *
- * @author kimchy (Shay Banon)
+ * @author kimchy (shay.banon)
  */
 @NotThreadSafe
 public class FastCharArrayWriter extends Writer {
 
+    /**
+     * A thread local based cache of {@link FastByteArrayOutputStream}.
+     */
+    public static class Cached {
+
+        private static final ThreadLocal<FastCharArrayWriter> cache = new ThreadLocal<FastCharArrayWriter>() {
+            @Override protected FastCharArrayWriter initialValue() {
+                return new FastCharArrayWriter();
+            }
+        };
+
+        /**
+         * Returns the cached thread local byte stream, with its internal stream cleared.
+         */
+        public static FastCharArrayWriter cached() {
+            FastCharArrayWriter os = cache.get();
+            os.reset();
+            return os;
+        }
+    }
+
     /**
      * The buffer where data is stored.
      */

modules/elasticsearch/src/main/java/org/elasticsearch/util/io/FastCharArrayWriter.java

@@ -27,10 +27,11 @@
 /**
  * A character stream whose source is a string that is <b>not thread safe</b>
  *
- * @author kimchy (Shay Banon)
+ * @author kimchy (shay.banon
+ *         )
  */
 @NotThreadSafe
-public class FastStringReader extends Reader {
+public class FastStringReader extends CharSequenceReader {
 
     private String str;
     private int length;
@@ -55,6 +56,18 @@ private void ensureOpen() throws IOException {
             throw new IOException("Stream closed");
     }
 
+    @Override public int length() {
+        return length;
+    }
+
+    @Override public char charAt(int index) {
+        return str.charAt(index);
+    }
+
+    @Override public CharSequence subSequence(int start, int end) {
+        return str.subSequence(start, end);
+    }
+
     /**
      * Reads a single character.
      *

modules/elasticsearch/src/main/java/org/elasticsearch/util/io/FastStringReader.java

@@ -0,0 +1,59 @@
+/*
+ * Licensed to Elastic Search and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Elastic Search licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.util.lucene.all;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.document.Fieldable;
+
+import java.io.IOException;
+import java.io.Reader;
+
+/**
+ * An all analyzer.
+ *
+ * @author kimchy (shay.banon)
+ */
+public class AllAnalyzer extends Analyzer {
+
+    private final Analyzer analyzer;
+
+    public AllAnalyzer(Analyzer analyzer) {
+        this.analyzer = analyzer;
+    }
+
+    @Override public TokenStream tokenStream(String fieldName, Reader reader) {
+        AllEntries allEntries = (AllEntries) reader;
+        return new AllTokenFilter(analyzer.tokenStream(fieldName, reader), allEntries);
+    }
+
+    @Override public TokenStream reusableTokenStream(String fieldName, Reader reader) throws IOException {
+        AllEntries allEntries = (AllEntries) reader;
+        return new AllTokenFilter(analyzer.reusableTokenStream(fieldName, reader), allEntries);
+    }
+
+    @Override public int getPositionIncrementGap(String fieldName) {
+        return analyzer.getPositionIncrementGap(fieldName);
+    }
+
+    @Override public int getOffsetGap(Fieldable field) {
+        return analyzer.getOffsetGap(field);
+    }
+}