Skip to content

Commit 9be3c5a

Browse files
rmuirrmuir
committed
Merge branch 'master' into off_my_back
2 parents 2f168b8 + 7368231 commit 9be3c5a

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

43 files changed

+714
-190
lines changed

‎core/src/main/java/org/elasticsearch/common/lucene/index/ElasticsearchDirectoryReader.java

Lines changed: 36 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,8 @@
1818
*/
1919
package org.elasticsearch.common.lucene.index;
2020

21-
import org.apache.lucene.index.DirectoryReader;
22-
import org.apache.lucene.index.FilterDirectoryReader;
23-
import org.apache.lucene.index.FilterLeafReader;
24-
import org.apache.lucene.index.LeafReader;
21+
import org.apache.lucene.index.*;
22+
import org.elasticsearch.common.SuppressForbidden;
2523
import org.elasticsearch.index.shard.ShardId;
2624

2725
import java.io.IOException;
@@ -76,4 +74,38 @@ public LeafReader wrap(LeafReader reader) {
7674
}
7775
}
7876

77+
/**
78+
* Adds the given listener to the provided directory reader. The reader must contain an {@link ElasticsearchDirectoryReader} in it's hierarchy
79+
* otherwise we can't safely install the listener.
80+
*
81+
* @throws IllegalArgumentException if the reader doesn't contain an {@link ElasticsearchDirectoryReader} in it's hierarchy
82+
*/
83+
@SuppressForbidden(reason = "This is the only sane way to add a ReaderClosedListener")
84+
public static void addReaderCloseListener(DirectoryReader reader, IndexReader.ReaderClosedListener listener) {
85+
ElasticsearchDirectoryReader elasticsearchDirectoryReader = getElasticsearchDirectoryReader(reader);
86+
if (elasticsearchDirectoryReader != null) {
87+
assert reader.getCoreCacheKey() == elasticsearchDirectoryReader.getCoreCacheKey();
88+
elasticsearchDirectoryReader.addReaderClosedListener(listener);
89+
return;
90+
}
91+
throw new IllegalArgumentException("Can't install close listener reader is not an ElasticsearchDirectoryReader/ElasticsearchLeafReader");
92+
}
93+
94+
/**
95+
* Tries to unwrap the given reader until the first {@link ElasticsearchDirectoryReader} instance is found or <code>null</code> if no instance is found;
96+
*/
97+
public static ElasticsearchDirectoryReader getElasticsearchDirectoryReader(DirectoryReader reader) {
98+
if (reader instanceof FilterDirectoryReader) {
99+
if (reader instanceof ElasticsearchDirectoryReader) {
100+
return (ElasticsearchDirectoryReader) reader;
101+
} else {
102+
// We need to use FilterDirectoryReader#getDelegate and not FilterDirectoryReader#unwrap, because
103+
// If there are multiple levels of filtered leaf readers then with the unwrap() method it immediately
104+
// returns the most inner leaf reader and thus skipping of over any other filtered leaf reader that
105+
// may be instance of ElasticsearchLeafReader. This can cause us to miss the shardId.
106+
return getElasticsearchDirectoryReader(((FilterDirectoryReader) reader).getDelegate());
107+
}
108+
}
109+
return null;
110+
}
79111
}

‎core/src/main/java/org/elasticsearch/common/lucene/index/ElasticsearchLeafReader.java

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,7 @@
1818
*/
1919
package org.elasticsearch.common.lucene.index;
2020

21-
import org.apache.lucene.index.DirectoryReader;
22-
import org.apache.lucene.index.FilterDirectoryReader;
23-
import org.apache.lucene.index.FilterLeafReader;
24-
import org.apache.lucene.index.LeafReader;
21+
import org.apache.lucene.index.*;
2522
import org.elasticsearch.index.shard.ShardId;
2623

2724
/**
@@ -38,7 +35,7 @@ public final class ElasticsearchLeafReader extends FilterLeafReader {
3835
*
3936
* @param in specified base reader.
4037
*/
41-
ElasticsearchLeafReader(LeafReader in, ShardId shardId) {
38+
public ElasticsearchLeafReader(LeafReader in, ShardId shardId) {
4239
super(in);
4340
this.shardId = shardId;
4441
}
@@ -55,8 +52,18 @@ public Object getCoreCacheKey() {
5552
return in.getCoreCacheKey();
5653
}
5754

58-
@Override
59-
public Object getCombinedCoreAndDeletesKey() {
60-
return in.getCombinedCoreAndDeletesKey();
55+
public static ElasticsearchLeafReader getElasticsearchLeafReader(LeafReader reader) {
56+
if (reader instanceof FilterLeafReader) {
57+
if (reader instanceof ElasticsearchLeafReader) {
58+
return (ElasticsearchLeafReader) reader;
59+
} else {
60+
// We need to use FilterLeafReader#getDelegate and not FilterLeafReader#unwrap, because
61+
// If there are multiple levels of filtered leaf readers then with the unwrap() method it immediately
62+
// returns the most inner leaf reader and thus skipping of over any other filtered leaf reader that
63+
// may be instance of ElasticsearchLeafReader. This can cause us to miss the shardId.
64+
return getElasticsearchLeafReader(((FilterLeafReader) reader).getDelegate());
65+
}
66+
}
67+
return null;
6168
}
6269
}

‎core/src/main/java/org/elasticsearch/common/lucene/uid/Versions.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
package org.elasticsearch.common.lucene.uid;
2121

22+
import org.apache.lucene.index.DirectoryReader;
2223
import org.apache.lucene.index.IndexReader;
2324
import org.apache.lucene.index.LeafReader;
2425
import org.apache.lucene.index.LeafReader.CoreClosedListener;

‎core/src/main/java/org/elasticsearch/index/engine/Engine.java

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -596,6 +596,13 @@ public IndexReader reader() {
596596
return searcher.getIndexReader();
597597
}
598598

599+
public DirectoryReader getDirectoryReader() {
600+
if (reader() instanceof DirectoryReader) {
601+
return (DirectoryReader) reader();
602+
}
603+
throw new IllegalStateException("Can't use " + reader().getClass() + " as a directory reader");
604+
}
605+
599606
public IndexSearcher searcher() {
600607
return searcher;
601608
}

‎core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
import org.elasticsearch.common.lucene.LoggerInfoStream;
4040
import org.elasticsearch.common.lucene.Lucene;
4141
import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader;
42+
import org.elasticsearch.common.lucene.index.ElasticsearchLeafReader;
4243
import org.elasticsearch.common.lucene.uid.Versions;
4344
import org.elasticsearch.common.math.MathUtils;
4445
import org.elasticsearch.common.settings.Settings;
@@ -905,9 +906,10 @@ private IndexWriter createWriter(boolean create) throws IOException {
905906
@Override
906907
public void warm(LeafReader reader) throws IOException {
907908
try {
908-
assert isMergedSegment(reader);
909+
LeafReader esLeafReader = new ElasticsearchLeafReader(reader, shardId);
910+
assert isMergedSegment(esLeafReader);
909911
if (warmer != null) {
910-
final Engine.Searcher searcher = new Searcher("warmer", searcherFactory.newSearcher(reader, null));
912+
final Engine.Searcher searcher = new Searcher("warmer", searcherFactory.newSearcher(esLeafReader, null));
911913
final IndicesWarmer.WarmerContext context = new IndicesWarmer.WarmerContext(shardId, searcher);
912914
warmer.warmNewReaders(context);
913915
}
@@ -949,6 +951,12 @@ final static class SearchFactory extends EngineSearcherFactory {
949951
@Override
950952
public IndexSearcher newSearcher(IndexReader reader, IndexReader previousReader) throws IOException {
951953
IndexSearcher searcher = super.newSearcher(reader, previousReader);
954+
if (reader instanceof LeafReader && isMergedSegment((LeafReader)reader)) {
955+
// we call newSearcher from the IndexReaderWarmer which warms segments during merging
956+
// in that case the reader is a LeafReader and all we need to do is to build a new Searcher
957+
// and return it since it does it's own warming for that particular reader.
958+
return searcher;
959+
}
952960
if (warmer != null) {
953961
// we need to pass a custom searcher that does not release anything on Engine.Search Release,
954962
// we will release explicitly
@@ -986,10 +994,11 @@ public IndexSearcher newSearcher(IndexReader reader, IndexReader previousReader)
986994
}
987995

988996
if (newSearcher != null) {
989-
IndicesWarmer.WarmerContext context = new IndicesWarmer.WarmerContext(shardId, new Searcher("warmer", newSearcher));
997+
IndicesWarmer.WarmerContext context = new IndicesWarmer.WarmerContext(shardId, new Searcher("new_reader_warming", newSearcher));
990998
warmer.warmNewReaders(context);
991999
}
992-
warmer.warmTopReader(new IndicesWarmer.WarmerContext(shardId, new Searcher("warmer", searcher)));
1000+
assert searcher.getIndexReader() instanceof ElasticsearchDirectoryReader : "this class needs an ElasticsearchDirectoryReader but got: " + searcher.getIndexReader().getClass();
1001+
warmer.warmTopReader(new IndicesWarmer.WarmerContext(shardId, new Searcher("top_reader_warming", searcher)));
9931002
} catch (Throwable e) {
9941003
if (isEngineClosed.get() == false) {
9951004
logger.warn("failed to prepare/warm", e);

‎core/src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
package org.elasticsearch.index.fielddata;
2121

22+
import org.apache.lucene.index.DirectoryReader;
2223
import org.apache.lucene.index.IndexReader;
2324
import org.apache.lucene.index.LeafReaderContext;
2425
import org.apache.lucene.search.DocIdSet;
@@ -235,11 +236,11 @@ IndexFieldData<?> build(Index index, @IndexSettings Settings indexSettings, Mapp
235236
CircuitBreakerService breakerService, MapperService mapperService);
236237
}
237238

238-
public static interface Global<FD extends AtomicFieldData> extends IndexFieldData<FD> {
239+
interface Global<FD extends AtomicFieldData> extends IndexFieldData<FD> {
239240

240-
IndexFieldData<FD> loadGlobal(IndexReader indexReader);
241+
IndexFieldData<FD> loadGlobal(DirectoryReader indexReader);
241242

242-
IndexFieldData<FD> localGlobalDirect(IndexReader indexReader) throws Exception;
243+
IndexFieldData<FD> localGlobalDirect(DirectoryReader indexReader) throws Exception;
243244

244245
}
245246

‎core/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataCache.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,10 @@
1919

2020
package org.elasticsearch.index.fielddata;
2121

22+
import org.apache.lucene.index.DirectoryReader;
2223
import org.apache.lucene.index.LeafReaderContext;
2324
import org.apache.lucene.index.IndexReader;
2425
import org.apache.lucene.util.Accountable;
25-
import org.elasticsearch.index.mapper.FieldMapper;
2626
import org.elasticsearch.index.mapper.MappedFieldType;
2727
import org.elasticsearch.index.shard.ShardId;
2828

@@ -33,7 +33,7 @@ public interface IndexFieldDataCache {
3333

3434
<FD extends AtomicFieldData, IFD extends IndexFieldData<FD>> FD load(LeafReaderContext context, IFD indexFieldData) throws Exception;
3535

36-
<FD extends AtomicFieldData, IFD extends IndexFieldData.Global<FD>> IFD load(final IndexReader indexReader, final IFD indexFieldData) throws Exception;
36+
<FD extends AtomicFieldData, IFD extends IndexFieldData.Global<FD>> IFD load(final DirectoryReader indexReader, final IFD indexFieldData) throws Exception;
3737

3838
/**
3939
* Clears all the field data stored cached in on this index.
@@ -67,7 +67,7 @@ public <FD extends AtomicFieldData, IFD extends IndexFieldData<FD>> FD load(Leaf
6767

6868
@Override
6969
@SuppressWarnings("unchecked")
70-
public <FD extends AtomicFieldData, IFD extends IndexFieldData.Global<FD>> IFD load(IndexReader indexReader, IFD indexFieldData) throws Exception {
70+
public <FD extends AtomicFieldData, IFD extends IndexFieldData.Global<FD>> IFD load(DirectoryReader indexReader, IFD indexFieldData) throws Exception {
7171
return (IFD) indexFieldData.localGlobalDirect(indexReader);
7272
}
7373

‎core/src/main/java/org/elasticsearch/index/fielddata/IndexOrdinalsFieldData.java

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
package org.elasticsearch.index.fielddata;
2121

22+
import org.apache.lucene.index.DirectoryReader;
2223
import org.apache.lucene.index.IndexReader;
2324

2425

@@ -33,12 +34,12 @@ public interface IndexOrdinalsFieldData extends IndexFieldData.Global<AtomicOrdi
3334
* potentially from a cache.
3435
*/
3536
@Override
36-
IndexOrdinalsFieldData loadGlobal(IndexReader indexReader);
37+
IndexOrdinalsFieldData loadGlobal(DirectoryReader indexReader);
3738

3839
/**
3940
* Load a global view of the ordinals for the given {@link IndexReader}.
4041
*/
4142
@Override
42-
IndexOrdinalsFieldData localGlobalDirect(IndexReader indexReader) throws Exception;
43+
IndexOrdinalsFieldData localGlobalDirect(DirectoryReader indexReader) throws Exception;
4344

4445
}

‎core/src/main/java/org/elasticsearch/index/fielddata/IndexParentChildFieldData.java

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
package org.elasticsearch.index.fielddata;
2121

22+
import org.apache.lucene.index.DirectoryReader;
2223
import org.apache.lucene.index.IndexReader;
2324

2425

@@ -34,12 +35,12 @@ public interface IndexParentChildFieldData extends IndexFieldData.Global<AtomicP
3435
* potentially from a cache.
3536
*/
3637
@Override
37-
IndexParentChildFieldData loadGlobal(IndexReader indexReader);
38+
IndexParentChildFieldData loadGlobal(DirectoryReader indexReader);
3839

3940
/**
4041
* Load a global view of the ordinals for the given {@link IndexReader}.
4142
*/
4243
@Override
43-
IndexParentChildFieldData localGlobalDirect(IndexReader indexReader) throws Exception;
44+
IndexParentChildFieldData localGlobalDirect(DirectoryReader indexReader) throws Exception;
4445

4546
}

‎core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsIndexFieldData.java

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
*/
1919
package org.elasticsearch.index.fielddata.ordinals;
2020

21+
import org.apache.lucene.index.DirectoryReader;
2122
import org.apache.lucene.index.IndexReader;
2223
import org.apache.lucene.index.LeafReaderContext;
2324
import org.apache.lucene.util.Accountable;
@@ -59,12 +60,12 @@ public AtomicOrdinalsFieldData loadDirect(LeafReaderContext context) throws Exce
5960
}
6061

6162
@Override
62-
public IndexOrdinalsFieldData loadGlobal(IndexReader indexReader) {
63+
public IndexOrdinalsFieldData loadGlobal(DirectoryReader indexReader) {
6364
return this;
6465
}
6566

6667
@Override
67-
public IndexOrdinalsFieldData localGlobalDirect(IndexReader indexReader) throws Exception {
68+
public IndexOrdinalsFieldData localGlobalDirect(DirectoryReader indexReader) throws Exception {
6869
return this;
6970
}
7071

‎core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexOrdinalsFieldData.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ public XFieldComparatorSource comparatorSource(@Nullable Object missingValue, Mu
5959
}
6060

6161
@Override
62-
public IndexOrdinalsFieldData loadGlobal(IndexReader indexReader) {
62+
public IndexOrdinalsFieldData loadGlobal(DirectoryReader indexReader) {
6363
if (indexReader.leaves().size() <= 1) {
6464
// ordinals are already global
6565
return this;
@@ -76,7 +76,7 @@ public IndexOrdinalsFieldData loadGlobal(IndexReader indexReader) {
7676
}
7777

7878
@Override
79-
public IndexOrdinalsFieldData localGlobalDirect(IndexReader indexReader) throws Exception {
79+
public IndexOrdinalsFieldData localGlobalDirect(DirectoryReader indexReader) throws Exception {
8080
return GlobalOrdinalsBuilder.build(indexReader, this, indexSettings, breakerService, logger);
8181
}
8282

‎core/src/main/java/org/elasticsearch/index/fielddata/plain/IndexIndexFieldData.java

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -19,11 +19,7 @@
1919

2020
package org.elasticsearch.index.fielddata.plain;
2121

22-
import org.apache.lucene.index.DocValues;
23-
import org.apache.lucene.index.IndexReader;
24-
import org.apache.lucene.index.LeafReaderContext;
25-
import org.apache.lucene.index.RandomAccessOrds;
26-
import org.apache.lucene.index.SortedDocValues;
22+
import org.apache.lucene.index.*;
2723
import org.apache.lucene.util.Accountable;
2824
import org.apache.lucene.util.BytesRef;
2925
import org.elasticsearch.common.settings.Settings;
@@ -123,12 +119,12 @@ public AtomicOrdinalsFieldData loadDirect(LeafReaderContext context)
123119
}
124120

125121
@Override
126-
public IndexOrdinalsFieldData loadGlobal(IndexReader indexReader) {
122+
public IndexOrdinalsFieldData loadGlobal(DirectoryReader indexReader) {
127123
return this;
128124
}
129125

130126
@Override
131-
public IndexOrdinalsFieldData localGlobalDirect(IndexReader indexReader) throws Exception {
127+
public IndexOrdinalsFieldData localGlobalDirect(DirectoryReader indexReader) throws Exception {
132128
return loadGlobal(indexReader);
133129
}
134130

‎core/src/main/java/org/elasticsearch/index/fielddata/plain/ParentChildIndexFieldData.java

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,7 @@ public IndexFieldData<?> build(Index index, @IndexSettings Settings indexSetting
135135
}
136136

137137
@Override
138-
public IndexParentChildFieldData loadGlobal(IndexReader indexReader) {
138+
public IndexParentChildFieldData loadGlobal(DirectoryReader indexReader) {
139139
if (indexReader.leaves().size() <= 1) {
140140
// ordinals are already global
141141
return this;
@@ -170,7 +170,7 @@ public OrdinalMapAndAtomicFieldData(OrdinalMap ordMap, AtomicParentChildFieldDat
170170
}
171171

172172
@Override
173-
public IndexParentChildFieldData localGlobalDirect(IndexReader indexReader) throws Exception {
173+
public IndexParentChildFieldData localGlobalDirect(DirectoryReader indexReader) throws Exception {
174174
final long startTime = System.nanoTime();
175175

176176
long ramBytesUsed = 0;
@@ -347,15 +347,15 @@ public Collection<Accountable> getChildResources() {
347347
}
348348

349349
@Override
350-
public IndexParentChildFieldData loadGlobal(IndexReader indexReader) {
350+
public IndexParentChildFieldData loadGlobal(DirectoryReader indexReader) {
351351
if (indexReader.getCoreCacheKey() == reader.getCoreCacheKey()) {
352352
return this;
353353
}
354354
throw new IllegalStateException();
355355
}
356356

357357
@Override
358-
public IndexParentChildFieldData localGlobalDirect(IndexReader indexReader) throws Exception {
358+
public IndexParentChildFieldData localGlobalDirect(DirectoryReader indexReader) throws Exception {
359359
return loadGlobal(indexReader);
360360
}
361361

‎core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVOrdinalsIndexFieldData.java

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
package org.elasticsearch.index.fielddata.plain;
2121

22+
import org.apache.lucene.index.DirectoryReader;
2223
import org.apache.lucene.index.LeafReaderContext;
2324
import org.apache.lucene.index.IndexReader;
2425
import org.elasticsearch.ElasticsearchException;
@@ -61,7 +62,7 @@ public AtomicOrdinalsFieldData loadDirect(LeafReaderContext context) throws Exce
6162
}
6263

6364
@Override
64-
public IndexOrdinalsFieldData loadGlobal(IndexReader indexReader) {
65+
public IndexOrdinalsFieldData loadGlobal(DirectoryReader indexReader) {
6566
if (indexReader.leaves().size() <= 1) {
6667
// ordinals are already global
6768
return this;
@@ -78,7 +79,7 @@ public IndexOrdinalsFieldData loadGlobal(IndexReader indexReader) {
7879
}
7980

8081
@Override
81-
public IndexOrdinalsFieldData localGlobalDirect(IndexReader indexReader) throws Exception {
82+
public IndexOrdinalsFieldData localGlobalDirect(DirectoryReader indexReader) throws Exception {
8283
return GlobalOrdinalsBuilder.build(indexReader, this, indexSettings, breakerService, logger);
8384
}
8485
}

0 commit comments

Comments
 (0)