Skip to content

Commit d927b2a

Browse files
DaveCTurnerDaveCTurner
committed
Close query cache on index service creation failure (elastic#48230)
Today it is possible that we create the `QueryCache` and then fail to create the owning `IndexService` and this means we do not close the `QueryCache` again. This commit addresses that leak. Fixes elastic#48186
1 parent 20d50c5 commit d927b2a

File tree

3 files changed

+116
-20
lines changed

3 files changed

+116
-20
lines changed

‎server/src/main/java/org/elasticsearch/index/IndexModule.java

Lines changed: 24 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import org.apache.lucene.search.similarities.Similarity;
2424
import org.apache.lucene.store.MMapDirectory;
2525
import org.apache.lucene.util.Constants;
26+
import org.apache.lucene.util.IOUtils;
2627
import org.apache.lucene.util.SetOnce;
2728
import org.elasticsearch.Version;
2829
import org.elasticsearch.client.Client;
@@ -35,6 +36,7 @@
3536
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
3637
import org.elasticsearch.env.NodeEnvironment;
3738
import org.elasticsearch.index.analysis.AnalysisRegistry;
39+
import org.elasticsearch.index.analysis.IndexAnalyzers;
3840
import org.elasticsearch.index.cache.query.DisabledQueryCache;
3941
import org.elasticsearch.index.cache.query.IndexQueryCache;
4042
import org.elasticsearch.index.cache.query.QueryCache;
@@ -388,22 +390,33 @@ public IndexService newIndexService(
388390
? (shard) -> null : indexSearcherWrapper.get();
389391
eventListener.beforeIndexCreated(indexSettings.getIndex(), indexSettings.getSettings());
390392
final IndexStore store = getIndexStore(indexSettings, indexStoreFactories);
391-
final QueryCache queryCache;
392-
if (indexSettings.getValue(INDEX_QUERY_CACHE_ENABLED_SETTING)) {
393-
BiFunction<IndexSettings, IndicesQueryCache, QueryCache> queryCacheProvider = forceQueryCacheProvider.get();
394-
if (queryCacheProvider == null) {
395-
queryCache = new IndexQueryCache(indexSettings, indicesQueryCache);
393+
QueryCache queryCache = null;
394+
IndexAnalyzers indexAnalyzers = null;
395+
boolean success = false;
396+
try {
397+
if (indexSettings.getValue(INDEX_QUERY_CACHE_ENABLED_SETTING)) {
398+
BiFunction<IndexSettings, IndicesQueryCache, QueryCache> queryCacheProvider = forceQueryCacheProvider.get();
399+
if (queryCacheProvider == null) {
400+
queryCache = new IndexQueryCache(indexSettings, indicesQueryCache);
401+
} else {
402+
queryCache = queryCacheProvider.apply(indexSettings, indicesQueryCache);
403+
}
396404
} else {
397-
queryCache = queryCacheProvider.apply(indexSettings, indicesQueryCache);
405+
queryCache = new DisabledQueryCache(indexSettings);
398406
}
399-
} else {
400-
queryCache = new DisabledQueryCache(indexSettings);
401-
}
402-
return new IndexService(indexSettings, environment, xContentRegistry,
407+
indexAnalyzers = analysisRegistry.build(indexSettings);
408+
final IndexService indexService = new IndexService(indexSettings, environment, xContentRegistry,
403409
new SimilarityService(indexSettings, scriptService, similarities),
404-
shardStoreDeleter, analysisRegistry, engineFactory, circuitBreakerService, bigArrays, threadPool, scriptService,
410+
shardStoreDeleter, indexAnalyzers, engineFactory, circuitBreakerService, bigArrays, threadPool, scriptService,
405411
client, queryCache, store, eventListener, searcherWrapperFactory, mapperRegistry,
406412
indicesFieldDataCache, searchOperationListeners, indexOperationListeners, namedWriteableRegistry);
413+
success = true;
414+
return indexService;
415+
} finally {
416+
if (success == false) {
417+
IOUtils.closeWhileHandlingException(queryCache, indexAnalyzers);
418+
}
419+
}
407420
}
408421

409422
private static IndexStore getIndexStore(

‎server/src/main/java/org/elasticsearch/index/IndexService.java

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,6 @@
4444
import org.elasticsearch.env.NodeEnvironment;
4545
import org.elasticsearch.env.ShardLock;
4646
import org.elasticsearch.env.ShardLockObtainFailedException;
47-
import org.elasticsearch.index.analysis.AnalysisRegistry;
4847
import org.elasticsearch.index.analysis.IndexAnalyzers;
4948
import org.elasticsearch.index.cache.IndexCache;
5049
import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
@@ -142,8 +141,7 @@ public IndexService(
142141
NamedXContentRegistry xContentRegistry,
143142
SimilarityService similarityService,
144143
ShardStoreDeleter shardStoreDeleter,
145-
AnalysisRegistry registry,
146-
EngineFactory engineFactory,
144+
IndexAnalyzers indexAnalyzers, EngineFactory engineFactory,
147145
CircuitBreakerService circuitBreakerService,
148146
BigArrays bigArrays,
149147
ThreadPool threadPool,
@@ -157,14 +155,14 @@ public IndexService(
157155
IndicesFieldDataCache indicesFieldDataCache,
158156
List<SearchOperationListener> searchOperationListeners,
159157
List<IndexingOperationListener> indexingOperationListeners,
160-
NamedWriteableRegistry namedWriteableRegistry) throws IOException {
158+
NamedWriteableRegistry namedWriteableRegistry) {
161159
super(indexSettings);
162160
this.indexSettings = indexSettings;
163161
this.xContentRegistry = xContentRegistry;
164162
this.similarityService = similarityService;
165163
this.namedWriteableRegistry = namedWriteableRegistry;
166164
this.circuitBreakerService = circuitBreakerService;
167-
this.mapperService = new MapperService(indexSettings, registry.build(indexSettings), xContentRegistry, similarityService,
165+
this.mapperService = new MapperService(indexSettings, indexAnalyzers, xContentRegistry, similarityService,
168166
mapperRegistry,
169167
// we parse all percolator queries as they would be parsed on shard 0
170168
() -> newQueryShardContext(0, null, System::currentTimeMillis, null));

‎server/src/test/java/org/elasticsearch/index/IndexModuleTests.java

Lines changed: 89 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
*/
1919
package org.elasticsearch.index;
2020

21+
import org.apache.lucene.analysis.Analyzer;
2122
import org.apache.lucene.index.AssertingDirectoryReader;
2223
import org.apache.lucene.index.DirectoryReader;
2324
import org.apache.lucene.index.FieldInvertState;
@@ -40,12 +41,15 @@
4041
import org.elasticsearch.common.settings.Settings;
4142
import org.elasticsearch.common.util.BigArrays;
4243
import org.elasticsearch.common.util.PageCacheRecycler;
44+
import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
4345
import org.elasticsearch.core.internal.io.IOUtils;
4446
import org.elasticsearch.env.Environment;
4547
import org.elasticsearch.env.NodeEnvironment;
4648
import org.elasticsearch.env.ShardLock;
4749
import org.elasticsearch.env.TestEnvironment;
4850
import org.elasticsearch.index.analysis.AnalysisRegistry;
51+
import org.elasticsearch.index.analysis.AnalyzerProvider;
52+
import org.elasticsearch.index.analysis.AnalyzerScope;
4953
import org.elasticsearch.index.cache.query.DisabledQueryCache;
5054
import org.elasticsearch.index.cache.query.IndexQueryCache;
5155
import org.elasticsearch.index.cache.query.QueryCache;
@@ -65,6 +69,7 @@
6569
import org.elasticsearch.index.store.IndexStore;
6670
import org.elasticsearch.indices.IndicesModule;
6771
import org.elasticsearch.indices.IndicesQueryCache;
72+
import org.elasticsearch.indices.analysis.AnalysisModule;
6873
import org.elasticsearch.indices.breaker.CircuitBreakerService;
6974
import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
7075
import org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason;
@@ -83,13 +88,17 @@
8388

8489
import java.io.IOException;
8590
import java.util.Collections;
91+
import java.util.HashSet;
8692
import java.util.Map;
93+
import java.util.Set;
8794
import java.util.concurrent.TimeUnit;
8895
import java.util.concurrent.atomic.AtomicBoolean;
8996
import java.util.function.Function;
9097

9198
import static java.util.Collections.emptyMap;
99+
import static java.util.Collections.singletonMap;
92100
import static org.hamcrest.Matchers.containsString;
101+
import static org.hamcrest.Matchers.empty;
93102
import static org.hamcrest.Matchers.hasToString;
94103
import static org.hamcrest.Matchers.instanceOf;
95104

@@ -351,11 +360,19 @@ public void testForceCustomQueryCache() throws IOException {
351360
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
352361
final IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("foo", settings);
353362
IndexModule module = new IndexModule(indexSettings, emptyAnalysisRegistry, new InternalEngineFactory(), Collections.emptyMap());
354-
module.forceQueryCacheProvider((a, b) -> new CustomQueryCache());
355-
expectThrows(AlreadySetException.class, () -> module.forceQueryCacheProvider((a, b) -> new CustomQueryCache()));
363+
final Set<CustomQueryCache> liveQueryCaches = new HashSet<>();
364+
module.forceQueryCacheProvider((a, b) -> {
365+
final CustomQueryCache customQueryCache = new CustomQueryCache(liveQueryCaches);
366+
liveQueryCaches.add(customQueryCache);
367+
return customQueryCache;
368+
});
369+
expectThrows(AlreadySetException.class, () -> module.forceQueryCacheProvider((a, b) -> {
370+
throw new AssertionError("never called");
371+
}));
356372
IndexService indexService = newIndexService(module);
357373
assertTrue(indexService.cache().query() instanceof CustomQueryCache);
358374
indexService.close("simon says", false);
375+
assertThat(liveQueryCaches, empty());
359376
}
360377

361378
public void testDefaultQueryCacheImplIsSelected() throws IOException {
@@ -376,12 +393,73 @@ public void testDisableQueryCacheHasPrecedenceOverForceQueryCache() throws IOExc
376393
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
377394
final IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("foo", settings);
378395
IndexModule module = new IndexModule(indexSettings, emptyAnalysisRegistry, new InternalEngineFactory(), Collections.emptyMap());
379-
module.forceQueryCacheProvider((a, b) -> new CustomQueryCache());
396+
module.forceQueryCacheProvider((a, b) -> new CustomQueryCache(null));
380397
IndexService indexService = newIndexService(module);
381398
assertTrue(indexService.cache().query() instanceof DisabledQueryCache);
382399
indexService.close("simon says", false);
383400
}
384401

402+
public void testCustomQueryCacheCleanedUpIfIndexServiceCreationFails() {
403+
Settings settings = Settings.builder()
404+
.put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
405+
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
406+
final IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("foo", settings);
407+
IndexModule module = new IndexModule(indexSettings, emptyAnalysisRegistry, new InternalEngineFactory(), Collections.emptyMap());
408+
final Set<CustomQueryCache> liveQueryCaches = new HashSet<>();
409+
module.forceQueryCacheProvider((a, b) -> {
410+
final CustomQueryCache customQueryCache = new CustomQueryCache(liveQueryCaches);
411+
liveQueryCaches.add(customQueryCache);
412+
return customQueryCache;
413+
});
414+
threadPool.shutdown(); // causes index service creation to fail
415+
expectThrows(EsRejectedExecutionException.class, () -> newIndexService(module));
416+
assertThat(liveQueryCaches, empty());
417+
}
418+
419+
public void testIndexAnalyzersCleanedUpIfIndexServiceCreationFails() {
420+
Settings settings = Settings.builder()
421+
.put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
422+
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
423+
final IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("foo", settings);
424+
425+
final HashSet<Analyzer> openAnalyzers = new HashSet<>();
426+
final AnalysisModule.AnalysisProvider<AnalyzerProvider<?>> analysisProvider = (i,e,n,s) -> new AnalyzerProvider<Analyzer>() {
427+
@Override
428+
public String name() {
429+
return "test";
430+
}
431+
432+
@Override
433+
public AnalyzerScope scope() {
434+
return AnalyzerScope.INDEX;
435+
}
436+
437+
@Override
438+
public Analyzer get() {
439+
final Analyzer analyzer = new Analyzer() {
440+
@Override
441+
protected TokenStreamComponents createComponents(String fieldName) {
442+
throw new AssertionError("should not be here");
443+
}
444+
445+
@Override
446+
public void close() {
447+
super.close();
448+
openAnalyzers.remove(this);
449+
}
450+
};
451+
openAnalyzers.add(analyzer);
452+
return analyzer;
453+
}
454+
};
455+
final AnalysisRegistry analysisRegistry = new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(),
456+
singletonMap("test", analysisProvider), emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap());
457+
IndexModule module = new IndexModule(indexSettings, analysisRegistry, new InternalEngineFactory(), Collections.emptyMap());
458+
threadPool.shutdown(); // causes index service creation to fail
459+
expectThrows(EsRejectedExecutionException.class, () -> newIndexService(module));
460+
assertThat(openAnalyzers, empty());
461+
}
462+
385463
public void testMmapNotAllowed() {
386464
String storeType = randomFrom(IndexModule.Type.HYBRIDFS.getSettingsKey(), IndexModule.Type.MMAPFS.getSettingsKey());
387465
final Settings settings = Settings.builder()
@@ -400,12 +478,19 @@ public void testMmapNotAllowed() {
400478

401479
class CustomQueryCache implements QueryCache {
402480

481+
private final Set<CustomQueryCache> liveQueryCaches;
482+
483+
CustomQueryCache(Set<CustomQueryCache> liveQueryCaches) {
484+
this.liveQueryCaches = liveQueryCaches;
485+
}
486+
403487
@Override
404488
public void clear(String reason) {
405489
}
406490

407491
@Override
408-
public void close() throws IOException {
492+
public void close() {
493+
assertTrue(liveQueryCaches == null || liveQueryCaches.remove(this));
409494
}
410495

411496
@Override

0 commit comments

Comments
 (0)