Skip to content

Commit 41ddbd4

Browse files
tlrxtlrx
committed
Allow to prewarm the cache for searchable snapshot shards (elastic#55322)
Relates elastic#50999
1 parent 75f8d8b commit 41ddbd4

File tree

13 files changed

+606
-51
lines changed

13 files changed

+606
-51
lines changed

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsConstants.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,4 +34,6 @@ public static boolean isSearchableSnapshotStore(Settings indexSettings) {
3434
return SEARCHABLE_SNAPSHOTS_FEATURE_ENABLED
3535
&& SNAPSHOT_DIRECTORY_FACTORY_KEY.equals(INDEX_STORE_TYPE_SETTING.get(indexSettings));
3636
}
37+
38+
public static final String SEARCHABLE_SNAPSHOTS_THREAD_POOL_NAME = "searchable_snapshots";
3739
}

‎x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/index/store/BaseSearchableSnapshotIndexInput.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot.FileInfo;
1313
import org.elasticsearch.index.snapshots.blobstore.SlicedInputStream;
1414
import org.elasticsearch.threadpool.ThreadPool;
15+
import org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsConstants;
1516

1617
import java.io.IOException;
1718
import java.io.InputStream;
@@ -136,6 +137,9 @@ protected final boolean assertCurrentThreadMayAccessBlobStore() {
136137
|| threadName.contains('[' + ThreadPool.Names.SEARCH + ']')
137138
|| threadName.contains('[' + ThreadPool.Names.SEARCH_THROTTLED + ']')
138139

140+
// Cache prewarming runs on a dedicated thread pool.
141+
|| threadName.contains('[' + SearchableSnapshotsConstants.SEARCHABLE_SNAPSHOTS_THREAD_POOL_NAME + ']')
142+
139143
// Today processExistingRecoveries considers all shards and constructs a shard store snapshot on this thread, this needs
140144
// addressing. TODO NORELEASE
141145
|| threadName.contains('[' + ThreadPool.Names.FETCH_SHARD_STORE + ']')

‎x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/index/store/SearchableSnapshotDirectory.java

Lines changed: 99 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,9 @@
55
*/
66
package org.elasticsearch.index.store;
77

8+
import org.apache.logging.log4j.LogManager;
9+
import org.apache.logging.log4j.Logger;
10+
import org.apache.logging.log4j.message.ParameterizedMessage;
811
import org.apache.lucene.index.IndexFileNames;
912
import org.apache.lucene.store.BaseDirectory;
1013
import org.apache.lucene.store.Directory;
@@ -18,8 +21,12 @@
1821
import org.elasticsearch.common.blobstore.BlobContainer;
1922
import org.elasticsearch.common.lucene.store.ByteArrayIndexInput;
2023
import org.elasticsearch.common.settings.Settings;
24+
import org.elasticsearch.common.unit.TimeValue;
2125
import org.elasticsearch.common.util.LazyInitializable;
26+
import org.elasticsearch.common.util.concurrent.AbstractRunnable;
2227
import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
28+
import org.elasticsearch.common.util.concurrent.CountDown;
29+
import org.elasticsearch.core.internal.io.IOUtils;
2330
import org.elasticsearch.index.IndexSettings;
2431
import org.elasticsearch.index.shard.ShardId;
2532
import org.elasticsearch.index.shard.ShardPath;
@@ -47,18 +54,22 @@
4754
import java.util.Map;
4855
import java.util.Objects;
4956
import java.util.Set;
57+
import java.util.concurrent.Executor;
5058
import java.util.concurrent.atomic.AtomicBoolean;
5159
import java.util.function.LongSupplier;
5260
import java.util.function.Supplier;
61+
import java.util.stream.Collectors;
5362

5463
import static org.apache.lucene.store.BufferedIndexInput.bufferSize;
5564
import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots.SNAPSHOT_CACHE_ENABLED_SETTING;
5665
import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots.SNAPSHOT_CACHE_EXCLUDED_FILE_TYPES_SETTING;
66+
import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots.SNAPSHOT_CACHE_PREWARM_ENABLED_SETTING;
5767
import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots.SNAPSHOT_INDEX_ID_SETTING;
5868
import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots.SNAPSHOT_REPOSITORY_SETTING;
5969
import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots.SNAPSHOT_SNAPSHOT_ID_SETTING;
6070
import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots.SNAPSHOT_SNAPSHOT_NAME_SETTING;
6171
import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots.SNAPSHOT_UNCACHED_CHUNK_SIZE_SETTING;
72+
import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsConstants.SEARCHABLE_SNAPSHOTS_THREAD_POOL_NAME;
6273

6374
/**
6475
* Implementation of {@link Directory} that exposes files from a snapshot as a Lucene directory. Because snapshot are immutable this
@@ -73,15 +84,19 @@
7384
*/
7485
public class SearchableSnapshotDirectory extends BaseDirectory {
7586

87+
private static final Logger logger = LogManager.getLogger(SearchableSnapshotDirectory.class);
88+
7689
private final Supplier<BlobContainer> blobContainerSupplier;
7790
private final Supplier<BlobStoreIndexShardSnapshot> snapshotSupplier;
7891
private final SnapshotId snapshotId;
7992
private final IndexId indexId;
8093
private final ShardId shardId;
8194
private final LongSupplier statsCurrentTimeNanosSupplier;
8295
private final Map<String, IndexInputStats> stats;
96+
private final ThreadPool threadPool;
8397
private final CacheService cacheService;
8498
private final boolean useCache;
99+
private final boolean prewarmCache;
85100
private final Set<String> excludedFileTypes;
86101
private final long uncachedChunkSize; // if negative use BlobContainer#readBlobPreferredLength, see #getUncachedChunkSize()
87102
private final Path cacheDir;
@@ -101,7 +116,8 @@ public SearchableSnapshotDirectory(
101116
Settings indexSettings,
102117
LongSupplier currentTimeNanosSupplier,
103118
CacheService cacheService,
104-
Path cacheDir
119+
Path cacheDir,
120+
ThreadPool threadPool
105121
) {
106122
super(new SingleInstanceLockFactory());
107123
this.snapshotSupplier = Objects.requireNonNull(snapshot);
@@ -115,8 +131,10 @@ public SearchableSnapshotDirectory(
115131
this.cacheDir = Objects.requireNonNull(cacheDir);
116132
this.closed = new AtomicBoolean(false);
117133
this.useCache = SNAPSHOT_CACHE_ENABLED_SETTING.get(indexSettings);
134+
this.prewarmCache = useCache ? SNAPSHOT_CACHE_PREWARM_ENABLED_SETTING.get(indexSettings) : false;
118135
this.excludedFileTypes = new HashSet<>(SNAPSHOT_CACHE_EXCLUDED_FILE_TYPES_SETTING.get(indexSettings));
119136
this.uncachedChunkSize = SNAPSHOT_UNCACHED_CHUNK_SIZE_SETTING.get(indexSettings).getBytes();
137+
this.threadPool = threadPool;
120138
this.loaded = false;
121139
assert invariant();
122140
}
@@ -142,6 +160,7 @@ protected final boolean assertCurrentThreadMayLoadSnapshot() {
142160
* @return true if the snapshot was loaded by executing this method, false otherwise
143161
*/
144162
public boolean loadSnapshot() {
163+
assert assertCurrentThreadMayLoadSnapshot();
145164
boolean alreadyLoaded = this.loaded;
146165
if (alreadyLoaded == false) {
147166
synchronized (this) {
@@ -150,10 +169,10 @@ public boolean loadSnapshot() {
150169
this.blobContainer = blobContainerSupplier.get();
151170
this.snapshot = snapshotSupplier.get();
152171
this.loaded = true;
172+
prewarmCache();
153173
}
154174
}
155175
}
156-
assert assertCurrentThreadMayLoadSnapshot();
157176
assert invariant();
158177
return alreadyLoaded == false;
159178
}
@@ -300,7 +319,7 @@ public IndexInput openInput(final String name, final IOContext context) throws I
300319

301320
final IndexInputStats inputStats = stats.computeIfAbsent(name, n -> createIndexInputStats(fileInfo.length()));
302321
if (useCache && isExcludedFromCache(name) == false) {
303-
return new CachedBlobContainerIndexInput(this, fileInfo, context, inputStats);
322+
return new CachedBlobContainerIndexInput(this, fileInfo, context, inputStats, cacheService.getRangeSize());
304323
} else {
305324
return new DirectBlobContainerIndexInput(
306325
blobContainer(),
@@ -331,12 +350,86 @@ public String toString() {
331350
return this.getClass().getSimpleName() + "@snapshotId=" + snapshotId + " lockFactory=" + lockFactory;
332351
}
333352

353+
private void prewarmCache() {
354+
if (prewarmCache) {
355+
final List<BlobStoreIndexShardSnapshot.FileInfo> cacheFiles = snapshot().indexFiles()
356+
.stream()
357+
.filter(file -> file.metadata().hashEqualsContents() == false)
358+
.filter(file -> isExcludedFromCache(file.physicalName()) == false)
359+
.collect(Collectors.toList());
360+
361+
final Executor executor = threadPool.executor(SEARCHABLE_SNAPSHOTS_THREAD_POOL_NAME);
362+
logger.debug("{} warming shard cache for [{}] files", shardId, cacheFiles.size());
363+
364+
for (BlobStoreIndexShardSnapshot.FileInfo cacheFile : cacheFiles) {
365+
final String fileName = cacheFile.physicalName();
366+
try {
367+
final IndexInput input = openInput(fileName, CachedBlobContainerIndexInput.CACHE_WARMING_CONTEXT);
368+
assert input instanceof CachedBlobContainerIndexInput : "expected cached index input but got " + input.getClass();
369+
370+
final long numberOfParts = cacheFile.numberOfParts();
371+
final CountDown countDown = new CountDown(Math.toIntExact(numberOfParts));
372+
for (long p = 0; p < numberOfParts; p++) {
373+
final int part = Math.toIntExact(p);
374+
// TODO use multiple workers to warm each part instead of filling the thread pool
375+
executor.execute(new AbstractRunnable() {
376+
@Override
377+
protected void doRun() throws Exception {
378+
ensureOpen();
379+
380+
logger.trace("warming cache for [{}] part [{}/{}]", fileName, part, numberOfParts);
381+
final long startTimeInNanos = statsCurrentTimeNanosSupplier.getAsLong();
382+
383+
final CachedBlobContainerIndexInput cachedIndexInput = (CachedBlobContainerIndexInput) input.clone();
384+
final int bytesRead = cachedIndexInput.prefetchPart(part); // TODO does not include any rate limitation
385+
assert bytesRead == cacheFile.partBytes(part);
386+
387+
logger.trace(
388+
() -> new ParameterizedMessage(
389+
"part [{}/{}] of [{}] warmed in [{}] ms",
390+
part,
391+
numberOfParts,
392+
fileName,
393+
TimeValue.timeValueNanos(statsCurrentTimeNanosSupplier.getAsLong() - startTimeInNanos).millis()
394+
)
395+
);
396+
}
397+
398+
@Override
399+
public void onFailure(Exception e) {
400+
logger.trace(
401+
() -> new ParameterizedMessage(
402+
"failed to warm cache for [{}] part [{}/{}]",
403+
fileName,
404+
part,
405+
numberOfParts
406+
),
407+
e
408+
);
409+
}
410+
411+
@Override
412+
public void onAfter() {
413+
if (countDown.countDown()) {
414+
IOUtils.closeWhileHandlingException(input);
415+
}
416+
}
417+
});
418+
}
419+
} catch (IOException e) {
420+
logger.trace(() -> new ParameterizedMessage("failed to warm cache for [{}]", fileName), e);
421+
}
422+
}
423+
}
424+
}
425+
334426
public static Directory create(
335427
RepositoriesService repositories,
336428
CacheService cache,
337429
IndexSettings indexSettings,
338430
ShardPath shardPath,
339-
LongSupplier currentTimeNanosSupplier
431+
LongSupplier currentTimeNanosSupplier,
432+
ThreadPool threadPool
340433
) throws IOException {
341434

342435
final Repository repository = repositories.repository(SNAPSHOT_REPOSITORY_SETTING.get(indexSettings.getSettings()));
@@ -371,7 +464,8 @@ public static Directory create(
371464
indexSettings.getSettings(),
372465
currentTimeNanosSupplier,
373466
cache,
374-
cacheDir
467+
cacheDir,
468+
threadPool
375469
)
376470
);
377471
}

‎x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/index/store/cache/CacheFile.java

Lines changed: 14 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,6 @@ protected void closeInternal() {
5151
private final ReleasableLock readLock;
5252

5353
private final SparseFileTracker tracker;
54-
private final int rangeSize;
5554
private final String description;
5655
private final Path file;
5756

@@ -61,12 +60,11 @@ protected void closeInternal() {
6160
@Nullable // if evicted, or there are no listeners
6261
private volatile FileChannel channel;
6362

64-
public CacheFile(String description, long length, Path file, int rangeSize) {
63+
public CacheFile(String description, long length, Path file) {
6564
this.tracker = new SparseFileTracker(file.toString(), length);
6665
this.description = Objects.requireNonNull(description);
6766
this.file = Objects.requireNonNull(file);
6867
this.listeners = new HashSet<>();
69-
this.rangeSize = rangeSize;
7068
this.evicted = false;
7169

7270
final ReentrantReadWriteLock cacheLock = new ReentrantReadWriteLock();
@@ -249,41 +247,35 @@ private void ensureOpen() {
249247
}
250248

251249
CompletableFuture<Integer> fetchRange(
252-
long position,
250+
long start,
251+
long end,
253252
CheckedBiFunction<Long, Long, Integer, IOException> onRangeAvailable,
254253
CheckedBiConsumer<Long, Long, IOException> onRangeMissing
255254
) {
256255
final CompletableFuture<Integer> future = new CompletableFuture<>();
257256
try {
258-
if (position < 0 || position > tracker.getLength()) {
259-
throw new IllegalArgumentException("Wrong read position [" + position + "]");
257+
if (start < 0 || start > tracker.getLength() || start > end || end > tracker.getLength()) {
258+
throw new IllegalArgumentException(
259+
"Invalid range [start=" + start + ", end=" + end + "] for length [" + tracker.getLength() + ']'
260+
);
260261
}
261-
262262
ensureOpen();
263-
final long rangeStart = (position / rangeSize) * rangeSize;
264-
final long rangeEnd = Math.min(rangeStart + rangeSize, tracker.getLength());
265-
266263
final List<SparseFileTracker.Gap> gaps = tracker.waitForRange(
267-
rangeStart,
268-
rangeEnd,
264+
start,
265+
end,
269266
ActionListener.wrap(
270-
rangeReady -> future.complete(onRangeAvailable.apply(rangeStart, rangeEnd)),
267+
rangeReady -> future.complete(onRangeAvailable.apply(start, end)),
271268
rangeFailure -> future.completeExceptionally(rangeFailure)
272269
)
273270
);
274271

275-
if (gaps.size() > 0) {
276-
final SparseFileTracker.Gap range = gaps.get(0);
277-
assert gaps.size() == 1 : "expected 1 range to fetch but got " + gaps.size();
278-
assert range.start == rangeStart : "range/gap start mismatch (" + range.start + ',' + rangeStart + ')';
279-
assert range.end == rangeEnd : "range/gap end mismatch (" + range.end + ',' + rangeEnd + ')';
280-
272+
for (SparseFileTracker.Gap gap : gaps) {
281273
try {
282274
ensureOpen();
283-
onRangeMissing.accept(rangeStart, rangeEnd);
284-
range.onResponse(null);
275+
onRangeMissing.accept(gap.start, gap.end);
276+
gap.onResponse(null);
285277
} catch (Exception e) {
286-
range.onFailure(e);
278+
gap.onFailure(e);
287279
}
288280
}
289281
} catch (Exception e) {

0 commit comments

Comments
 (0)