Skip to content

Commit f9ccb5f

Browse files
committed
Fix small reads of multi-part blobs
1 parent a5c7bec commit f9ccb5f

File tree

3 files changed

+90
-53
lines changed

3 files changed

+90
-53
lines changed

x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/index/store/BaseSearchableSnapshotIndexInput.java

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,10 +54,13 @@ protected InputStream openInputStream(final long position, final long length) th
5454
@Override
5555
protected InputStream openSlice(long slice) throws IOException {
5656
final long currentPart = startPart + slice;
57+
final long startInPart = (currentPart == startPart) ? getRelativePositionInPart(position) : 0L;
58+
final long endInPart
59+
= (currentPart == endPart) ? getRelativePositionInPart(position + length) : getLengthOfPart(currentPart);
5760
return blobContainer.readBlob(
5861
fileInfo.partName(currentPart),
59-
(currentPart == startPart) ? getRelativePositionInPart(position) : 0L,
60-
(currentPart == endPart) ? getRelativePositionInPart(length) : getLengthOfPart(currentPart)
62+
startInPart,
63+
endInPart - startInPart
6164
);
6265
}
6366
};

x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/cache/CachedBlobContainerIndexInputTests.java

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333

3434
import static org.elasticsearch.index.store.cache.TestUtils.createCacheService;
3535
import static org.elasticsearch.index.store.cache.TestUtils.singleBlobContainer;
36+
import static org.elasticsearch.index.store.cache.TestUtils.singleSplitBlobContainer;
3637
import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots.SNAPSHOT_CACHE_ENABLED_SETTING;
3738
import static org.hamcrest.Matchers.equalTo;
3839

@@ -52,12 +53,15 @@ public void testRandomReads() throws IOException {
5253

5354
final String blobName = randomUnicodeOfLength(10);
5455
final StoreFileMetaData metaData = new StoreFileMetaData(fileName, input.length, "_na", Version.CURRENT.luceneVersion);
56+
57+
final int partSize = randomBoolean() ? input.length : randomIntBetween(1, input.length);
58+
5559
final BlobStoreIndexShardSnapshot snapshot = new BlobStoreIndexShardSnapshot(snapshotId.getName(), 0L,
56-
List.of(new BlobStoreIndexShardSnapshot.FileInfo(blobName, metaData, new ByteSizeValue(input.length))), 0L, 0L, 0, 0L);
60+
List.of(new BlobStoreIndexShardSnapshot.FileInfo(blobName, metaData, new ByteSizeValue(partSize))), 0L, 0L, 0, 0L);
5761

58-
final BlobContainer singleBlobContainer = singleBlobContainer(blobName, input);
62+
final BlobContainer singleBlobContainer = singleSplitBlobContainer(blobName, input, partSize);
5963
final BlobContainer blobContainer;
60-
if (input.length <= cacheService.getCacheSize()) {
64+
if (input.length == partSize && input.length <= cacheService.getCacheSize()) {
6165
blobContainer = new CountingBlobContainer(singleBlobContainer, cacheService.getRangeSize());
6266
} else {
6367
blobContainer = singleBlobContainer;

x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/cache/TestUtils.java

Lines changed: 78 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -70,70 +70,100 @@ public static void assertCounter(IndexInputStats.Counter counter, long total, lo
7070
* Any attempt to read a different blob will throw a {@link FileNotFoundException}
7171
*/
7272
static BlobContainer singleBlobContainer(final String blobName, final byte[] blobContent) {
73-
return new BlobContainer() {
74-
73+
return new MostlyUnimplementedFakeBlobContainer() {
7574
@Override
7675
public InputStream readBlob(String name, long position, long length) throws IOException {
7776
if (blobName.equals(name) == false) {
7877
throw new FileNotFoundException("Blob not found: " + name);
7978
}
80-
return Streams.limitStream(new ByteArrayInputStream(blobContent, Math.toIntExact(position), blobContent.length), length);
79+
assert position + length <= blobContent.length
80+
: "cannot read [" + position + "-" + (position + length) + "] from array of length [" + blobContent.length + "]";
81+
return Streams.limitStream(new ByteArrayInputStream(blobContent, Math.toIntExact(position),
82+
blobContent.length - Math.toIntExact(position)), length);
8183
}
84+
};
85+
}
8286

83-
@Override
84-
public long readBlobPreferredLength() {
85-
return Long.MAX_VALUE;
86-
}
87+
static BlobContainer singleSplitBlobContainer(final String blobName, final byte[] blobContent, final int partSize) {
88+
if (partSize >= blobContent.length) {
89+
return singleBlobContainer(blobName, blobContent);
90+
} else {
91+
final String prefix = blobName + ".part";
92+
return new MostlyUnimplementedFakeBlobContainer() {
93+
@Override
94+
public InputStream readBlob(String name, long position, long length) throws IOException {
95+
if (name.startsWith(prefix) == false) {
96+
throw new FileNotFoundException("Blob not found: " + name);
97+
}
98+
assert position + length <= blobContent.length
99+
: "cannot read [" + position + "-" + (position + length) + "] from array part of length [" + partSize + "]";
100+
final int partNumber = Integer.parseInt(name.substring(prefix.length()));
101+
final int positionInBlob = Math.toIntExact(position) + partSize * partNumber;
102+
assert positionInBlob + length <= blobContent.length
103+
: "cannot read [" + positionInBlob + "-" + (positionInBlob + length) + "] from array of length ["
104+
+ blobContent.length + "]";
105+
return Streams.limitStream(new ByteArrayInputStream(blobContent,
106+
positionInBlob, blobContent.length - positionInBlob), length);
107+
}
108+
};
109+
}
110+
}
87111

88-
@Override
89-
public Map<String, BlobMetaData> listBlobs() {
90-
throw unsupportedException();
91-
}
112+
private static class MostlyUnimplementedFakeBlobContainer implements BlobContainer {
92113

93-
@Override
94-
public BlobPath path() {
95-
throw unsupportedException();
96-
}
114+
@Override
115+
public long readBlobPreferredLength() {
116+
return Long.MAX_VALUE;
117+
}
97118

98-
@Override
99-
public InputStream readBlob(String blobName) {
100-
throw unsupportedException();
101-
}
119+
@Override
120+
public Map<String, BlobMetaData> listBlobs() {
121+
throw unsupportedException();
122+
}
102123

103-
@Override
104-
public void writeBlob(String blobName, InputStream inputStream, long blobSize, boolean failIfAlreadyExists) {
105-
throw unsupportedException();
106-
}
124+
@Override
125+
public BlobPath path() {
126+
throw unsupportedException();
127+
}
107128

108-
@Override
109-
public void writeBlobAtomic(String blobName, InputStream inputStream, long blobSize, boolean failIfAlreadyExists) {
110-
throw unsupportedException();
111-
}
129+
@Override
130+
public InputStream readBlob(String blobName) {
131+
throw unsupportedException();
132+
}
112133

113-
@Override
114-
public DeleteResult delete() {
115-
throw unsupportedException();
116-
}
134+
@Override
135+
public void writeBlob(String blobName, InputStream inputStream, long blobSize, boolean failIfAlreadyExists) {
136+
throw unsupportedException();
137+
}
117138

118-
@Override
119-
public void deleteBlobsIgnoringIfNotExists(List<String> blobNames) {
120-
throw unsupportedException();
121-
}
139+
@Override
140+
public void writeBlobAtomic(String blobName, InputStream inputStream, long blobSize, boolean failIfAlreadyExists) {
141+
throw unsupportedException();
142+
}
122143

123-
@Override
124-
public Map<String, BlobContainer> children() {
125-
throw unsupportedException();
126-
}
144+
@Override
145+
public DeleteResult delete() {
146+
throw unsupportedException();
147+
}
127148

128-
@Override
129-
public Map<String, BlobMetaData> listBlobsByPrefix(String blobNamePrefix) {
130-
throw unsupportedException();
131-
}
149+
@Override
150+
public void deleteBlobsIgnoringIfNotExists(List<String> blobNames) {
151+
throw unsupportedException();
152+
}
132153

133-
private UnsupportedOperationException unsupportedException() {
134-
assert false : "this operation is not supported and should have not be called";
135-
return new UnsupportedOperationException("This operation is not supported");
136-
}
137-
};
154+
@Override
155+
public Map<String, BlobContainer> children() {
156+
throw unsupportedException();
157+
}
158+
159+
@Override
160+
public Map<String, BlobMetaData> listBlobsByPrefix(String blobNamePrefix) {
161+
throw unsupportedException();
162+
}
163+
164+
private UnsupportedOperationException unsupportedException() {
165+
assert false : "this operation is not supported and should have not be called";
166+
return new UnsupportedOperationException("This operation is not supported");
167+
}
138168
}
139169
}

0 commit comments

Comments
 (0)