Skip to content

Commit 8d2c172

Browse files
tlrxtlrx
authored
Include min/max/average file size in Searchable Snapshots Stats API (#70294)
This commit adds the minimum, maximum and average length of files in the Searchable Snapshots Stats API.
1 parent 6e1eb44 commit 8d2c172

File tree

13 files changed

+192
-41
lines changed

13 files changed

+192
-41
lines changed

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/searchablesnapshots/SearchableSnapshotShardStats.java

Lines changed: 55 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
import org.elasticsearch.common.io.stream.StreamInput;
1212
import org.elasticsearch.common.io.stream.StreamOutput;
1313
import org.elasticsearch.common.io.stream.Writeable;
14+
import org.elasticsearch.common.unit.ByteSizeValue;
1415
import org.elasticsearch.common.unit.TimeValue;
1516
import org.elasticsearch.common.xcontent.ToXContentObject;
1617
import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -125,7 +126,9 @@ public static class CacheIndexInputStats implements Writeable, ToXContentObject
125126

126127
private final String fileExt;
127128
private final long numFiles;
128-
private final long totalSize;
129+
private final ByteSizeValue totalSize;
130+
private final ByteSizeValue minSize;
131+
private final ByteSizeValue maxSize;
129132

130133
private final long openCount;
131134
private final long closeCount;
@@ -144,7 +147,8 @@ public static class CacheIndexInputStats implements Writeable, ToXContentObject
144147
private final Counter blobStoreBytesRequested;
145148
private final long currentIndexCacheFills;
146149

147-
public CacheIndexInputStats(String fileExt, long numFiles, long totalSize, long openCount, long closeCount,
150+
public CacheIndexInputStats(String fileExt, long numFiles, ByteSizeValue totalSize, ByteSizeValue minSize, ByteSizeValue maxSize,
151+
long openCount, long closeCount,
148152
Counter forwardSmallSeeks, Counter backwardSmallSeeks,
149153
Counter forwardLargeSeeks, Counter backwardLargeSeeks,
150154
Counter contiguousReads, Counter nonContiguousReads,
@@ -154,6 +158,8 @@ public CacheIndexInputStats(String fileExt, long numFiles, long totalSize, long
154158
this.fileExt = fileExt;
155159
this.numFiles = numFiles;
156160
this.totalSize = totalSize;
161+
this.minSize = minSize;
162+
this.maxSize = maxSize;
157163
this.openCount = openCount;
158164
this.closeCount = closeCount;
159165
this.forwardSmallSeeks = forwardSmallSeeks;
@@ -180,7 +186,15 @@ public CacheIndexInputStats(String fileExt, long numFiles, long totalSize, long
180186
}
181187
this.fileExt = in.readString();
182188
this.numFiles = in.readVLong();
183-
this.totalSize = in.readVLong();
189+
if (in.getVersion().before(Version.V_8_0_0)) { // TODO adapt BWC version after backport
190+
this.totalSize = new ByteSizeValue(in.readVLong());
191+
this.minSize = ByteSizeValue.ZERO;
192+
this.maxSize = ByteSizeValue.ZERO;
193+
} else {
194+
this.totalSize = new ByteSizeValue(in);
195+
this.minSize = new ByteSizeValue(in);
196+
this.maxSize = new ByteSizeValue(in);
197+
}
184198
this.openCount = in.readVLong();
185199
this.closeCount = in.readVLong();
186200
this.forwardSmallSeeks = new Counter(in);
@@ -207,7 +221,9 @@ public static CacheIndexInputStats combine(CacheIndexInputStats cis1, CacheIndex
207221
return new CacheIndexInputStats(
208222
cis1.fileExt,
209223
cis1.numFiles + cis2.numFiles,
210-
cis1.totalSize + cis2.totalSize,
224+
new ByteSizeValue(Math.addExact(cis1.totalSize.getBytes(), cis2.totalSize.getBytes())),
225+
new ByteSizeValue(Math.min(cis1.minSize.getBytes(), cis2.minSize.getBytes())),
226+
new ByteSizeValue(Math.max(cis1.maxSize.getBytes(), cis2.maxSize.getBytes())),
211227
cis1.openCount + cis2.openCount,
212228
cis1.closeCount + cis2.closeCount,
213229
cis1.forwardSmallSeeks.add(cis2.forwardSmallSeeks),
@@ -236,7 +252,13 @@ public void writeTo(StreamOutput out) throws IOException {
236252
}
237253
out.writeString(fileExt);
238254
out.writeVLong(numFiles);
239-
out.writeVLong(totalSize);
255+
if (out.getVersion().before(Version.V_8_0_0)) { // TODO adapt BWC version after backport
256+
out.writeVLong(totalSize.getBytes());
257+
} else {
258+
totalSize.writeTo(out);
259+
minSize.writeTo(out);
260+
maxSize.writeTo(out);
261+
}
240262
out.writeVLong(openCount);
241263
out.writeVLong(closeCount);
242264

@@ -263,10 +285,23 @@ public long getNumFiles() {
263285
return numFiles;
264286
}
265287

266-
public long getTotalSize() {
288+
public ByteSizeValue getTotalSize() {
267289
return totalSize;
268290
}
269291

292+
public ByteSizeValue getMinSize() {
293+
return minSize;
294+
}
295+
296+
public ByteSizeValue getMaxSize() {
297+
return maxSize;
298+
}
299+
300+
public ByteSizeValue getAverageSize() {
301+
final double average = (double) totalSize.getBytes()/ (double) numFiles;
302+
return new ByteSizeValue(Math.round(average));
303+
}
304+
270305
public long getOpenCount() {
271306
return openCount;
272307
}
@@ -333,9 +368,16 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
333368
{
334369
builder.field("file_ext", getFileExt());
335370
builder.field("num_files", getNumFiles());
336-
builder.field("total_size", getTotalSize());
337371
builder.field("open_count", getOpenCount());
338372
builder.field("close_count", getCloseCount());
373+
{
374+
builder.startObject("size");
375+
builder.humanReadableField("total_in_bytes", "total", getTotalSize());
376+
builder.humanReadableField("min_in_bytes", "min", getMinSize());
377+
builder.humanReadableField("max_in_bytes", "max", getMaxSize());
378+
builder.humanReadableField("average_in_bytes", "average", getAverageSize());
379+
builder.endObject();
380+
}
339381
builder.field("contiguous_bytes_read", getContiguousReads(), params);
340382
builder.field("non_contiguous_bytes_read", getNonContiguousReads(), params);
341383
builder.field("cached_bytes_read", getCachedBytesRead(), params);
@@ -371,10 +413,12 @@ public boolean equals(Object other) {
371413
}
372414
CacheIndexInputStats stats = (CacheIndexInputStats) other;
373415
return numFiles == stats.numFiles
374-
&& totalSize == stats.totalSize
375416
&& openCount == stats.openCount
376417
&& closeCount == stats.closeCount
377418
&& Objects.equals(fileExt, stats.fileExt)
419+
&& Objects.equals(totalSize, stats.totalSize)
420+
&& Objects.equals(minSize, stats.minSize)
421+
&& Objects.equals(maxSize, stats.maxSize)
378422
&& Objects.equals(forwardSmallSeeks, stats.forwardSmallSeeks)
379423
&& Objects.equals(backwardSmallSeeks, stats.backwardSmallSeeks)
380424
&& Objects.equals(forwardLargeSeeks, stats.forwardLargeSeeks)
@@ -392,7 +436,9 @@ public boolean equals(Object other) {
392436

393437
@Override
394438
public int hashCode() {
395-
return Objects.hash(fileExt, numFiles, totalSize, openCount, closeCount,
439+
return Objects.hash(fileExt, numFiles, totalSize,
440+
minSize, maxSize,
441+
openCount, closeCount,
396442
forwardSmallSeeks, backwardSmallSeeks,
397443
forwardLargeSeeks, backwardLargeSeeks,
398444
contiguousReads, nonContiguousReads,

‎x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/searchablesnapshots/SearchableSnapshotShardStatsTests.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
import org.elasticsearch.cluster.routing.ShardRoutingState;
1111
import org.elasticsearch.cluster.routing.TestShardRouting;
1212
import org.elasticsearch.common.io.stream.Writeable;
13+
import org.elasticsearch.common.unit.ByteSizeValue;
1314
import org.elasticsearch.repositories.IndexId;
1415
import org.elasticsearch.snapshots.SnapshotId;
1516
import org.elasticsearch.test.AbstractWireSerializingTestCase;
@@ -42,7 +43,8 @@ protected SearchableSnapshotShardStats createTestInstance() {
4243
}
4344

4445
private CacheIndexInputStats randomCacheIndexInputStats() {
45-
return new CacheIndexInputStats(randomAlphaOfLength(10), randomNonNegativeLong(), randomNonNegativeLong(),
46+
return new CacheIndexInputStats(randomAlphaOfLength(10), randomNonNegativeLong(), new ByteSizeValue(randomNonNegativeLong()),
47+
new ByteSizeValue(randomNonNegativeLong()), new ByteSizeValue(randomNonNegativeLong()),
4648
randomNonNegativeLong(), randomNonNegativeLong(),
4749
randomCounter(), randomCounter(),
4850
randomCounter(), randomCounter(),

‎x-pack/plugin/searchable-snapshots/qa/rest/src/test/resources/rest-api-spec/test/stats.yml

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -66,8 +66,8 @@ teardown:
6666
---
6767
"Tests searchable snapshots stats":
6868
- skip:
69-
version: " - 7.7.99"
70-
reason: searchable snapshots introduced in 7.8.0
69+
version: " - 7.99.99"
70+
reason: searchable snapshots response format changed in 8.0.0
7171

7272
- do:
7373
catch: missing
@@ -139,10 +139,14 @@ teardown:
139139
- is_true: total
140140
- is_true: total.0.file_ext
141141
- gt: { total.0.num_files: 0 }
142-
- gt: { total.0.total_size: 0 }
143142
- gt: { total.0.open_count: 0 }
144143
- gt: { total.0.close_count: 0 }
145144

145+
- gt: { total.0.size.total_in_bytes: 0 }
146+
- gt: { total.0.size.min_in_bytes: 0 }
147+
- gt: { total.0.size.max_in_bytes: 0 }
148+
- gt: { total.0.size.average_in_bytes: 0 }
149+
146150
- gte: { total.0.contiguous_bytes_read.count: 0 }
147151
- gte: { total.0.contiguous_bytes_read.sum: 0 }
148152
- gte: { total.0.contiguous_bytes_read.min: 0 }
@@ -215,10 +219,14 @@ teardown:
215219

216220
- is_true: indices.docs.total.0.file_ext
217221
- gt: { indices.docs.total.0.num_files: 0 }
218-
- gt: { indices.docs.total.0.total_size: 0 }
219222
- gt: { indices.docs.total.0.open_count: 0 }
220223
- gt: { indices.docs.total.0.close_count: 0 }
221224

225+
- gt: { indices.docs.total.0.size.total_in_bytes: 0 }
226+
- gt: { indices.docs.total.0.size.min_in_bytes: 0 }
227+
- gt: { indices.docs.total.0.size.max_in_bytes: 0 }
228+
- gt: { indices.docs.total.0.size.average_in_bytes: 0 }
229+
222230
- gte: { indices.docs.total.0.contiguous_bytes_read.count: 0 }
223231
- gte: { indices.docs.total.0.contiguous_bytes_read.sum: 0 }
224232
- gte: { indices.docs.total.0.contiguous_bytes_read.min: 0 }
@@ -295,10 +303,14 @@ teardown:
295303

296304
- is_true: indices.docs.shards.0.0.files.0.file_ext
297305
- gt: { indices.docs.shards.0.0.files.0.num_files: 0 }
298-
- gt: { indices.docs.shards.0.0.files.0.total_size: 0 }
299306
- gt: { indices.docs.shards.0.0.files.0.open_count: 0 }
300307
- gt: { indices.docs.shards.0.0.files.0.close_count: 0 }
301308

309+
- gt: { indices.docs.shards.0.0.files.0.size.total_in_bytes: 0 }
310+
- gt: { indices.docs.shards.0.0.files.0.size.min_in_bytes: 0 }
311+
- gt: { indices.docs.shards.0.0.files.0.size.max_in_bytes: 0 }
312+
- gt: { indices.docs.shards.0.0.files.0.size.average_in_bytes: 0 }
313+
302314
- gte: { indices.docs.shards.0.0.files.0.contiguous_bytes_read.count: 0 }
303315
- gte: { indices.docs.shards.0.0.files.0.contiguous_bytes_read.sum: 0 }
304316
- gte: { indices.docs.shards.0.0.files.0.contiguous_bytes_read.min: 0 }
@@ -373,6 +385,10 @@ teardown:
373385

374386
- is_true: indices.docs.shards.0.0.files.0.cached_bytes_written.time
375387
- is_true: indices.docs.shards.0.0.files.0.direct_bytes_read.time
388+
- is_true: indices.docs.shards.0.0.files.0.size.total
389+
- is_true: indices.docs.shards.0.0.files.0.size.min
390+
- is_true: indices.docs.shards.0.0.files.0.size.max
391+
- is_true: indices.docs.shards.0.0.files.0.size.average
376392

377393
- do:
378394
searchable_snapshots.stats:

‎x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1440,7 +1440,7 @@ private void assertSearchableSnapshotStats(String indexName, boolean cacheEnable
14401440
final long totalSize = statsResponse.getStats()
14411441
.stream()
14421442
.flatMap(s -> s.getStats().stream())
1443-
.mapToLong(SearchableSnapshotShardStats.CacheIndexInputStats::getTotalSize)
1443+
.mapToLong(stat -> stat.getTotalSize().getBytes())
14441444
.sum();
14451445
final Set<String> nodeIdsWithLargeEnoughCache = new HashSet<>();
14461446
for (ObjectCursor<DiscoveryNode> nodeCursor : client().admin()
@@ -1478,7 +1478,22 @@ private void assertSearchableSnapshotStats(String indexName, boolean cacheEnable
14781478
);
14791479
assertThat(
14801480
"Unexpected file length for " + fileExt + " of shard " + shardRouting,
1481-
indexInputStats.getTotalSize(),
1481+
indexInputStats.getTotalSize().getBytes(),
1482+
greaterThan(0L)
1483+
);
1484+
assertThat(
1485+
"Unexpected min. file length for " + fileExt + " of shard " + shardRouting,
1486+
indexInputStats.getMinSize().getBytes(),
1487+
greaterThan(0L)
1488+
);
1489+
assertThat(
1490+
"Unexpected max. file length for " + fileExt + " of shard " + shardRouting,
1491+
indexInputStats.getMaxSize().getBytes(),
1492+
greaterThan(0L)
1493+
);
1494+
assertThat(
1495+
"Unexpected average file length for " + fileExt + " of shard " + shardRouting,
1496+
indexInputStats.getAverageSize().getBytes(),
14821497
greaterThan(0L)
14831498
);
14841499

‎x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/index/store/IndexInputStats.java

Lines changed: 25 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,11 @@ public class IndexInputStats {
2525
/* A threshold beyond which an index input seeking is counted as "large" */
2626
static final ByteSizeValue SEEKING_THRESHOLD = new ByteSizeValue(8, ByteSizeUnit.MB);
2727

28-
private final int numFiles;
28+
private final long numFiles;
2929
private final long totalSize;
30+
private final long minSize;
31+
private final long maxSize;
32+
3033
private final long seekingThreshold;
3134
private final LongSupplier currentTimeNanos;
3235

@@ -52,13 +55,22 @@ public class IndexInputStats {
5255
private final Counter blobStoreBytesRequested = new Counter();
5356
private final AtomicLong currentIndexCacheFills = new AtomicLong();
5457

55-
public IndexInputStats(int numFiles, long totalSize, LongSupplier currentTimeNanos) {
56-
this(numFiles, totalSize, SEEKING_THRESHOLD.getBytes(), currentTimeNanos);
58+
public IndexInputStats(long numFiles, long totalSize, long minSize, long maxSize, LongSupplier currentTimeNanos) {
59+
this(numFiles, totalSize, minSize, maxSize, SEEKING_THRESHOLD.getBytes(), currentTimeNanos);
5760
}
5861

59-
public IndexInputStats(int numFiles, long totalSize, long seekingThreshold, LongSupplier currentTimeNanos) {
62+
public IndexInputStats(
63+
long numFiles,
64+
long totalSize,
65+
long minSize,
66+
long maxSize,
67+
long seekingThreshold,
68+
LongSupplier currentTimeNanos
69+
) {
6070
this.numFiles = numFiles;
6171
this.totalSize = totalSize;
72+
this.minSize = minSize;
73+
this.maxSize = maxSize;
6274
this.seekingThreshold = seekingThreshold;
6375
this.currentTimeNanos = currentTimeNanos;
6476
}
@@ -137,14 +149,22 @@ public Releasable addIndexCacheFill() {
137149
};
138150
}
139151

140-
public int getNumFiles() {
152+
public long getNumFiles() {
141153
return numFiles;
142154
}
143155

144156
public long getTotalSize() {
145157
return totalSize;
146158
}
147159

160+
public long getMinSize() {
161+
return minSize;
162+
}
163+
164+
public long getMaxSize() {
165+
return maxSize;
166+
}
167+
148168
public LongAdder getOpened() {
149169
return opened;
150170
}

‎x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/index/store/SearchableSnapshotDirectory.java

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -360,8 +360,8 @@ public void clearCache() {
360360
}
361361
}
362362

363-
protected IndexInputStats createIndexInputStats(final int numFiles, final long totalSize) {
364-
return new IndexInputStats(numFiles, totalSize, statsCurrentTimeNanosSupplier);
363+
protected IndexInputStats createIndexInputStats(long numFiles, long totalSize, long minSize, long maxSize) {
364+
return new IndexInputStats(numFiles, totalSize, minSize, maxSize, statsCurrentTimeNanosSupplier);
365365
}
366366

367367
public CacheKey createCacheKey(String fileName) {
@@ -395,13 +395,13 @@ public IndexInput openInput(final String name, final IOContext context) throws I
395395

396396
final String ext = getNonNullFileExt(name);
397397
final IndexInputStats inputStats = stats.computeIfAbsent(ext, n -> {
398-
// get all fileInfo with same extension
399-
final Tuple<Integer, Long> fileExtCompoundStats = files().stream()
400-
.filter(fi -> ext.equals(getNonNullFileExt(fi.physicalName())))
401-
.map(fi -> Tuple.tuple(1, fi.length()))
402-
.reduce((t1, t2) -> Tuple.tuple(t1.v1() + t2.v1(), t1.v2() + t2.v2()))
403-
.get();
404-
return createIndexInputStats(fileExtCompoundStats.v1(), fileExtCompoundStats.v2());
398+
final IndexInputStats.Counter counter = new IndexInputStats.Counter();
399+
for (BlobStoreIndexShardSnapshot.FileInfo file : files()) {
400+
if (ext.equals(getNonNullFileExt(file.physicalName()))) {
401+
counter.add(file.length());
402+
}
403+
}
404+
return createIndexInputStats(counter.count(), counter.total(), counter.min(), counter.max());
405405
});
406406
if (useCache && isExcludedFromCache(name) == false) {
407407
if (partial) {

‎x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/SearchableSnapshotsStatsResponse.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@ private static List<CacheIndexInputStats> computeCompound(Stream<CacheIndexInput
6565
.stream()
6666
.filter(o -> o.isEmpty() == false)
6767
.map(Optional::get)
68+
.sorted(Comparator.comparing(CacheIndexInputStats::getFileExt))
6869
.collect(toList());
6970
}
7071

0 commit comments

Comments
 (0)