Skip to content

Commit 990ac4c

Browse files
original-brownbearoriginal-brownbear
authored
Some Cleanup in BlobStoreRepository (elastic#43323) (elastic#44043)
* Some Cleanup in BlobStoreRepository * Extracted from elastic#42833: * Dry up index and shard path handling * Shorten XContent handling
1 parent 9089820 commit 990ac4c

File tree

3 files changed

+30
-50
lines changed

3 files changed

+30
-50
lines changed

‎server/src/main/java/org/elasticsearch/repositories/RepositoryData.java

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -308,7 +308,7 @@ public List<IndexId> resolveNewIndices(final List<String> indicesToResolve) {
308308
* Writes the snapshots metadata and the related indices metadata to x-content, omitting the
309309
* incompatible snapshots.
310310
*/
311-
public XContentBuilder snapshotsToXContent(final XContentBuilder builder, final ToXContent.Params params) throws IOException {
311+
public XContentBuilder snapshotsToXContent(final XContentBuilder builder) throws IOException {
312312
builder.startObject();
313313
// write the snapshots list
314314
builder.startArray(SNAPSHOTS);
@@ -453,14 +453,12 @@ public static RepositoryData snapshotsFromXContent(final XContentParser parser,
453453
/**
454454
* Writes the incompatible snapshot ids to x-content.
455455
*/
456-
public XContentBuilder incompatibleSnapshotsToXContent(final XContentBuilder builder, final ToXContent.Params params)
457-
throws IOException {
458-
456+
public XContentBuilder incompatibleSnapshotsToXContent(XContentBuilder builder) throws IOException {
459457
builder.startObject();
460458
// write the incompatible snapshots list
461459
builder.startArray(INCOMPATIBLE_SNAPSHOTS);
462460
for (final SnapshotId snapshot : getIncompatibleSnapshotIds()) {
463-
snapshot.toXContent(builder, params);
461+
snapshot.toXContent(builder, ToXContent.EMPTY_PARAMS);
464462
}
465463
builder.endArray();
466464
builder.endObject();

‎server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java

Lines changed: 24 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -51,8 +51,6 @@
5151
import org.elasticsearch.common.component.AbstractLifecycleComponent;
5252
import org.elasticsearch.common.compress.NotXContentException;
5353
import org.elasticsearch.common.io.stream.BytesStreamOutput;
54-
import org.elasticsearch.common.io.stream.OutputStreamStreamOutput;
55-
import org.elasticsearch.common.io.stream.StreamOutput;
5654
import org.elasticsearch.common.lucene.Lucene;
5755
import org.elasticsearch.common.lucene.store.InputStreamIndexInput;
5856
import org.elasticsearch.common.metrics.CounterMetric;
@@ -62,8 +60,6 @@
6260
import org.elasticsearch.common.util.set.Sets;
6361
import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
6462
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
65-
import org.elasticsearch.common.xcontent.ToXContent;
66-
import org.elasticsearch.common.xcontent.XContentBuilder;
6763
import org.elasticsearch.common.xcontent.XContentFactory;
6864
import org.elasticsearch.common.xcontent.XContentHelper;
6965
import org.elasticsearch.common.xcontent.XContentParser;
@@ -401,10 +397,7 @@ public void initializeSnapshot(SnapshotId snapshotId, List<IndexId> indices, Met
401397

402398
// write the index metadata for each index in the snapshot
403399
for (IndexId index : indices) {
404-
final IndexMetaData indexMetaData = clusterMetaData.index(index.getName());
405-
final BlobPath indexPath = basePath().add("indices").add(index.getId());
406-
final BlobContainer indexMetaDataBlobContainer = blobStore().blobContainer(indexPath);
407-
indexMetaDataFormat.write(indexMetaData, indexMetaDataBlobContainer, snapshotId.getUUID());
400+
indexMetaDataFormat.write(clusterMetaData.index(index.getName()), indexContainer(index), snapshotId.getUUID());
408401
}
409402
} catch (IOException ex) {
410403
throw new SnapshotCreationException(metadata.name(), snapshotId, ex);
@@ -452,7 +445,7 @@ public void deleteSnapshot(SnapshotId snapshotId, long repositoryStateId, Action
452445
snapshotId,
453446
ActionListener.map(listener, v -> {
454447
try {
455-
blobStore().blobContainer(basePath().add("indices")).deleteBlobsIgnoringIfNotExists(
448+
blobStore().blobContainer(indicesPath()).deleteBlobsIgnoringIfNotExists(
456449
unreferencedIndices.stream().map(IndexId::getId).collect(Collectors.toList()));
457450
} catch (IOException e) {
458451
logger.warn(() ->
@@ -504,9 +497,8 @@ protected void doRun() {
504497
}
505498

506499
private void deleteIndexMetaDataBlobIgnoringErrors(SnapshotId snapshotId, IndexId indexId) {
507-
BlobContainer indexMetaDataBlobContainer = blobStore().blobContainer(basePath().add("indices").add(indexId.getId()));
508500
try {
509-
indexMetaDataFormat.delete(indexMetaDataBlobContainer, snapshotId.getUUID());
501+
indexMetaDataFormat.delete(indexContainer(indexId), snapshotId.getUUID());
510502
} catch (IOException ex) {
511503
logger.warn(() -> new ParameterizedMessage("[{}] failed to delete metadata for index [{}]",
512504
snapshotId, indexId.getName()), ex);
@@ -570,8 +562,19 @@ public MetaData getSnapshotGlobalMetaData(final SnapshotId snapshotId) {
570562

571563
@Override
572564
public IndexMetaData getSnapshotIndexMetaData(final SnapshotId snapshotId, final IndexId index) throws IOException {
573-
final BlobPath indexPath = basePath().add("indices").add(index.getId());
574-
return indexMetaDataFormat.read(blobStore().blobContainer(indexPath), snapshotId.getUUID());
565+
return indexMetaDataFormat.read(indexContainer(index), snapshotId.getUUID());
566+
}
567+
568+
private BlobPath indicesPath() {
569+
return basePath().add("indices");
570+
}
571+
572+
private BlobContainer indexContainer(IndexId indexId) {
573+
return blobStore().blobContainer(indicesPath().add(indexId.getId()));
574+
}
575+
576+
private BlobContainer shardContainer(IndexId indexId, ShardId shardId) {
577+
return blobStore().blobContainer(indicesPath().add(indexId.getId()).add(Integer.toString(shardId.getId())));
575578
}
576579

577580
/**
@@ -619,10 +622,9 @@ public String startVerification() {
619622
String seed = UUIDs.randomBase64UUID();
620623
byte[] testBytes = Strings.toUTF8Bytes(seed);
621624
BlobContainer testContainer = blobStore().blobContainer(basePath().add(testBlobPrefix(seed)));
622-
String blobName = "master.dat";
623625
BytesArray bytes = new BytesArray(testBytes);
624626
try (InputStream stream = bytes.streamInput()) {
625-
testContainer.writeBlobAtomic(blobName, stream, bytes.length(), true);
627+
testContainer.writeBlobAtomic("master.dat", stream, bytes.length(), true);
626628
}
627629
return seed;
628630
}
@@ -695,7 +697,7 @@ public RepositoryData getRepositoryData() {
695697
}
696698
}
697699

698-
public static String testBlobPrefix(String seed) {
700+
private static String testBlobPrefix(String seed) {
699701
return TESTS_FILE + seed;
700702
}
701703

@@ -715,19 +717,10 @@ protected void writeIndexGen(final RepositoryData repositoryData, final long rep
715717
"] - possibly due to simultaneous snapshot deletion requests");
716718
}
717719
final long newGen = currentGen + 1;
718-
final BytesReference snapshotsBytes;
719-
try (BytesStreamOutput bStream = new BytesStreamOutput()) {
720-
try (StreamOutput stream = new OutputStreamStreamOutput(bStream)) {
721-
XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON, stream);
722-
repositoryData.snapshotsToXContent(builder, ToXContent.EMPTY_PARAMS);
723-
builder.close();
724-
}
725-
snapshotsBytes = bStream.bytes();
726-
}
727720
// write the index file
728721
final String indexBlob = INDEX_FILE_PREFIX + Long.toString(newGen);
729722
logger.debug("Repository [{}] writing new index generational blob [{}]", metadata.name(), indexBlob);
730-
writeAtomic(indexBlob, snapshotsBytes, true);
723+
writeAtomic(indexBlob, BytesReference.bytes(repositoryData.snapshotsToXContent(XContentFactory.jsonBuilder())), true);
731724
// write the current generation to the index-latest file
732725
final BytesReference genBytes;
733726
try (BytesStreamOutput bStream = new BytesStreamOutput()) {
@@ -754,17 +747,9 @@ protected void writeIndexGen(final RepositoryData repositoryData, final long rep
754747
*/
755748
void writeIncompatibleSnapshots(RepositoryData repositoryData) throws IOException {
756749
assert isReadOnly() == false; // can not write to a read only repository
757-
final BytesReference bytes;
758-
try (BytesStreamOutput bStream = new BytesStreamOutput()) {
759-
try (StreamOutput stream = new OutputStreamStreamOutput(bStream)) {
760-
XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON, stream);
761-
repositoryData.incompatibleSnapshotsToXContent(builder, ToXContent.EMPTY_PARAMS);
762-
builder.close();
763-
}
764-
bytes = bStream.bytes();
765-
}
766750
// write the incompatible snapshots blob
767-
writeAtomic(INCOMPATIBLE_SNAPSHOTS_BLOB, bytes, false);
751+
writeAtomic(INCOMPATIBLE_SNAPSHOTS_BLOB,
752+
BytesReference.bytes(repositoryData.incompatibleSnapshotsToXContent(XContentFactory.jsonBuilder())), false);
768753
}
769754

770755
/**
@@ -857,9 +842,8 @@ public void restoreShard(Store store, SnapshotId snapshotId,
857842
Version version, IndexId indexId, ShardId snapshotShardId, RecoveryState recoveryState) {
858843
ShardId shardId = store.shardId();
859844
final Context context = new Context(snapshotId, indexId, shardId, snapshotShardId);
860-
BlobPath path = basePath().add("indices").add(indexId.getId()).add(Integer.toString(snapshotShardId.getId()));
861-
BlobContainer blobContainer = blobStore().blobContainer(path);
862-
final RestoreContext snapshotContext = new RestoreContext(shardId, snapshotId, recoveryState, blobContainer);
845+
final RestoreContext snapshotContext =
846+
new RestoreContext(shardId, snapshotId, recoveryState, shardContainer(indexId, snapshotShardId));
863847
try {
864848
BlobStoreIndexShardSnapshot snapshot = context.loadSnapshot();
865849
SnapshotFiles snapshotFiles = new SnapshotFiles(snapshot.snapshot(), snapshot.indexFiles());
@@ -935,8 +919,7 @@ private class Context {
935919
Context(SnapshotId snapshotId, IndexId indexId, ShardId shardId, ShardId snapshotShardId) {
936920
this.snapshotId = snapshotId;
937921
this.shardId = shardId;
938-
blobContainer = blobStore().blobContainer(basePath().add("indices").add(indexId.getId())
939-
.add(Integer.toString(snapshotShardId.getId())));
922+
blobContainer = shardContainer(indexId, snapshotShardId);
940923
}
941924

942925
/**

‎server/src/test/java/org/elasticsearch/repositories/RepositoryDataTests.java

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@
2222
import org.elasticsearch.ElasticsearchParseException;
2323
import org.elasticsearch.common.UUIDs;
2424
import org.elasticsearch.common.bytes.BytesReference;
25-
import org.elasticsearch.common.xcontent.ToXContent;
2625
import org.elasticsearch.common.xcontent.XContent;
2726
import org.elasticsearch.common.xcontent.XContentBuilder;
2827
import org.elasticsearch.common.xcontent.XContentParser;
@@ -61,7 +60,7 @@ public void testEqualsAndHashCode() {
6160
public void testXContent() throws IOException {
6261
RepositoryData repositoryData = generateRandomRepoData();
6362
XContentBuilder builder = JsonXContent.contentBuilder();
64-
repositoryData.snapshotsToXContent(builder, ToXContent.EMPTY_PARAMS);
63+
repositoryData.snapshotsToXContent(builder);
6564
try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) {
6665
long gen = (long) randomIntBetween(0, 500);
6766
RepositoryData fromXContent = RepositoryData.snapshotsFromXContent(parser, gen);
@@ -166,7 +165,7 @@ public void testIndexThatReferencesAnUnknownSnapshot() throws IOException {
166165
final RepositoryData repositoryData = generateRandomRepoData();
167166

168167
XContentBuilder builder = XContentBuilder.builder(xContent);
169-
repositoryData.snapshotsToXContent(builder, ToXContent.EMPTY_PARAMS);
168+
repositoryData.snapshotsToXContent(builder);
170169
RepositoryData parsedRepositoryData;
171170
try (XContentParser xParser = createParser(builder)) {
172171
parsedRepositoryData = RepositoryData.snapshotsFromXContent(xParser, repositoryData.getGenId());
@@ -197,7 +196,7 @@ public void testIndexThatReferencesAnUnknownSnapshot() throws IOException {
197196
indexSnapshots, new ArrayList<>(parsedRepositoryData.getIncompatibleSnapshotIds()));
198197

199198
final XContentBuilder corruptedBuilder = XContentBuilder.builder(xContent);
200-
corruptedRepositoryData.snapshotsToXContent(corruptedBuilder, ToXContent.EMPTY_PARAMS);
199+
corruptedRepositoryData.snapshotsToXContent(corruptedBuilder);
201200

202201
try (XContentParser xParser = createParser(corruptedBuilder)) {
203202
ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () ->

0 commit comments

Comments
 (0)