Skip to content

Commit 94e4cb6

Browse files
dnhatndnhatn
authored
Bootstrap a new history_uuid when force allocating a stale primary (#33432)
This commit ensures that we bootstrap a new history_uuid when force allocating a stale primary. A stale primary should never be the source of an operation-based recovery to another shard which exists before the forced-allocation. Closes #26712
1 parent f27c3dc commit 94e4cb6

File tree

27 files changed

+194
-164
lines changed

27 files changed

+194
-164
lines changed

‎server/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -27,10 +27,11 @@
2727
import org.elasticsearch.cluster.Diff;
2828
import org.elasticsearch.cluster.metadata.IndexMetaData;
2929
import org.elasticsearch.cluster.metadata.MetaData;
30+
import org.elasticsearch.cluster.routing.RecoverySource.EmptyStoreRecoverySource;
31+
import org.elasticsearch.cluster.routing.RecoverySource.ExistingStoreRecoverySource;
3032
import org.elasticsearch.cluster.routing.RecoverySource.LocalShardsRecoverySource;
3133
import org.elasticsearch.cluster.routing.RecoverySource.PeerRecoverySource;
3234
import org.elasticsearch.cluster.routing.RecoverySource.SnapshotRecoverySource;
33-
import org.elasticsearch.cluster.routing.RecoverySource.StoreRecoverySource;
3435
import org.elasticsearch.common.Randomness;
3536
import org.elasticsearch.common.collect.ImmutableOpenIntMap;
3637
import org.elasticsearch.common.io.stream.StreamInput;
@@ -386,7 +387,7 @@ private Builder initializeAsRestore(IndexMetaData indexMetaData, SnapshotRecover
386387
if (asNew && ignoreShards.contains(shardNumber)) {
387388
// This shards wasn't completely snapshotted - restore it as new shard
388389
indexShardRoutingBuilder.addShard(ShardRouting.newUnassigned(shardId, primary,
389-
primary ? StoreRecoverySource.EMPTY_STORE_INSTANCE : PeerRecoverySource.INSTANCE, unassignedInfo));
390+
primary ? EmptyStoreRecoverySource.INSTANCE : PeerRecoverySource.INSTANCE, unassignedInfo));
390391
} else {
391392
indexShardRoutingBuilder.addShard(ShardRouting.newUnassigned(shardId, primary,
392393
primary ? recoverySource : PeerRecoverySource.INSTANCE, unassignedInfo));
@@ -410,13 +411,13 @@ private Builder initializeEmpty(IndexMetaData indexMetaData, UnassignedInfo unas
410411
final RecoverySource primaryRecoverySource;
411412
if (indexMetaData.inSyncAllocationIds(shardNumber).isEmpty() == false) {
412413
// we have previous valid copies for this shard. use them for recovery
413-
primaryRecoverySource = StoreRecoverySource.EXISTING_STORE_INSTANCE;
414+
primaryRecoverySource = ExistingStoreRecoverySource.INSTANCE;
414415
} else if (indexMetaData.getResizeSourceIndex() != null) {
415416
// this is a new index but the initial shards should merged from another index
416417
primaryRecoverySource = LocalShardsRecoverySource.INSTANCE;
417418
} else {
418419
// a freshly created index with no restriction
419-
primaryRecoverySource = StoreRecoverySource.EMPTY_STORE_INSTANCE;
420+
primaryRecoverySource = EmptyStoreRecoverySource.INSTANCE;
420421
}
421422
IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(shardId);
422423
for (int i = 0; i <= indexMetaData.getNumberOfReplicas(); i++) {

‎server/src/main/java/org/elasticsearch/cluster/routing/RecoverySource.java

Lines changed: 64 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,8 @@
3434
/**
3535
* Represents the recovery source of a shard. Available recovery types are:
3636
*
37-
* - {@link StoreRecoverySource} recovery from the local store (empty or with existing data)
37+
* - {@link EmptyStoreRecoverySource} recovery from an empty store
38+
* - {@link ExistingStoreRecoverySource} recovery from an existing store
3839
* - {@link PeerRecoverySource} recovery from a primary on another node
3940
* - {@link SnapshotRecoverySource} recovery from a snapshot
4041
* - {@link LocalShardsRecoverySource} recovery from other shards of another index on the same node
@@ -59,8 +60,8 @@ public void addAdditionalFields(XContentBuilder builder, ToXContent.Params param
5960
public static RecoverySource readFrom(StreamInput in) throws IOException {
6061
Type type = Type.values()[in.readByte()];
6162
switch (type) {
62-
case EMPTY_STORE: return StoreRecoverySource.EMPTY_STORE_INSTANCE;
63-
case EXISTING_STORE: return StoreRecoverySource.EXISTING_STORE_INSTANCE;
63+
case EMPTY_STORE: return EmptyStoreRecoverySource.INSTANCE;
64+
case EXISTING_STORE: return new ExistingStoreRecoverySource(in);
6465
case PEER: return PeerRecoverySource.INSTANCE;
6566
case SNAPSHOT: return new SnapshotRecoverySource(in);
6667
case LOCAL_SHARDS: return LocalShardsRecoverySource.INSTANCE;
@@ -91,6 +92,10 @@ public enum Type {
9192

9293
public abstract Type getType();
9394

95+
public boolean shouldBootstrapNewHistoryUUID() {
96+
return false;
97+
}
98+
9499
@Override
95100
public boolean equals(Object o) {
96101
if (this == o) return true;
@@ -107,25 +112,68 @@ public int hashCode() {
107112
}
108113

109114
/**
110-
* recovery from an existing on-disk store or a fresh copy
115+
* Recovery from a fresh copy
111116
*/
112-
public abstract static class StoreRecoverySource extends RecoverySource {
113-
public static final StoreRecoverySource EMPTY_STORE_INSTANCE = new StoreRecoverySource() {
114-
@Override
115-
public Type getType() {
116-
return Type.EMPTY_STORE;
117+
public static final class EmptyStoreRecoverySource extends RecoverySource {
118+
public static final EmptyStoreRecoverySource INSTANCE = new EmptyStoreRecoverySource();
119+
120+
@Override
121+
public Type getType() {
122+
return Type.EMPTY_STORE;
123+
}
124+
125+
@Override
126+
public String toString() {
127+
return "new shard recovery";
128+
}
129+
}
130+
131+
/**
132+
* Recovery from an existing on-disk store
133+
*/
134+
public static final class ExistingStoreRecoverySource extends RecoverySource {
135+
public static final ExistingStoreRecoverySource INSTANCE = new ExistingStoreRecoverySource(false);
136+
public static final ExistingStoreRecoverySource FORCE_STALE_PRIMARY_INSTANCE = new ExistingStoreRecoverySource(true);
137+
138+
private final boolean bootstrapNewHistoryUUID;
139+
140+
private ExistingStoreRecoverySource(boolean bootstrapNewHistoryUUID) {
141+
this.bootstrapNewHistoryUUID = bootstrapNewHistoryUUID;
142+
}
143+
144+
private ExistingStoreRecoverySource(StreamInput in) throws IOException {
145+
if (in.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) {
146+
bootstrapNewHistoryUUID = in.readBoolean();
147+
} else {
148+
bootstrapNewHistoryUUID = false;
117149
}
118-
};
119-
public static final StoreRecoverySource EXISTING_STORE_INSTANCE = new StoreRecoverySource() {
120-
@Override
121-
public Type getType() {
122-
return Type.EXISTING_STORE;
150+
}
151+
152+
@Override
153+
public void addAdditionalFields(XContentBuilder builder, Params params) throws IOException {
154+
builder.field("bootstrap_new_history_uuid", bootstrapNewHistoryUUID);
155+
}
156+
157+
@Override
158+
protected void writeAdditionalFields(StreamOutput out) throws IOException {
159+
if (out.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) {
160+
out.writeBoolean(bootstrapNewHistoryUUID);
123161
}
124-
};
162+
}
163+
164+
@Override
165+
public boolean shouldBootstrapNewHistoryUUID() {
166+
return bootstrapNewHistoryUUID;
167+
}
168+
169+
@Override
170+
public Type getType() {
171+
return Type.EXISTING_STORE;
172+
}
125173

126174
@Override
127175
public String toString() {
128-
return getType() == Type.EMPTY_STORE ? "new shard recovery" : "existing recovery";
176+
return "existing store recovery; bootstrap_history_uuid=" + bootstrapNewHistoryUUID;
129177
}
130178
}
131179

‎server/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,14 +19,13 @@
1919

2020
package org.elasticsearch.cluster.routing;
2121

22+
import org.elasticsearch.cluster.routing.RecoverySource.ExistingStoreRecoverySource;
2223
import org.elasticsearch.cluster.routing.RecoverySource.PeerRecoverySource;
23-
import org.elasticsearch.cluster.routing.RecoverySource.StoreRecoverySource;
2424
import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator;
2525
import org.elasticsearch.common.Nullable;
2626
import org.elasticsearch.common.io.stream.StreamInput;
2727
import org.elasticsearch.common.io.stream.StreamOutput;
2828
import org.elasticsearch.common.io.stream.Writeable;
29-
import org.elasticsearch.common.xcontent.ToXContent.Params;
3029
import org.elasticsearch.common.xcontent.ToXContentObject;
3130
import org.elasticsearch.common.xcontent.XContentBuilder;
3231
import org.elasticsearch.index.Index;
@@ -318,7 +317,7 @@ public ShardRouting moveToUnassigned(UnassignedInfo unassignedInfo) {
318317
final RecoverySource recoverySource;
319318
if (active()) {
320319
if (primary()) {
321-
recoverySource = StoreRecoverySource.EXISTING_STORE_INSTANCE;
320+
recoverySource = ExistingStoreRecoverySource.INSTANCE;
322321
} else {
323322
recoverySource = PeerRecoverySource.INSTANCE;
324323
}

‎server/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateEmptyPrimaryAllocationCommand.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121

2222
import org.elasticsearch.cluster.node.DiscoveryNode;
2323
import org.elasticsearch.cluster.routing.RecoverySource;
24-
import org.elasticsearch.cluster.routing.RecoverySource.StoreRecoverySource;
24+
import org.elasticsearch.cluster.routing.RecoverySource.EmptyStoreRecoverySource;
2525
import org.elasticsearch.cluster.routing.RoutingNode;
2626
import org.elasticsearch.cluster.routing.RoutingNodes;
2727
import org.elasticsearch.cluster.routing.ShardRouting;
@@ -136,7 +136,7 @@ public RerouteExplanation execute(RoutingAllocation allocation, boolean explain)
136136
}
137137

138138
initializeUnassignedShard(allocation, routingNodes, routingNode, shardRouting, unassignedInfoToUpdate,
139-
StoreRecoverySource.EMPTY_STORE_INSTANCE);
139+
EmptyStoreRecoverySource.INSTANCE);
140140

141141
return new RerouteExplanation(this, allocation.decision(Decision.YES, name() + " (allocation command)", "ignore deciders"));
142142
}

‎server/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateStalePrimaryAllocationCommand.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -129,7 +129,8 @@ public RerouteExplanation execute(RoutingAllocation allocation, boolean explain)
129129
"trying to allocate an existing primary shard [" + index + "][" + shardId + "], while no such shard has ever been active");
130130
}
131131

132-
initializeUnassignedShard(allocation, routingNodes, routingNode, shardRouting);
132+
initializeUnassignedShard(allocation, routingNodes, routingNode, shardRouting, null,
133+
RecoverySource.ExistingStoreRecoverySource.FORCE_STALE_PRIMARY_INSTANCE);
133134
return new RerouteExplanation(this, allocation.decision(Decision.YES, name() + " (allocation command)", "ignore deciders"));
134135
}
135136

‎server/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -398,6 +398,9 @@ private void internalRecoverFromStore(IndexShard indexShard) throws IndexShardRe
398398
indexShard.shardPath().resolveTranslog(), maxSeqNo, shardId, indexShard.getPendingPrimaryTerm());
399399
store.associateIndexWithNewTranslog(translogUUID);
400400
} else if (indexShouldExists) {
401+
if (recoveryState.getRecoverySource().shouldBootstrapNewHistoryUUID()) {
402+
store.bootstrapNewHistory();
403+
}
401404
// since we recover from local, just fill the files and size
402405
try {
403406
final RecoveryState.Index index = recoveryState.getIndex();

‎server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -377,7 +377,7 @@ static GroupShardsIterator<SearchShardIterator> getShardsIter(String index, Orig
377377
ArrayList<ShardRouting> unassigned = new ArrayList<>();
378378

379379
ShardRouting routing = ShardRouting.newUnassigned(new ShardId(new Index(index, "_na_"), i), true,
380-
RecoverySource.StoreRecoverySource.EMPTY_STORE_INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foobar"));
380+
RecoverySource.EmptyStoreRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foobar"));
381381
routing = routing.initialize(primaryNode.getId(), i + "p", 0);
382382
routing.started();
383383
started.add(routing);

‎server/src/test/java/org/elasticsearch/cluster/routing/AllocationIdTests.java

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919

2020
package org.elasticsearch.cluster.routing;
2121

22-
import org.elasticsearch.cluster.routing.RecoverySource.StoreRecoverySource;
22+
import org.elasticsearch.cluster.routing.RecoverySource.ExistingStoreRecoverySource;
2323
import org.elasticsearch.common.bytes.BytesReference;
2424
import org.elasticsearch.common.xcontent.ToXContent;
2525
import org.elasticsearch.common.xcontent.XContentFactory;
@@ -37,7 +37,7 @@
3737
public class AllocationIdTests extends ESTestCase {
3838
public void testShardToStarted() {
3939
logger.info("-- create unassigned shard");
40-
ShardRouting shard = ShardRouting.newUnassigned(new ShardId("test","_na_", 0), true, StoreRecoverySource.EXISTING_STORE_INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null));
40+
ShardRouting shard = ShardRouting.newUnassigned(new ShardId("test","_na_", 0), true, ExistingStoreRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null));
4141
assertThat(shard.allocationId(), nullValue());
4242

4343
logger.info("-- initialize the shard");
@@ -57,7 +57,7 @@ public void testShardToStarted() {
5757

5858
public void testSuccessfulRelocation() {
5959
logger.info("-- build started shard");
60-
ShardRouting shard = ShardRouting.newUnassigned(new ShardId("test","_na_", 0), true, StoreRecoverySource.EXISTING_STORE_INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null));
60+
ShardRouting shard = ShardRouting.newUnassigned(new ShardId("test","_na_", 0), true, ExistingStoreRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null));
6161
shard = shard.initialize("node1", null, -1);
6262
shard = shard.moveToStarted();
6363

@@ -80,7 +80,7 @@ public void testSuccessfulRelocation() {
8080

8181
public void testCancelRelocation() {
8282
logger.info("-- build started shard");
83-
ShardRouting shard = ShardRouting.newUnassigned(new ShardId("test","_na_", 0), true, StoreRecoverySource.EXISTING_STORE_INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null));
83+
ShardRouting shard = ShardRouting.newUnassigned(new ShardId("test","_na_", 0), true, ExistingStoreRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null));
8484
shard = shard.initialize("node1", null, -1);
8585
shard = shard.moveToStarted();
8686

@@ -100,7 +100,7 @@ public void testCancelRelocation() {
100100

101101
public void testMoveToUnassigned() {
102102
logger.info("-- build started shard");
103-
ShardRouting shard = ShardRouting.newUnassigned(new ShardId("test","_na_", 0), true, StoreRecoverySource.EXISTING_STORE_INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null));
103+
ShardRouting shard = ShardRouting.newUnassigned(new ShardId("test","_na_", 0), true, ExistingStoreRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null));
104104
shard = shard.initialize("node1", null, -1);
105105
shard = shard.moveToStarted();
106106

‎server/src/test/java/org/elasticsearch/cluster/routing/GroupShardsIteratorTests.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ public void testIterate() {
7777

7878
public ShardRouting newRouting(Index index, int id, boolean started) {
7979
ShardRouting shardRouting = ShardRouting.newUnassigned(new ShardId(index, id), true,
80-
RecoverySource.StoreRecoverySource.EMPTY_STORE_INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
80+
RecoverySource.EmptyStoreRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
8181
shardRouting = ShardRoutingHelper.initialize(shardRouting, "some node");
8282
if (started) {
8383
shardRouting = ShardRoutingHelper.moveToStarted(shardRouting);

0 commit comments

Comments
 (0)