Skip to content

Commit 130c832

Browse files
vigyasharmaDaveCTurner
vigyasharma
authored and
DaveCTurner
committed
Validate routing commands using updated routing state (#42066)
When multiple commands are called in sequence, fetch shards from mutable, up-to-date routing nodes to ensure each command's changes are visible to subsequent commands. This addresses an issue uncovered during work on #41050.
1 parent aea600f commit 130c832

File tree

4 files changed

+112
-12
lines changed

4 files changed

+112
-12
lines changed

‎server/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateEmptyPrimaryAllocationCommand.java

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -110,13 +110,20 @@ public RerouteExplanation execute(RoutingAllocation allocation, boolean explain)
110110
return explainOrThrowMissingRoutingNode(allocation, explain, discoNode);
111111
}
112112

113-
final ShardRouting shardRouting;
114113
try {
115-
shardRouting = allocation.routingTable().shardRoutingTable(index, shardId).primaryShard();
114+
allocation.routingTable().shardRoutingTable(index, shardId).primaryShard();
116115
} catch (IndexNotFoundException | ShardNotFoundException e) {
117116
return explainOrThrowRejectedCommand(explain, allocation, e);
118117
}
119-
if (shardRouting.unassigned() == false) {
118+
119+
ShardRouting shardRouting = null;
120+
for (ShardRouting shard : allocation.routingNodes().unassigned()) {
121+
if (shard.getIndexName().equals(index) && shard.getId() == shardId && shard.primary()) {
122+
shardRouting = shard;
123+
break;
124+
}
125+
}
126+
if (shardRouting == null) {
120127
return explainOrThrowRejectedCommand(explain, allocation, "primary [" + index + "][" + shardId + "] is already assigned");
121128
}
122129

‎server/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateReplicaAllocationCommand.java

Lines changed: 20 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@
2323
import org.elasticsearch.cluster.routing.RoutingNode;
2424
import org.elasticsearch.cluster.routing.RoutingNodes;
2525
import org.elasticsearch.cluster.routing.ShardRouting;
26-
import org.elasticsearch.cluster.routing.ShardRoutingState;
2726
import org.elasticsearch.cluster.routing.allocation.RerouteExplanation;
2827
import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
2928
import org.elasticsearch.cluster.routing.allocation.decider.Decision;
@@ -35,6 +34,7 @@
3534
import org.elasticsearch.index.shard.ShardNotFoundException;
3635

3736
import java.io.IOException;
37+
import java.util.ArrayList;
3838
import java.util.List;
3939

4040
/**
@@ -101,20 +101,34 @@ public RerouteExplanation execute(RoutingAllocation allocation, boolean explain)
101101
return explainOrThrowMissingRoutingNode(allocation, explain, discoNode);
102102
}
103103

104-
final ShardRouting primaryShardRouting;
105104
try {
106-
primaryShardRouting = allocation.routingTable().shardRoutingTable(index, shardId).primaryShard();
105+
allocation.routingTable().shardRoutingTable(index, shardId).primaryShard();
107106
} catch (IndexNotFoundException | ShardNotFoundException e) {
108107
return explainOrThrowRejectedCommand(explain, allocation, e);
109108
}
110-
if (primaryShardRouting.unassigned()) {
109+
110+
ShardRouting primaryShardRouting = null;
111+
for (RoutingNode node : allocation.routingNodes()) {
112+
for (ShardRouting shard : node) {
113+
if (shard.getIndexName().equals(index) && shard.getId() == shardId && shard.primary()) {
114+
primaryShardRouting = shard;
115+
break;
116+
}
117+
}
118+
}
119+
if (primaryShardRouting == null) {
111120
return explainOrThrowRejectedCommand(explain, allocation,
112121
"trying to allocate a replica shard [" + index + "][" + shardId +
113122
"], while corresponding primary shard is still unassigned");
114123
}
115124

116-
List<ShardRouting> replicaShardRoutings =
117-
allocation.routingTable().shardRoutingTable(index, shardId).replicaShardsWithState(ShardRoutingState.UNASSIGNED);
125+
List<ShardRouting> replicaShardRoutings = new ArrayList<>();
126+
for (ShardRouting shard : allocation.routingNodes().unassigned()) {
127+
if (shard.getIndexName().equals(index) && shard.getId() == shardId && shard.primary() == false) {
128+
replicaShardRoutings.add(shard);
129+
}
130+
}
131+
118132
ShardRouting shardRouting;
119133
if (replicaShardRoutings.isEmpty()) {
120134
return explainOrThrowRejectedCommand(explain, allocation,

‎server/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateStalePrimaryAllocationCommand.java

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -108,13 +108,20 @@ public RerouteExplanation execute(RoutingAllocation allocation, boolean explain)
108108
return explainOrThrowMissingRoutingNode(allocation, explain, discoNode);
109109
}
110110

111-
final ShardRouting shardRouting;
112111
try {
113-
shardRouting = allocation.routingTable().shardRoutingTable(index, shardId).primaryShard();
112+
allocation.routingTable().shardRoutingTable(index, shardId).primaryShard();
114113
} catch (IndexNotFoundException | ShardNotFoundException e) {
115114
return explainOrThrowRejectedCommand(explain, allocation, e);
116115
}
117-
if (shardRouting.unassigned() == false) {
116+
117+
ShardRouting shardRouting = null;
118+
for (ShardRouting shard : allocation.routingNodes().unassigned()) {
119+
if (shard.getIndexName().equals(index) && shard.getId() == shardId && shard.primary()) {
120+
shardRouting = shard;
121+
break;
122+
}
123+
}
124+
if (shardRouting == null) {
118125
return explainOrThrowRejectedCommand(explain, allocation, "primary [" + index + "][" + shardId + "] is already assigned");
119126
}
120127

‎server/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationCommandsTests.java

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -677,4 +677,76 @@ public void testMoveShardFromNonDataNode() {
677677
assertEquals("[move_allocation] can't move [test][0] from " + node2 + " to " + node1 +
678678
": source [" + node2.getName() + "] is not a data node.", e.getMessage());
679679
}
680+
681+
public void testConflictingCommandsInSingleRequest() {
682+
AllocationService allocation = createAllocationService(Settings.builder()
683+
.put(EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(), "none")
684+
.put(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), "none")
685+
.build());
686+
687+
final String index1 = "test1";
688+
final String index2 = "test2";
689+
final String index3 = "test3";
690+
logger.info("--> building initial routing table");
691+
MetaData metaData = MetaData.builder()
692+
.put(IndexMetaData.builder(index1).settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1)
693+
.putInSyncAllocationIds(0, Collections.singleton("randomAllocID"))
694+
.putInSyncAllocationIds(1, Collections.singleton("randomAllocID2")))
695+
.put(IndexMetaData.builder(index2).settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1)
696+
.putInSyncAllocationIds(0, Collections.singleton("randomAllocID"))
697+
.putInSyncAllocationIds(1, Collections.singleton("randomAllocID2")))
698+
.put(IndexMetaData.builder(index3).settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1)
699+
.putInSyncAllocationIds(0, Collections.singleton("randomAllocID"))
700+
.putInSyncAllocationIds(1, Collections.singleton("randomAllocID2")))
701+
.build();
702+
RoutingTable routingTable = RoutingTable.builder()
703+
.addAsRecovery(metaData.index(index1))
704+
.addAsRecovery(metaData.index(index2))
705+
.addAsRecovery(metaData.index(index3))
706+
.build();
707+
ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
708+
.metaData(metaData).routingTable(routingTable).build();
709+
710+
final String node1 = "node1";
711+
final String node2 = "node2";
712+
clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder()
713+
.add(newNode(node1))
714+
.add(newNode(node2))
715+
).build();
716+
final ClusterState finalClusterState = allocation.reroute(clusterState, "reroute");
717+
718+
logger.info("--> allocating same index primary in multiple commands should fail");
719+
assertThat(expectThrows(IllegalArgumentException.class, () -> {
720+
allocation.reroute(finalClusterState,
721+
new AllocationCommands(
722+
new AllocateStalePrimaryAllocationCommand(index1, 0, node1, true),
723+
new AllocateStalePrimaryAllocationCommand(index1, 0, node2, true)
724+
), false, false);
725+
}).getMessage(), containsString("primary [" + index1 + "][0] is already assigned"));
726+
727+
assertThat(expectThrows(IllegalArgumentException.class, () -> {
728+
allocation.reroute(finalClusterState,
729+
new AllocationCommands(
730+
new AllocateEmptyPrimaryAllocationCommand(index2, 0, node1, true),
731+
new AllocateEmptyPrimaryAllocationCommand(index2, 0, node2, true)
732+
), false, false);
733+
}).getMessage(), containsString("primary [" + index2 + "][0] is already assigned"));
734+
735+
736+
clusterState = allocation.reroute(clusterState,
737+
new AllocationCommands(new AllocateEmptyPrimaryAllocationCommand(index3, 0, node1, true)), false, false).getClusterState();
738+
clusterState = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
739+
740+
final ClusterState updatedClusterState = clusterState;
741+
assertThat(updatedClusterState.getRoutingNodes().node(node1).shardsWithState(STARTED).size(), equalTo(1));
742+
743+
logger.info("--> subsequent replica allocation fails as all configured replicas have been allocated");
744+
assertThat(expectThrows(IllegalArgumentException.class, () -> {
745+
allocation.reroute(updatedClusterState,
746+
new AllocationCommands(
747+
new AllocateReplicaAllocationCommand(index3, 0, node2),
748+
new AllocateReplicaAllocationCommand(index3, 0, node2)
749+
), false, false);
750+
}).getMessage(), containsString("all copies of [" + index3 + "][0] are already assigned. Use the move allocation command instead"));
751+
}
680752
}

0 commit comments

Comments
 (0)