Improve awareness allocation explanation (#69371)

DaveCTurner · web-flow · commit 1a8f2273a2af · 2021-02-23T18:52:53.000Z
Users sometimes struggle with unassigned shards due to the awareness
decider, especially if the decider is using more zones than expected
(e.g. a typo in an attribute value, or a stale forced-awareness config).
It is hard to see why there are more zones than expected, or even really
to tell from the message that this is the problem.

This commit adds detail about the names of the zones that the allocation
decider is using, so that users can see for themselves whether that is
the source of the problem, and rewords it slightly to make it easier to
understand.
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AwarenessAllocationDecider.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AwarenessAllocationDecider.java
@@ -23,8 +23,10 @@
 import java.util.List;
 import java.util.Map;
 import java.util.function.Function;
+import java.util.stream.StreamSupport;
 
 import static java.util.Collections.emptyList;
+import static java.util.stream.Collectors.toList;
 import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_AUTO_EXPAND_REPLICAS_SETTING;
 
 /**
@@ -186,25 +188,44 @@ private Decision underCapacity(ShardRouting shardRouting, RoutingNode node, Rout
             final int currentNodeCount = shardPerAttribute.get(node.node().getAttributes().get(awarenessAttribute));
             final int maximumNodeCount = (shardCount + numberOfAttributes - 1) / numberOfAttributes; // ceil(shardCount/numberOfAttributes)
             if (currentNodeCount > maximumNodeCount) {
-                return debug ? debugNoTooManyCopies(shardCount, awarenessAttribute, numberOfAttributes, currentNodeCount, maximumNodeCount)
+                return debug ? debugNoTooManyCopies(
+                        shardCount,
+                        awarenessAttribute,
+                        node.node().getAttributes().get(awarenessAttribute),
+                        numberOfAttributes,
+                        StreamSupport.stream(nodesPerAttribute.keys().spliterator(), false).map(c -> c.value).sorted().collect(toList()),
+                        fullValues == null ? null : fullValues.stream().sorted().collect(toList()),
+                        currentNodeCount,
+                        maximumNodeCount)
                         : Decision.NO;
             }
         }
 
         return YES_ALL_MET;
     }
 
-    private static Decision debugNoTooManyCopies(int shardCount, String awarenessAttribute, int numberOfAttributes, int currentNodeCount,
-                                                 int maximumNodeCount) {
+    private static Decision debugNoTooManyCopies(
+            int shardCount,
+            String attributeName,
+            String attributeValue,
+            int numberOfAttributes,
+            List<String> realAttributes,
+            List<String> forcedAttributes,
+            int currentNodeCount,
+            int maximumNodeCount) {
         return Decision.single(Decision.Type.NO, NAME,
-                "there are too many copies of the shard allocated to nodes with attribute [%s], there are [%d] total configured " +
-                        "shard copies for this shard id and [%d] total attribute values, expected the allocated shard count per " +
-                        "attribute [%d] to be less than or equal to the upper bound of the required number of shards per attribute [%d]",
-                awarenessAttribute,
+                "there are [%d] copies of this shard and [%d] values for attribute [%s] (%s from nodes in the cluster and %s) so there " +
+                        "may be at most [%d] copies of this shard allocated to nodes with each value, but (including this copy) there " +
+                        "would be [%d] copies allocated to nodes with [node.attr.%s: %s]",
                 shardCount,
                 numberOfAttributes,
+                attributeName,
+                realAttributes,
+                forcedAttributes == null ? "no forced awareness" : forcedAttributes + " from forced awareness",
+                maximumNodeCount,
                 currentNodeCount,
-                maximumNodeCount);
+                attributeName,
+                attributeValue);
     }
 
     private static Decision debugNoMissingAttribute(String awarenessAttribute, List<String> awarenessAttributes) {
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/AwarenessAllocationTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/AwarenessAllocationTests.java
@@ -17,20 +17,27 @@
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.routing.RoutingNode;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.allocation.command.AllocationCommands;
 import org.elasticsearch.cluster.routing.allocation.command.CancelAllocationCommand;
 import org.elasticsearch.cluster.routing.allocation.command.MoveAllocationCommand;
+import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
 import org.elasticsearch.cluster.routing.allocation.decider.AwarenessAllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.ClusterRebalanceAllocationDecider;
+import org.elasticsearch.cluster.routing.allocation.decider.Decision;
+import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 
 import java.util.HashMap;
 import java.util.Map;
+import java.util.function.UnaryOperator;
+import java.util.stream.StreamSupport;
 
 import static java.util.Collections.emptyMap;
+import static java.util.Collections.singletonList;
 import static java.util.Collections.singletonMap;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.RELOCATING;
@@ -937,4 +944,85 @@ public void testNodesWithoutAttributeAreIgnored() {
         assertTrue(clusterState.getRoutingNodes().node("X-0").isEmpty());
     }
 
+    public void testExplanation() {
+        testExplanation(Settings.builder()
+                        .put(AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING.getKey(), "zone"),
+                UnaryOperator.identity(),
+                "there are [5] copies of this shard and [2] values for attribute [zone] ([a, b] from nodes in the cluster and " +
+                        "no forced awareness) so there may be at most [3] copies of this shard allocated to nodes with each " +
+                        "value, but (including this copy) there would be [4] copies allocated to nodes with [node.attr.zone: a]");
+    }
+
+    public void testExplanationWithMissingAttribute() {
+        testExplanation(Settings.builder()
+                        .put(AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING.getKey(), "zone"),
+                n -> n.add(newNode("X-0", emptyMap())),
+                "there are [5] copies of this shard and [2] values for attribute [zone] ([a, b] from nodes in the cluster and " +
+                        "no forced awareness) so there may be at most [3] copies of this shard allocated to nodes with each " +
+                        "value, but (including this copy) there would be [4] copies allocated to nodes with [node.attr.zone: a]");
+    }
+
+    public void testExplanationWithForcedAttributes() {
+        testExplanation(Settings.builder()
+                        .put(AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING.getKey(), "zone")
+                        .put(AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_FORCE_GROUP_SETTING.getKey() + "zone.values",
+                                "b,c"),
+                UnaryOperator.identity(),
+                "there are [5] copies of this shard and [3] values for attribute [zone] ([a, b] from nodes in the cluster and " +
+                        "[b, c] from forced awareness) so there may be at most [2] copies of this shard allocated to nodes with each " +
+                        "value, but (including this copy) there would be [3] copies allocated to nodes with [node.attr.zone: a]");
+    }
+
+    private void testExplanation(
+            Settings.Builder settingsBuilder,
+            UnaryOperator<DiscoveryNodes.Builder> nodesOperator,
+            String expectedMessage) {
+        final Settings settings = settingsBuilder
+                .build();
+
+        final AllocationService strategy = createAllocationService(settings);
+
+        final Metadata metadata = Metadata.builder()
+                .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(4))
+                .build();
+
+        final ClusterState clusterState = applyStartedShardsUntilNoChange(
+                ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.get(Settings.EMPTY))
+                        .metadata(metadata)
+                        .routingTable(RoutingTable.builder()
+                                .addAsNew(metadata.index("test"))
+                                .build())
+                        .nodes(nodesOperator.apply(DiscoveryNodes.builder()
+                                .add(newNode("A-0", singletonMap("zone", "a")))
+                                .add(newNode("A-1", singletonMap("zone", "a")))
+                                .add(newNode("A-2", singletonMap("zone", "a")))
+                                .add(newNode("A-3", singletonMap("zone", "a")))
+                                .add(newNode("A-4", singletonMap("zone", "a")))
+                                .add(newNode("B-0", singletonMap("zone", "b"))))
+                        ).build(), strategy);
+
+        final ShardRouting unassignedShard = clusterState.getRoutingNodes().shardsWithState(UNASSIGNED).get(0);
+
+        final AwarenessAllocationDecider decider = new AwarenessAllocationDecider(
+                settings,
+                new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
+
+        final RoutingNode emptyNode = StreamSupport.stream(clusterState.getRoutingNodes().spliterator(), false)
+                .filter(RoutingNode::isEmpty).findFirst().orElseThrow(AssertionError::new);
+
+        final RoutingAllocation routingAllocation = new RoutingAllocation(
+                new AllocationDeciders(singletonList(decider)),
+                clusterState.getRoutingNodes(),
+                clusterState,
+                null,
+                null,
+                0L);
+        routingAllocation.debugDecision(true);
+
+        final Decision decision = decider.canAllocate(unassignedShard, emptyNode, routingAllocation);
+        assertThat(decision.type(), equalTo(Decision.Type.NO));
+        assertThat(decision.label(), equalTo("awareness"));
+        assertThat(decision.getExplanation(), equalTo(expectedMessage));
+    }
+
 }
