Blocked allocations on primary causes RED health

If the allocation decision for a primary shard was NO, this should
cause the cluster health for the shard to go RED, even if the shard
belongs to a newly created index or is part of cluster recovery.

Relates elastic#9126

Author: Ali Beyad

core/src/main/java/org/elasticsearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java

@@ -101,9 +101,7 @@ protected void masterOperation(IndicesShardStoresRequest request, ClusterState s
             }
             for (IndexShardRoutingTable routing : indexShardRoutingTables) {
                 final int shardId = routing.shardId().id();
-                ClusterShardHealth shardHealth = new ClusterShardHealth(shardId,
-                                                                        routing,
-                                                                        indexMetaData.activeAllocationIds(shardId).isEmpty());
+                ClusterShardHealth shardHealth = new ClusterShardHealth(shardId, routing, indexMetaData);
                 if (request.shardStatuses().contains(shardHealth.getStatus())) {
                     shardIdsToFetch.add(routing.shardId());
                 }

core/src/main/java/org/elasticsearch/cluster/health/ClusterIndexHealth.java

@@ -54,7 +54,7 @@ public ClusterIndexHealth(final IndexMetaData indexMetaData, final IndexRoutingT
 
         for (IndexShardRoutingTable shardRoutingTable : indexRoutingTable) {
             int shardId = shardRoutingTable.shardId().id();
-            shards.put(shardId, new ClusterShardHealth(shardId, shardRoutingTable, indexMetaData.activeAllocationIds(shardId).isEmpty()));
+            shards.put(shardId, new ClusterShardHealth(shardId, shardRoutingTable, indexMetaData));
         }
 
         // update the index status

core/src/main/java/org/elasticsearch/cluster/health/ClusterShardHealth.java

@@ -19,9 +19,12 @@
 
 package org.elasticsearch.cluster.health;
 
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
+import org.elasticsearch.cluster.routing.UnassignedInfo.AllocationStatus;
+import org.elasticsearch.cluster.routing.UnassignedInfo.Reason;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -38,7 +41,7 @@ public final class ClusterShardHealth implements Writeable {
     private final int unassignedShards;
     private final boolean primaryActive;
 
-    public ClusterShardHealth(final int shardId, final IndexShardRoutingTable shardRoutingTable, final boolean noActiveAllocationIds) {
+    public ClusterShardHealth(final int shardId, final IndexShardRoutingTable shardRoutingTable, final IndexMetaData indexMetaData) {
         this.shardId = shardId;
         int computeActiveShards = 0;
         int computeRelocatingShards = 0;
@@ -66,7 +69,7 @@ public ClusterShardHealth(final int shardId, final IndexShardRoutingTable shardR
                 computeStatus = ClusterHealthStatus.YELLOW;
             }
         } else {
-            computeStatus = UnassignedInfo.unassignedPrimaryShardHealth(primaryRouting.unassignedInfo(), noActiveAllocationIds);
+            computeStatus = getInactivePrimaryHealth(primaryRouting, indexMetaData);
         }
         this.status = computeStatus;
         this.activeShards = computeActiveShards;
@@ -125,4 +128,35 @@ public void writeTo(final StreamOutput out) throws IOException {
         out.writeBoolean(primaryActive);
     }
 
+    /**
+     * Checks if an inactive primary shard should cause the cluster health to go RED.
+     *
+     * Normally, an inactive primary shard in an index should cause the cluster health to be RED.  However,
+     * there are exceptions where a health status of RED is inappropriate, namely in these scenarios:
+     *   1. Index Creation.  When an index is first created, the primary shards are in the initializing state, so
+     *      there is a small window where the cluster health is RED due to the primaries not being activated yet.
+     *      However, this leads to a false sense that the cluster is in an unhealthy state, when in reality, its
+     *      simply a case of needing to wait for the primaries to initialize.
+     *   2. When a cluster is in the recovery state, and the shard never had any allocation ids assigned to it,
+     *      which indicates the index was created and before allocation of the primary occurred for this shard,
+     *      a cluster restart happened.
+     *
+     * Here, we check for these scenarios and set the cluster health to YELLOW if any are applicable.
+     *
+     * NB: this method should *not* be called on active shards nor on non-primary shards.
+     */
+    public static ClusterHealthStatus getInactivePrimaryHealth(final ShardRouting shardRouting, final IndexMetaData indexMetaData) {
+        assert shardRouting.primary() : "cannot invoke on a replica shard: " + shardRouting;
+        assert shardRouting.active() == false : "cannot invoke on an active shard: " + shardRouting;
+        assert shardRouting.unassignedInfo() != null : "cannot invoke on a shard with no UnassignedInfo: " + shardRouting;
+        final UnassignedInfo unassignedInfo = shardRouting.unassignedInfo();
+        if (unassignedInfo.getLastAllocationStatus() != AllocationStatus.DECIDERS_NO
+                && shardRouting.allocatedPostIndexCreate(indexMetaData) == false
+                && (unassignedInfo.getReason() == Reason.INDEX_CREATED || unassignedInfo.getReason() == Reason.CLUSTER_RECOVERED)) {
+            return ClusterHealthStatus.YELLOW;
+        } else {
+            return ClusterHealthStatus.RED;
+        }
+    }
+
 }

core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java

@@ -25,6 +25,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.UnassignedInfo.AllocationStatus;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.collect.Tuple;
@@ -641,14 +642,27 @@ public List<ShardRouting> ignored() {
          * Should be used with caution, typically,
          * the correct usage is to removeAndIgnore from the iterator.
          * @see #ignored()
-         * @see UnassignedIterator#removeAndIgnore()
+         * @see UnassignedIterator#removeAndIgnore(AllocationStatus)
          * @see #isIgnoredEmpty()
+         * @return true iff the decision caused a change to the unassigned info
          */
-        public void ignoreShard(ShardRouting shard) {
+        public boolean ignoreShard(ShardRouting shard, AllocationStatus allocationStatus) {
+            boolean changed = false;
             if (shard.primary()) {
                 ignoredPrimaries++;
+                UnassignedInfo currInfo = shard.unassignedInfo();
+                assert currInfo != null;
+                if (allocationStatus.equals(currInfo.getLastAllocationStatus()) == false) {
+                    UnassignedInfo newInfo = new UnassignedInfo(currInfo.getReason(), currInfo.getMessage(), currInfo.getFailure(),
+                                                                currInfo.getNumFailedAllocations(), currInfo.getUnassignedTimeInNanos(),
+                                                                currInfo.getUnassignedTimeInMillis(), currInfo.isDelayed(),
+                                                                allocationStatus);
+                    shard = shard.updateUnassignedInfo(newInfo);
+                    changed = true;
+                }
             }
             ignored.add(shard);
+            return changed;
         }
 
         public class UnassignedIterator implements Iterator<ShardRouting> {
@@ -685,10 +699,13 @@ public ShardRouting initialize(String nodeId, @Nullable String existingAllocatio
              * will be added back to unassigned once the metadata is constructed again).
              * Typically this is used when an allocation decision prevents a shard from being allocated such
              * that subsequent consumers of this API won't try to allocate this shard again.
+             *
+             * @param attempt the result of the allocation attempt
+             * @return true iff the decision caused an update to the unassigned info
              */
-            public void removeAndIgnore() {
+            public boolean removeAndIgnore(AllocationStatus attempt) {
                 innerRemove();
-                ignoreShard(current);
+                return ignoreShard(current, attempt);
             }
 
             private void updateShardRouting(ShardRouting shardRouting) {
@@ -721,7 +738,7 @@ public ShardRouting demotePrimaryToReplicaShard() {
             }
 
             /**
-             * Unsupported operation, just there for the interface. Use {@link #removeAndIgnore()} or
+             * Unsupported operation, just there for the interface. Use {@link #removeAndIgnore(AllocationStatus)} or
              * {@link #initialize(String, String, long)}.
              */
             @Override
@@ -747,8 +764,8 @@ public boolean isEmpty() {
 
         /**
          * Returns <code>true</code> iff any unassigned shards are marked as temporarily ignored.
-         * @see UnassignedShards#ignoreShard(ShardRouting)
-         * @see UnassignedIterator#removeAndIgnore()
+         * @see UnassignedShards#ignoreShard(ShardRouting, AllocationStatus)
+         * @see UnassignedIterator#removeAndIgnore(AllocationStatus)
          */
         public boolean isIgnoredEmpty() {
             return ignored.isEmpty();
@@ -878,6 +895,7 @@ public static boolean assertShardStats(RoutingNodes routingNodes) {
         assert inactiveShardCount == routingNodes.inactiveShardCount :
                 "Inactive Shard count [" + inactiveShardCount + "] but RoutingNodes returned inactive shards [" + routingNodes.inactiveShardCount + "]";
         assert routingNodes.getRelocatingShardCount() == relocating : "Relocating shards mismatch [" + routingNodes.getRelocatingShardCount() + "] but expected [" + relocating + "]";
+
         return true;
     }