Merge pull request #168 from elastic/master

🤖 ESQL: Merge upstream

Author: github-actions[bot]

docs/changelog/87723.yaml

@@ -0,0 +1,5 @@
+pr: 87723 
+summary: Optimize log cluster health performance.
+area: Allocation
+type: enhancement
+issues: []

docs/reference/migration/migrate_8_3.asciidoc

@@ -9,9 +9,6 @@ your application to {es} 8.3.
 
 See also <<release-highlights>> and <<es-release-notes>>.
 
-coming::[8.3.0-SNAPSHOT]
-
-
 
 [discrete]
 [[breaking-changes-8.3]]

docs/reference/release-notes.asciidoc

@@ -7,6 +7,7 @@
 This section summarizes the changes in each release.
 
 * <<release-notes-8.4.0>>
+* <<release-notes-8.3.1>>
 * <<release-notes-8.3.0>>
 * <<release-notes-8.2.3>>
 * <<release-notes-8.2.2>>
@@ -27,6 +28,7 @@ This section summarizes the changes in each release.
 --
 
 include::release-notes/8.4.0.asciidoc[]
+include::release-notes/8.3.1.asciidoc[]
 include::release-notes/8.3.0.asciidoc[]
 include::release-notes/8.2.3.asciidoc[]
 include::release-notes/8.2.2.asciidoc[]

docs/reference/release-notes/8.3.1.asciidoc

@@ -0,0 +1,39 @@
+[[release-notes-8.3.1]]
+== {es} version 8.3.1
+
+Also see <<breaking-changes-8.3,Breaking changes in 8.3>>.
+
+[[bug-8.3.1]]
+[float]
+=== Bug fixes
+
+Audit::
+* Support removing ignore filters for audit logging {es-pull}87675[#87675] (issue: {es-issue}68588[#68588])
+
+Ingest::
+* Don't ignore pipeline for upserts in bulk api {es-pull}87719[#87719] (issue: {es-issue}87131[#87131])
+* Geoip processor should respect the `ignore_missing` in case of missing database {es-pull}87793[#87793] (issue: {es-issue}87345[#87345])
+
+Machine Learning::
+* Improve trained model stats API performance {es-pull}87978[#87978]
+
+Snapshot/Restore::
+* Use the provided SAS token without SDK sanitation that can produce invalid signatures {es-pull}88155[#88155] (issue: {es-issue}88140[#88140])
+
+Stats::
+* Run `TransportClusterInfoActions` on MANAGEMENT pool {es-pull}87679[#87679]
+
+Transform::
+* Execute `_refresh` (with system permissions) separately from `delete by query` (with user permissions) {es-pull}88005[#88005] (issue: {es-issue}88001[#88001])
+
+[[enhancement-8.3.1]]
+[float]
+=== Enhancements
+
+Discovery-Plugins::
+* Remove redundant jackson dependencies from discovery-azure {es-pull}87898[#87898]
+
+Performance::
+* Warn about impact of large readahead on search {es-pull}88007[#88007]
+
+

server/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java

@@ -61,18 +61,21 @@ public class IndexRoutingTable implements SimpleDiffable<IndexRoutingTable> {
     // note, we assume that when the index routing is created, ShardRoutings are created for all possible number of
     // shards with state set to UNASSIGNED
     private final IndexShardRoutingTable[] shards;
-
+    private final boolean allShardsActive;
     private final List<ShardRouting> allActiveShards;
 
     IndexRoutingTable(Index index, IndexShardRoutingTable[] shards) {
         this.index = index;
         this.shuffler = new RotationShardShuffler(Randomness.get().nextInt());
         this.shards = shards;
+        int totalShardCount = 0;
         List<ShardRouting> allActiveShards = new ArrayList<>();
         for (IndexShardRoutingTable shard : shards) {
             allActiveShards.addAll(shard.activeShards());
+            totalShardCount += shard.size();
         }
         this.allActiveShards = CollectionUtils.wrapUnmodifiableOrEmptySingleton(allActiveShards);
+        this.allShardsActive = totalShardCount == allActiveShards.size();
     }
 
     /**
@@ -217,6 +220,10 @@ public boolean allPrimaryShardsActive() {
         return primaryShardsActive() == shards.length;
     }
 
+    public boolean allShardsActive() {
+        return this.allShardsActive;
+    }
+
     /**
      * Calculates the number of primary shards in active state in routing table
      *

server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java

@@ -14,13 +14,14 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.RestoreInProgress;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
-import org.elasticsearch.cluster.health.ClusterStateHealth;
 import org.elasticsearch.cluster.metadata.AutoExpandReplicas;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.metadata.SingleNodeShutdownMetadata;
 import org.elasticsearch.cluster.metadata.SingleNodeShutdownMetadata.Type;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.IndexRoutingTable;
+import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
 import org.elasticsearch.cluster.routing.RerouteService;
 import org.elasticsearch.cluster.routing.RoutingNode;
 import org.elasticsearch.cluster.routing.RoutingNodes;
@@ -39,6 +40,7 @@
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.gateway.GatewayAllocator;
 import org.elasticsearch.gateway.PriorityComparator;
+import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.snapshots.SnapshotsInfoService;
 
 import java.util.ArrayList;
@@ -55,6 +57,7 @@
 
 import static java.util.Collections.emptyList;
 import static java.util.Collections.singletonList;
+import static org.elasticsearch.cluster.health.ClusterShardHealth.getInactivePrimaryHealth;
 import static org.elasticsearch.cluster.routing.UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING;
 
 /**
@@ -168,7 +171,7 @@ private static ClusterState buildResultAndLogHealthChange(ClusterState oldState,
         }
         final ClusterState newState = newStateBuilder.build();
 
-        logClusterHealthStateChange(new ClusterStateHealth(oldState), new ClusterStateHealth(newState), reason);
+        logClusterHealthStateChange(oldState, newState, reason);
 
         return newState;
     }
@@ -495,13 +498,10 @@ public ClusterState reroute(ClusterState clusterState, String reason) {
         return buildResultAndLogHealthChange(clusterState, allocation, reason);
     }
 
-    private static void logClusterHealthStateChange(
-        ClusterStateHealth previousStateHealth,
-        ClusterStateHealth newStateHealth,
-        String reason
-    ) {
-        ClusterHealthStatus previousHealth = previousStateHealth.getStatus();
-        ClusterHealthStatus currentHealth = newStateHealth.getStatus();
+    private static void logClusterHealthStateChange(final ClusterState previousState, final ClusterState newState, String reason) {
+        ClusterHealthStatus previousHealth = getHealthStatus(previousState);
+        ClusterHealthStatus currentHealth = getHealthStatus(newState);
+
         if (previousHealth.equals(currentHealth) == false) {
             logger.info(
                 new ESLogMessage("Cluster health status changed from [{}] to [{}] (reason: [{}]).").argAndField(
@@ -513,6 +513,40 @@ private static void logClusterHealthStateChange(
         }
     }
 
+    public static ClusterHealthStatus getHealthStatus(final ClusterState clusterState) {
+        if (clusterState.blocks().hasGlobalBlockWithStatus(RestStatus.SERVICE_UNAVAILABLE)) {
+            return ClusterHealthStatus.RED;
+        }
+
+        ClusterHealthStatus computeStatus = ClusterHealthStatus.GREEN;
+        for (String index : clusterState.metadata().getConcreteAllIndices()) {
+            IndexRoutingTable indexRoutingTable = clusterState.routingTable().index(index);
+            if (indexRoutingTable == null) {
+                continue;
+            }
+            if (indexRoutingTable.allShardsActive()) {
+                // GREEN index
+                continue;
+            }
+
+            for (int i = 0; i < indexRoutingTable.size(); i++) {
+                IndexShardRoutingTable indexShardRoutingTable = indexRoutingTable.shard(i);
+                ShardRouting primary = indexShardRoutingTable.primaryShard();
+                if (primary.active()) {
+                    // index has inactive replicas
+                    computeStatus = ClusterHealthStatus.YELLOW;
+                    continue;
+                }
+                computeStatus = getInactivePrimaryHealth(primary);
+                if (computeStatus == ClusterHealthStatus.RED) {
+                    logger.debug("One of inactive primary shard {} causes cluster state RED.", primary.shardId());
+                    return ClusterHealthStatus.RED;
+                }
+            }
+        }
+        return computeStatus;
+    }
+
     private static boolean hasDeadNodes(RoutingAllocation allocation) {
         for (RoutingNode routingNode : allocation.routingNodes()) {
             if (allocation.nodes().getDataNodes().containsKey(routingNode.nodeId()) == false) {