Consider node ready on any file settings applied (#107738)

Readiness is blocked until file based settings having been applied. Yet in
the case that an error occurs in a subsequent file based settings
update, readiness will stop working. This was unintentional; if file
based settings have already been applied once, subsequent failures to
apply file based settings should not affect the running system.

This commit adjusts the readiness check to consider any file based
settings having been applied as allowing the node to be ready.

In a follow up (#107744) we are going to prevent readiness in case 
initial file settings of a new cluster cannot be applied.
---------

Co-authored-by: Moritz Mack <[email protected]>

Author: rjernst

server/src/internalClusterTest/java/org/elasticsearch/readiness/ReadinessClusterIT.java

@@ -247,6 +247,7 @@ private void writeFileSettings(String json) throws Exception {
         logger.info("--> New file settings: [{}]", Strings.format(json, version));
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/107744")
     public void testNotReadyOnBadFileSettings() throws Exception {
         internalCluster().setBootstrapMasterNodeIndex(0);
         logger.info("--> start data node / non master node");

server/src/main/java/org/elasticsearch/readiness/ReadinessService.java

@@ -254,7 +254,7 @@ public void clusterChanged(ClusterChangedEvent event) {
     // protected to allow mock service to override
     protected boolean areFileSettingsApplied(ClusterState clusterState) {
         ReservedStateMetadata fileSettingsMetadata = clusterState.metadata().reservedStateMetadata().get(FileSettingsService.NAMESPACE);
-        return fileSettingsMetadata != null && fileSettingsMetadata.errorMetadata() == null;
+        return fileSettingsMetadata != null;
     }
 
     private void setReady(boolean ready) {

server/src/test/java/org/elasticsearch/readiness/ReadinessServiceTests.java

@@ -14,6 +14,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.metadata.NodesShutdownMetadata;
+import org.elasticsearch.cluster.metadata.ReservedStateErrorMetadata;
 import org.elasticsearch.cluster.metadata.ReservedStateMetadata;
 import org.elasticsearch.cluster.metadata.SingleNodeShutdownMetadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -44,8 +45,11 @@
 import java.net.UnknownHostException;
 import java.nio.channels.ServerSocketChannel;
 import java.util.Collections;
+import java.util.List;
 import java.util.Set;
 
+import static org.elasticsearch.cluster.metadata.ReservedStateErrorMetadata.ErrorKind.TRANSIENT;
+
 public class ReadinessServiceTests extends ESTestCase implements ReadinessClientProbe {
     private ClusterService clusterService;
     private ReadinessService readinessService;
@@ -291,4 +295,33 @@ public void testStatusChange() throws Exception {
         readinessService.stop();
         readinessService.close();
     }
+
+    public void testFileSettingsUpdateError() throws Exception {
+        // ensure an update to file settings that results in an error state doesn't cause readiness to fail
+        // since the current file settings already applied cleanly
+        readinessService.start();
+
+        // initially the service isn't ready because initial cluster state has not been applied yet
+        assertFalse(readinessService.ready());
+
+        var fileSettingsState = new ReservedStateMetadata.Builder(FileSettingsService.NAMESPACE).version(21L)
+            .errorMetadata(new ReservedStateErrorMetadata(22L, TRANSIENT, List.of("dummy error")));
+        ClusterState state = ClusterState.builder(new ClusterName("cluster"))
+            .nodes(
+                DiscoveryNodes.builder()
+                    .add(DiscoveryNodeUtils.create("node2", new TransportAddress(TransportAddress.META_ADDRESS, 9201)))
+                    .add(httpTransport.node)
+                    .masterNodeId(httpTransport.node.getId())
+                    .localNodeId(httpTransport.node.getId())
+            )
+            .metadata(new Metadata.Builder().put(fileSettingsState.build()))
+            .build();
+
+        ClusterChangedEvent event = new ClusterChangedEvent("test", state, ClusterState.EMPTY_STATE);
+        readinessService.clusterChanged(event);
+        assertTrue(readinessService.ready());
+
+        readinessService.stop();
+        readinessService.close();
+    }
 }