Fix up committed configuration on fake Zen1 nodes (#40065)

Today we test Zen1/Zen2 compatibility by running 7.x nodes with a "fake" Zen1
implementation. However this is not a truly faithful test because these nodes
do known how to properly deserialize a 7.x cluster state, voting configurations
and all, whereas a real Zen1 node is in 6.7 and ignores the coordination
metadata.

We only ever apply a cluster state that's been committed, which in Zen2
involves setting the last-committed configuration to equal the last-accepted
configuration. Zen1 knows nothing about this adjustment, so it is possible for
these to differ. This breaks the assertion that the cluster states are equal on
all nodes after integration tests.

This commit fixes this by implementing this adjustment in Zen1 before applying
a cluster state.

Fixes #40055.

Author: DaveCTurner

server/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java

@@ -460,6 +460,12 @@ private void applyChanges(UpdateTask task, ClusterState previousClusterState, Cl
 
         nodeConnectionsService.disconnectFromNodesExcept(newClusterState.nodes());
 
+        assert newClusterState.coordinationMetaData().getLastAcceptedConfiguration()
+            .equals(newClusterState.coordinationMetaData().getLastCommittedConfiguration())
+            : newClusterState.coordinationMetaData().getLastAcceptedConfiguration()
+            + " vs " + newClusterState.coordinationMetaData().getLastCommittedConfiguration()
+            + " on " + newClusterState.nodes().getLocalNode();
+
         logger.debug("set locally applied cluster state to version {}", newClusterState.version());
         state.set(newClusterState);
 

server/src/main/java/org/elasticsearch/discovery/zen/PendingClusterStatesQueue.java

@@ -22,6 +22,8 @@
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.coordination.CoordinationMetaData;
+import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 
 import java.util.ArrayList;
@@ -246,7 +248,16 @@ public synchronized ClusterState getNextClusterStateToProcess() {
             }
         }
         assert stateToProcess.committed() : "should only return committed cluster state. found " + stateToProcess.state;
-        return stateToProcess.state;
+
+        final ClusterState committedState = stateToProcess.state;
+        final CoordinationMetaData coordinationMetaData = committedState.coordinationMetaData();
+        if (coordinationMetaData.getLastAcceptedConfiguration().equals(coordinationMetaData.getLastCommittedConfiguration())) {
+            return committedState;
+        } else {
+            return ClusterState.builder(committedState).metaData(MetaData.builder(committedState.metaData())
+                .coordinationMetaData(CoordinationMetaData.builder(coordinationMetaData)
+                    .lastCommittedConfiguration(committedState.getLastAcceptedConfiguration()).build())).build();
+        }
     }
 
     /** returns all pending states, committed or not */