Suppress warning on request if transport not ready (#69686)

Today we bind to our transport address(es) very early in the startup of
a node so that we know the addresses to which we're bound, even though
we are not yet ready to handle any requests. If we receive a request in
this state then we throw an `IllegalStateException` which results in a
logged warning and the connection being closed. In practice, this
happens straight away since the first request on the connection, the
handshake, is sent as soon as it's open.

This commit introduces a `TransportNotReadyException` for this specific
case, and suppresses the noisy logging on such exceptions.

Relates #44939
Relates #16746
Closes #61356

Author: DaveCTurner

server/src/main/java/org/elasticsearch/transport/TcpTransport.java

@@ -633,6 +633,9 @@ static void handleException(TcpChannel channel, Exception e, Lifecycle lifecycle
         } else if (e instanceof StreamCorruptedException) {
             logger.warn(() -> new ParameterizedMessage("{}, [{}], closing connection", e.getMessage(), channel));
             CloseableChannel.closeChannel(channel);
+        } else if (e instanceof TransportNotReadyException) {
+            logger.debug(() -> new ParameterizedMessage("{} on [{}], closing connection", e.getMessage(), channel));
+            CloseableChannel.closeChannel(channel);
         } else {
             logger.warn(() -> new ParameterizedMessage("exception caught on transport layer [{}], closing connection", channel), e);
             // close the channel, which will cause a node to be disconnected if relevant

server/src/main/java/org/elasticsearch/transport/TransportNotReadyException.java

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.transport;
+
+/**
+ * Exception indicating that the {@link TransportService} received a request before it was ready to handle it, so the request should be
+ * rejected and the connection closed. Never goes over the wire, so it's just a {@link RuntimeException}.
+ */
+public class TransportNotReadyException extends RuntimeException {
+    public TransportNotReadyException() {
+        super("transport not ready yet to handle incoming requests");
+    }
+
+    @Override
+    public Throwable fillInStackTrace() {
+        return this; // stack trace is uninteresting
+    }
+}

server/src/main/java/org/elasticsearch/transport/TransportService.java

@@ -1043,7 +1043,7 @@ public <Request extends TransportRequest> void registerRequestHandler(String act
     @Override
     public void onRequestReceived(long requestId, String action) {
         if (handleIncomingRequests.get() == false) {
-            throw new IllegalStateException("transport not ready yet to handle incoming requests");
+            throw new TransportNotReadyException();
         }
         if (tracerLog.isTraceEnabled() && shouldTraceAction(action)) {
             tracerLog.trace("[{}][{}] received request", requestId, action);

server/src/test/java/org/elasticsearch/transport/TcpTransportTests.java

@@ -368,6 +368,9 @@ public void testExceptionHandling() throws IllegalAccessException {
         testExceptionHandling(new StreamCorruptedException("simulated"),
             new MockLogAppender.SeenEventExpectation("message", "org.elasticsearch.transport.TcpTransport",
                 Level.WARN, "simulated, [*], closing connection"));
+        testExceptionHandling(new TransportNotReadyException(),
+            new MockLogAppender.SeenEventExpectation("message", "org.elasticsearch.transport.TcpTransport",
+                Level.DEBUG, "transport not ready yet to handle incoming requests on [*], closing connection"));
     }
 
     private void testExceptionHandling(Exception exception,