Stop NodeTests from timing out in certain cases (#49202) (#49503)

The NodeTests class contains tests that check behavior when shutting
down a node. This involves starting a node, performing some operation,
stopping the node, and then awaiting the close of the node. Part of
closing a node is the termination of the node's ThreadPool. ThreadPool
termination semantics can be deceiving. The ThreadPool#terminate method
takes a timeout value and the first oddity is that the terminate method
can take two times the timeout value before returning. Internally this
method acts on the ExecutorService instances that are held by the
ThreadPool. First, an orderly shutdown is attempted and pending tasks
are allowed to execute while waiting for the timeout value. If any of
the ExecutorService instances have not terminated, a call is made to
attempt to stop all active tasks (usually using interrupts) and then
waits for up to the timeout value a second time for the termination of
the ExecutorService instances. This means that if use a large value
when waiting for a node to close, we may not attempt to interrupt any
threads that are in a blocking call before the test times out.

In order to avoid causing these tests to time out, this change reduces
the timeout passed to Node#awaitClose to 10 seconds from 1 day. This
will allow blocked threads to be interrupted before the test suite
fails due to the timeout.

Closes #44256
Closes #42350
Closes #44435

Author: jaymode

server/src/test/java/org/elasticsearch/node/NodeTests.java

@@ -18,7 +18,6 @@
  */
 package org.elasticsearch.node;
 
-import org.apache.lucene.util.Constants;
 import org.apache.lucene.util.LuceneTestCase;
 import org.elasticsearch.bootstrap.BootstrapCheck;
 import org.elasticsearch.bootstrap.BootstrapContext;
@@ -151,7 +150,6 @@ private static Settings.Builder baseSettings() {
     }
 
     public void testCloseOnOutstandingTask() throws Exception {
-        assumeFalse("https://github.com/elastic/elasticsearch/issues/44256", Constants.WINDOWS);
         Node node = new MockNode(baseSettings().build(), basePlugins());
         node.start();
         ThreadPool threadpool = node.injector().getInstance(ThreadPool.class);
@@ -164,7 +162,7 @@ public void testCloseOnOutstandingTask() throws Exception {
         threadRunning.await();
         node.close();
         shouldRun.set(false);
-        assertTrue(node.awaitClose(1, TimeUnit.DAYS));
+        assertTrue(node.awaitClose(10L, TimeUnit.SECONDS));
     }
 
     public void testCloseRaceWithTaskExecution() throws Exception {
@@ -210,7 +208,7 @@ public void testCloseRaceWithTaskExecution() throws Exception {
         closeThread.join();
 
         shouldRun.set(false);
-        assertTrue(node.awaitClose(1, TimeUnit.DAYS));
+        assertTrue(node.awaitClose(10L, TimeUnit.SECONDS));
     }
 
     public void testAwaitCloseTimeoutsOnNonInterruptibleTask() throws Exception {
@@ -227,7 +225,7 @@ public void testAwaitCloseTimeoutsOnNonInterruptibleTask() throws Exception {
         node.close();
         assertFalse(node.awaitClose(0, TimeUnit.MILLISECONDS));
         shouldRun.set(false);
-        assertTrue(node.awaitClose(1, TimeUnit.DAYS));
+        assertTrue(node.awaitClose(10L, TimeUnit.SECONDS));
     }
 
     public void testCloseOnInterruptibleTask() throws Exception {
@@ -251,7 +249,7 @@ public void testCloseOnInterruptibleTask() throws Exception {
         });
         threadRunning.await();
         node.close();
-        // close should not interrput ongoing tasks
+        // close should not interrupt ongoing tasks
         assertFalse(interrupted.get());
         // but awaitClose should
         node.awaitClose(0, TimeUnit.SECONDS);
@@ -270,7 +268,7 @@ public void testCloseOnLeakedIndexReaderReference() throws Exception {
         Searcher searcher = shard.acquireSearcher("test");
         node.close();
 
-        IllegalStateException e = expectThrows(IllegalStateException.class, () -> node.awaitClose(1, TimeUnit.DAYS));
+        IllegalStateException e = expectThrows(IllegalStateException.class, () -> node.awaitClose(10L, TimeUnit.SECONDS));
         searcher.close();
         assertThat(e.getMessage(), containsString("Something is leaking index readers or store references"));
     }
@@ -286,7 +284,7 @@ public void testCloseOnLeakedStoreReference() throws Exception {
         shard.store().incRef();
         node.close();
 
-        IllegalStateException e = expectThrows(IllegalStateException.class, () -> node.awaitClose(1, TimeUnit.DAYS));
+        IllegalStateException e = expectThrows(IllegalStateException.class, () -> node.awaitClose(10L, TimeUnit.SECONDS));
         shard.store().decRef();
         assertThat(e.getMessage(), containsString("Something is leaking index readers or store references"));
     }