Transport threads and _hot_threads (#90482)

Add special handling for reported CPU time
of transport threads in the hot threads
report. Transport threads always run, but
their other time is typically idle instead
of actually runnable.

Closes #90334

Author: grcevski

docs/changelog/90482.yaml

@@ -0,0 +1,6 @@
+pr: 90482
+summary: Transport threads and `_hot_threads`
+area: Infra/Core
+type: enhancement
+issues:
+ - 90334

server/src/main/java/org/elasticsearch/monitor/jvm/HotThreads.java

@@ -15,6 +15,7 @@
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.transport.Transports;
 
 import java.lang.management.ManagementFactory;
 import java.lang.management.ThreadInfo;
@@ -313,12 +314,15 @@ String innerDetect(ThreadMXBean threadBean, SunThreadInfo sunThreadInfo, long cu
                 case CPU -> {
                     double percentCpu = getTimeSharePercentage(topThread.getCpuTime());
                     double percentOther = getTimeSharePercentage(topThread.getOtherTime());
+                    double percentTotal = (Transports.isTransportThread(threadName)) ? percentCpu : percentOther + percentCpu;
+                    String otherLabel = (Transports.isTransportThread(threadName)) ? "idle" : "other";
                     sb.append(
                         String.format(
                             Locale.ROOT,
-                            "%n%4.1f%% [cpu=%1.1f%%, other=%1.1f%%] (%s out of %s) %s usage by thread '%s'%n",
-                            percentOther + percentCpu,
+                            "%n%4.1f%% [cpu=%1.1f%%, %s=%1.1f%%] (%s out of %s) %s usage by thread '%s'%n",
+                            percentTotal,
                             percentCpu,
+                            otherLabel,
                             percentOther,
                             TimeValue.timeValueNanos(topThread.getCpuTime() + topThread.getOtherTime()),
                             interval,
@@ -412,8 +416,8 @@ static int similarity(ThreadInfo threadInfo, ThreadInfo threadInfo0) {
 
     static class ThreadTimeAccumulator {
         private final long threadId;
+        private final String threadName;
         private final TimeValue interval;
-
         private long cpuTime;
         private long blockedTime;
         private long waitedTime;
@@ -425,6 +429,7 @@ static class ThreadTimeAccumulator {
             this.cpuTime = cpuTime;
             this.allocatedBytes = allocatedBytes;
             this.threadId = info.getThreadId();
+            this.threadName = info.getThreadName();
             this.interval = interval;
         }
 
@@ -457,6 +462,8 @@ public long getRunnableTime() {
             // not running, or it has been asleep forever.
             if (getCpuTime() == 0) {
                 return 0;
+            } else if (Transports.isTransportThread(threadName)) {
+                return getCpuTime();
             }
             return Math.max(interval.nanos() - getWaitedTime() - getBlockedTime(), 0);
         }
@@ -468,7 +475,7 @@ public long getOtherTime() {
                 return 0;
             }
 
-            return Math.max(getRunnableTime() - getCpuTime(), 0);
+            return Math.max(interval.nanos() - getWaitedTime() - getBlockedTime() - getCpuTime(), 0);
         }
 
         public long getBlockedTime() {

server/src/main/java/org/elasticsearch/transport/Transports.java

@@ -37,7 +37,16 @@ public enum Transports {
      * networking threads.
      */
     public static boolean isTransportThread(Thread t) {
-        final String threadName = t.getName();
+        return isTransportThread(t.getName());
+    }
+
+    /**
+     * Utility method to detect whether a thread is a network thread. Typically
+     * used in assertions to make sure that we do not call blocking code from
+     * networking threads.
+     * @param threadName the name of the thread
+     */
+    public static boolean isTransportThread(String threadName) {
         for (String s : TRANSPORT_THREAD_NAMES) {
             if (threadName.contains(s)) {
                 return true;