[7.x] Add fs iotime in Nodes Stats API (#67861) (#72754)

This adds io_time_in_millis to Nodes stats API

(cherry picked from commit 71a9cfe)
Signed-off-by: Andrei Dan <[email protected]>

Co-authored-by: zhangchao <[email protected]>

Author: andreidan

server/src/main/java/org/elasticsearch/monitor/fs/FsInfo.java

@@ -174,6 +174,8 @@ public static class DeviceStats implements Writeable, ToXContentFragment {
         final long previousWritesCompleted;
         final long currentSectorsWritten;
         final long previousSectorsWritten;
+        final long currentIOTime;
+        final long previousIOTime;
 
         public DeviceStats(
                 final int majorDeviceNumber,
@@ -183,6 +185,7 @@ public DeviceStats(
                 final long currentSectorsRead,
                 final long currentWritesCompleted,
                 final long currentSectorsWritten,
+                final long currentIOTime,
                 final DeviceStats previousDeviceStats) {
             this(
                     majorDeviceNumber,
@@ -195,7 +198,9 @@ public DeviceStats(
                     currentSectorsRead,
                     previousDeviceStats != null ? previousDeviceStats.currentSectorsRead : -1,
                     currentWritesCompleted,
-                    previousDeviceStats != null ? previousDeviceStats.currentWritesCompleted : -1);
+                    previousDeviceStats != null ? previousDeviceStats.currentWritesCompleted : -1,
+                    currentIOTime,
+                    previousDeviceStats != null ? previousDeviceStats.currentIOTime : -1);
         }
 
         private DeviceStats(
@@ -209,7 +214,9 @@ private DeviceStats(
                 final long currentSectorsRead,
                 final long previousSectorsRead,
                 final long currentWritesCompleted,
-                final long previousWritesCompleted) {
+                final long previousWritesCompleted,
+                final long currentIOTime,
+                final long previousIOTime) {
             this.majorDeviceNumber = majorDeviceNumber;
             this.minorDeviceNumber = minorDeviceNumber;
             this.deviceName = deviceName;
@@ -221,6 +228,8 @@ private DeviceStats(
             this.previousSectorsRead = previousSectorsRead;
             this.currentSectorsWritten = currentSectorsWritten;
             this.previousSectorsWritten = previousSectorsWritten;
+            this.currentIOTime = currentIOTime;
+            this.previousIOTime = previousIOTime;
         }
 
         public DeviceStats(StreamInput in) throws IOException {
@@ -235,6 +244,13 @@ public DeviceStats(StreamInput in) throws IOException {
             previousSectorsRead = in.readLong();
             currentSectorsWritten = in.readLong();
             previousSectorsWritten = in.readLong();
+            if (in.getVersion().onOrAfter(Version.V_7_14_0)) {
+                currentIOTime = in.readLong();
+                previousIOTime = in.readLong();
+            } else {
+                currentIOTime = -1;
+                previousIOTime = -1;
+            }
         }
 
         @Override
@@ -250,6 +266,10 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeLong(previousSectorsRead);
             out.writeLong(currentSectorsWritten);
             out.writeLong(previousSectorsWritten);
+            if (out.getVersion().onOrAfter(Version.V_7_14_0)) {
+                out.writeLong(currentIOTime);
+                out.writeLong(previousIOTime);
+            }
         }
 
         public long operations() {
@@ -283,6 +303,12 @@ public long writeKilobytes() {
             return (currentSectorsWritten - previousSectorsWritten) / 2;
         }
 
+        public long ioTimeInMillis() {
+            if (previousIOTime == -1) return -1;
+
+            return (currentIOTime - previousIOTime);
+        }
+
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             builder.field("device_name", deviceName);
@@ -291,6 +317,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             builder.field(IoStats.WRITE_OPERATIONS, writeOperations());
             builder.field(IoStats.READ_KILOBYTES, readKilobytes());
             builder.field(IoStats.WRITE_KILOBYTES, writeKilobytes());
+            builder.field(IoStats.IO_TIMEMS, ioTimeInMillis());
             return builder;
         }
 
@@ -303,13 +330,16 @@ public static class IoStats implements Writeable, ToXContentFragment {
         private static final String WRITE_OPERATIONS = "write_operations";
         private static final String READ_KILOBYTES = "read_kilobytes";
         private static final String WRITE_KILOBYTES = "write_kilobytes";
+        private static final String IO_TIMEMS = "io_time_in_millis";
+
 
         final DeviceStats[] devicesStats;
         final long totalOperations;
         final long totalReadOperations;
         final long totalWriteOperations;
         final long totalReadKilobytes;
         final long totalWriteKilobytes;
+        final long totalIOTimeInMillis;
 
         public IoStats(final DeviceStats[] devicesStats) {
             this.devicesStats = devicesStats;
@@ -318,18 +348,21 @@ public IoStats(final DeviceStats[] devicesStats) {
             long totalWriteOperations = 0;
             long totalReadKilobytes = 0;
             long totalWriteKilobytes = 0;
+            long totalIOTimeInMillis = 0;
             for (DeviceStats deviceStats : devicesStats) {
                 totalOperations += deviceStats.operations() != -1 ? deviceStats.operations() : 0;
                 totalReadOperations += deviceStats.readOperations() != -1 ? deviceStats.readOperations() : 0;
                 totalWriteOperations += deviceStats.writeOperations() != -1 ? deviceStats.writeOperations() : 0;
                 totalReadKilobytes += deviceStats.readKilobytes() != -1 ? deviceStats.readKilobytes() : 0;
                 totalWriteKilobytes += deviceStats.writeKilobytes() != -1 ? deviceStats.writeKilobytes() : 0;
+                totalIOTimeInMillis += deviceStats.ioTimeInMillis() != -1 ? deviceStats.ioTimeInMillis() : 0;
             }
             this.totalOperations = totalOperations;
             this.totalReadOperations = totalReadOperations;
             this.totalWriteOperations = totalWriteOperations;
             this.totalReadKilobytes = totalReadKilobytes;
             this.totalWriteKilobytes = totalWriteKilobytes;
+            this.totalIOTimeInMillis = totalIOTimeInMillis;
         }
 
         public IoStats(StreamInput in) throws IOException {
@@ -344,6 +377,11 @@ public IoStats(StreamInput in) throws IOException {
             this.totalWriteOperations = in.readLong();
             this.totalReadKilobytes = in.readLong();
             this.totalWriteKilobytes = in.readLong();
+            if (in.getVersion().onOrAfter(Version.V_7_14_0)) {
+                this.totalIOTimeInMillis = in.readLong();
+            } else {
+                this.totalIOTimeInMillis = -1;
+            }
         }
 
         @Override
@@ -357,6 +395,9 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeLong(totalWriteOperations);
             out.writeLong(totalReadKilobytes);
             out.writeLong(totalWriteKilobytes);
+            if (out.getVersion().onOrAfter(Version.V_7_14_0)) {
+                out.writeLong(totalIOTimeInMillis);
+            }
         }
 
         public DeviceStats[] getDevicesStats() {
@@ -383,6 +424,10 @@ public long getTotalWriteKilobytes() {
             return totalWriteKilobytes;
         }
 
+        public long getTotalIOTimeMillis() {
+            return totalIOTimeInMillis;
+        }
+
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             if (devicesStats.length > 0) {
@@ -400,6 +445,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
                 builder.field(WRITE_OPERATIONS, totalWriteOperations);
                 builder.field(READ_KILOBYTES, totalReadKilobytes);
                 builder.field(WRITE_KILOBYTES, totalWriteKilobytes);
+                builder.field(IO_TIMEMS, totalIOTimeInMillis);
                 builder.endObject();
             }
             return builder;

server/src/main/java/org/elasticsearch/monitor/fs/FsProbe.java

@@ -86,6 +86,7 @@ final FsInfo.IoStats ioStats(final Set<Tuple<Integer, Integer>> devicesNumbers,
                     final long sectorsRead = Long.parseLong(fields[5]);
                     final long writesCompleted = Long.parseLong(fields[7]);
                     final long sectorsWritten = Long.parseLong(fields[9]);
+                    final long ioTime = Long.parseLong(fields[12]);
                     final FsInfo.DeviceStats deviceStats =
                             new FsInfo.DeviceStats(
                                     majorDeviceNumber,
@@ -95,6 +96,7 @@ final FsInfo.IoStats ioStats(final Set<Tuple<Integer, Integer>> devicesNumbers,
                                     sectorsRead,
                                     writesCompleted,
                                     sectorsWritten,
+                                    ioTime,
                                     deviceMap.get(Tuple.tuple(majorDeviceNumber, minorDeviceNumber)));
                     devicesStats.add(deviceStats);
                 }

server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java

@@ -189,6 +189,7 @@ public void testSerialization() throws IOException {
                     assertEquals(ioStats.getTotalReadOperations(), deserializedIoStats.getTotalReadOperations());
                     assertEquals(ioStats.getTotalWriteKilobytes(), deserializedIoStats.getTotalWriteKilobytes());
                     assertEquals(ioStats.getTotalWriteOperations(), deserializedIoStats.getTotalWriteOperations());
+                    assertEquals(ioStats.getTotalIOTimeMillis(), deserializedIoStats.getTotalIOTimeMillis());
                     assertEquals(ioStats.getDevicesStats().length, deserializedIoStats.getDevicesStats().length);
                     for (int i = 0; i < ioStats.getDevicesStats().length; i++) {
                         FsInfo.DeviceStats deviceStats = ioStats.getDevicesStats()[i];
@@ -198,6 +199,7 @@ public void testSerialization() throws IOException {
                         assertEquals(deviceStats.readOperations(), deserializedDeviceStats.readOperations());
                         assertEquals(deviceStats.writeKilobytes(), deserializedDeviceStats.writeKilobytes());
                         assertEquals(deviceStats.writeOperations(), deserializedDeviceStats.writeOperations());
+                        assertEquals(deviceStats.ioTimeInMillis(), deserializedDeviceStats.ioTimeInMillis());
                     }
                 }
                 if (nodeStats.getTransport() == null) {
@@ -416,10 +418,12 @@ public static NodeStats createNodeStats() {
             for (int i = 0; i < numDeviceStats; i++) {
                 FsInfo.DeviceStats previousDeviceStats = randomBoolean() ? null :
                         new FsInfo.DeviceStats(randomInt(), randomInt(), randomAlphaOfLengthBetween(3, 10),
-                                randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), null);
+                            randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(),
+                            randomNonNegativeLong(), null);
                 deviceStatsArray[i] =
                     new FsInfo.DeviceStats(randomInt(), randomInt(), randomAlphaOfLengthBetween(3, 10), randomNonNegativeLong(),
-                        randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), previousDeviceStats);
+                        randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(),
+                        previousDeviceStats);
             }
             FsInfo.IoStats ioStats = new FsInfo.IoStats(deviceStatsArray);
             int numPaths = randomIntBetween(0, 10);

server/src/test/java/org/elasticsearch/monitor/fs/DeviceStatsTests.java

@@ -22,6 +22,7 @@ public void testDeviceStats() {
         final int sectorsRead = randomIntBetween(8 * readsCompleted, 16 * readsCompleted);
         final int writesCompleted = randomIntBetween(1, 1 << 16);
         final int sectorsWritten = randomIntBetween(8 * writesCompleted, 16 * writesCompleted);
+        final int ioTime = randomIntBetween(1, 1 << 16);
 
         FsInfo.DeviceStats previous = new FsInfo.DeviceStats(
             majorDeviceNumber,
@@ -31,6 +32,7 @@ public void testDeviceStats() {
             sectorsRead,
             writesCompleted,
             sectorsWritten,
+            ioTime,
             null);
         FsInfo.DeviceStats current = new FsInfo.DeviceStats(
             majorDeviceNumber,
@@ -40,12 +42,14 @@ public void testDeviceStats() {
             sectorsRead + 16384,
             writesCompleted + 2048,
             sectorsWritten + 32768,
+            ioTime + 128,
             previous);
         assertThat(current.operations(), equalTo(1024L + 2048L));
         assertThat(current.readOperations(), equalTo(1024L));
         assertThat(current.writeOperations(), equalTo(2048L));
         assertThat(current.readKilobytes(), equalTo(16384L / 2));
         assertThat(current.writeKilobytes(), equalTo(32768L / 2));
+        assertThat(current.ioTimeInMillis(), equalTo(128L));
     }
 
 }

server/src/test/java/org/elasticsearch/monitor/fs/FsProbeTests.java

@@ -60,6 +60,8 @@ public void testFsInfo() throws IOException {
                     assertThat(deviceStats.previousWritesCompleted, equalTo(-1L));
                     assertThat(deviceStats.currentSectorsWritten, greaterThanOrEqualTo(0L));
                     assertThat(deviceStats.previousSectorsWritten, equalTo(-1L));
+                    assertThat(deviceStats.currentIOTime, greaterThanOrEqualTo(0L));
+                    assertThat(deviceStats.previousIOTime, equalTo(-1L));
                 }
             } else {
                 assertNull(stats.getIoStats());
@@ -178,6 +180,8 @@ List<String> readProcDiskStats() throws IOException {
         assertThat(first.devicesStats[0].previousWritesCompleted, equalTo(-1L));
         assertThat(first.devicesStats[0].currentSectorsWritten, equalTo(118857776L));
         assertThat(first.devicesStats[0].previousSectorsWritten, equalTo(-1L));
+        assertThat(first.devicesStats[0].currentIOTime, equalTo(1918440L));
+        assertThat(first.devicesStats[0].previousIOTime, equalTo(-1L));
         assertThat(first.devicesStats[1].majorDeviceNumber, equalTo(253));
         assertThat(first.devicesStats[1].minorDeviceNumber, equalTo(2));
         assertThat(first.devicesStats[1].deviceName, equalTo("dm-2"));
@@ -189,6 +193,8 @@ List<String> readProcDiskStats() throws IOException {
         assertThat(first.devicesStats[1].previousWritesCompleted, equalTo(-1L));
         assertThat(first.devicesStats[1].currentSectorsWritten, equalTo(64126096L));
         assertThat(first.devicesStats[1].previousSectorsWritten, equalTo(-1L));
+        assertThat(first.devicesStats[1].currentIOTime, equalTo(1058193L));
+        assertThat(first.devicesStats[1].previousIOTime, equalTo(-1L));
 
         diskStats.set(Arrays.asList(
                 " 259       0 nvme0n1 336870 0 7928397 82876 10264393 0 182986405 52451610 0 2971042 52536492",
@@ -213,6 +219,8 @@ List<String> readProcDiskStats() throws IOException {
         assertThat(second.devicesStats[0].previousWritesCompleted, equalTo(8398869L));
         assertThat(second.devicesStats[0].currentSectorsWritten, equalTo(118857776L));
         assertThat(second.devicesStats[0].previousSectorsWritten, equalTo(118857776L));
+        assertThat(second.devicesStats[0].currentIOTime, equalTo(1918444L));
+        assertThat(second.devicesStats[0].previousIOTime, equalTo(1918440L));
         assertThat(second.devicesStats[1].majorDeviceNumber, equalTo(253));
         assertThat(second.devicesStats[1].minorDeviceNumber, equalTo(2));
         assertThat(second.devicesStats[1].deviceName, equalTo("dm-2"));
@@ -224,12 +232,15 @@ List<String> readProcDiskStats() throws IOException {
         assertThat(second.devicesStats[1].previousWritesCompleted, equalTo(1371977L));
         assertThat(second.devicesStats[1].currentSectorsWritten, equalTo(64128568L));
         assertThat(second.devicesStats[1].previousSectorsWritten, equalTo(64126096L));
+        assertThat(second.devicesStats[1].currentIOTime, equalTo(1058347L));
+        assertThat(second.devicesStats[1].previousIOTime, equalTo(1058193L));
 
         assertThat(second.totalOperations, equalTo(575L));
         assertThat(second.totalReadOperations, equalTo(261L));
         assertThat(second.totalWriteOperations, equalTo(314L));
         assertThat(second.totalReadKilobytes, equalTo(2392L));
         assertThat(second.totalWriteKilobytes, equalTo(1236L));
+        assertThat(second.totalIOTimeInMillis, equalTo(158L));
     }
 
     public void testAdjustForHugeFilesystems() throws Exception {