From 5ca73c686e1c96382209a401209da8aff38d9f98 Mon Sep 17 00:00:00 2001 From: Dan Hermann Date: Thu, 4 Feb 2021 12:22:50 -0600 Subject: [PATCH 1/2] workaround for JDK bug with total mem on Debian8 --- .../org/elasticsearch/monitor/os/OsProbe.java | 57 ++++++++++++++++- .../monitor/os/OsProbeTests.java | 62 ++++++++++++++++++- 2 files changed, 117 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/monitor/os/OsProbe.java b/server/src/main/java/org/elasticsearch/monitor/os/OsProbe.java index e2ae49ebf427a..a557baeb0d488 100644 --- a/server/src/main/java/org/elasticsearch/monitor/os/OsProbe.java +++ b/server/src/main/java/org/elasticsearch/monitor/os/OsProbe.java @@ -100,11 +100,16 @@ public long getTotalPhysicalMemorySize() { return 0; } try { - final long totalMem = (long) getTotalPhysicalMemorySize.invoke(osMxBean); + long totalMem = (long) getTotalPhysicalMemorySize.invoke(osMxBean); if (totalMem < 0) { logger.debug("OS reported a negative total memory value [{}]", totalMem); return 0; } + if (totalMem == 0 && isDebian8()) { + // workaround for JDK bug on debian8: https://github.com/elastic/elasticsearch/issues/67089#issuecomment-756114654 + totalMem = getTotalMemFromProcMeminfo(); + } + return totalMem; } catch (Exception e) { logger.warn("exception retrieving total physical memory", e); @@ -641,6 +646,56 @@ List readOsRelease() throws IOException { } } + /** + * Returns the lines from /proc/meminfo as a workaround for JDK bugs that prevent retrieval of total system memory + * on some Linux variants such as Debian8. + */ + @SuppressForbidden(reason = "access /proc/meminfo") + List readProcMeminfo() throws IOException { + final List lines; + if (Files.exists(PathUtils.get("/proc/meminfo"))) { + lines = Files.readAllLines(PathUtils.get("/proc/meminfo")); + assert lines != null && lines.isEmpty() == false; + return lines; + } else { + return Collections.emptyList(); + } + } + + /** + * Retrieves system total memory in bytes from /proc/meminfo + */ + long getTotalMemFromProcMeminfo() throws IOException { + List meminfoLines = readProcMeminfo(); + final List memTotalLines = meminfoLines.stream().filter(line -> line.startsWith("MemTotal")).collect(Collectors.toList()); + assert memTotalLines.size() <= 1 : memTotalLines; + final Optional maybeMemTotalLine = memTotalLines.size() == 1 ? Optional.of(memTotalLines.get(0)) : Optional.empty(); + if (maybeMemTotalLine.isPresent()) { + final String memTotalLine = maybeMemTotalLine.get(); + int beginIdx = memTotalLine.indexOf("MemTotal:"); + int endIdx = memTotalLine.lastIndexOf(" kB"); + if (beginIdx + 9 < endIdx) { + final String memTotalString = memTotalLine.substring(beginIdx + 9, endIdx).trim(); + try { + long memTotalInKb = Long.parseLong(memTotalString); + return memTotalInKb * 1024; + } catch (NumberFormatException e) { + logger.warn("Unable to retrieve total memory from meminfo line [" + memTotalLine + "]"); + return 0; + } + } else { + logger.warn("Unable to retrieve total memory from meminfo line [" + memTotalLine + "]"); + return 0; + } + } else { + return 0; + } + } + + boolean isDebian8() throws IOException { + return Constants.LINUX && getPrettyName().equals("Debian GNU/Linux 8 (jessie)"); + } + public OsStats osStats() { final OsStats.Cpu cpu = new OsStats.Cpu(getSystemCpuPercent(), getSystemLoadAverage()); final OsStats.Mem mem = new OsStats.Mem(getTotalPhysicalMemorySize(), getFreePhysicalMemorySize()); diff --git a/server/src/test/java/org/elasticsearch/monitor/os/OsProbeTests.java b/server/src/test/java/org/elasticsearch/monitor/os/OsProbeTests.java index 6ac8e715cb463..66f2c20e67b68 100644 --- a/server/src/test/java/org/elasticsearch/monitor/os/OsProbeTests.java +++ b/server/src/test/java/org/elasticsearch/monitor/os/OsProbeTests.java @@ -254,6 +254,50 @@ public void testCgroupProbeWithMissingMemory() { assertNull(cgroup); } + public void testGetTotalMemFromProcMeminfo() throws Exception { + // missing MemTotal line + var meminfoLines = Arrays.asList( + "MemFree: 8467692 kB", + "MemAvailable: 39646240 kB", + "Buffers: 4699504 kB", + "Cached: 23290380 kB", + "SwapCached: 0 kB", + "Active: 43637908 kB", + "Inactive: 8130280 kB" + ); + OsProbe probe = buildStubOsProbe(true, "", List.of(), meminfoLines); + assertThat(probe.getTotalMemFromProcMeminfo(), equalTo(0L)); + + // MemTotal line with invalid value + meminfoLines = Arrays.asList( + "MemTotal: invalid kB", + "MemFree: 8467692 kB", + "MemAvailable: 39646240 kB", + "Buffers: 4699504 kB", + "Cached: 23290380 kB", + "SwapCached: 0 kB", + "Active: 43637908 kB", + "Inactive: 8130280 kB" + ); + probe = buildStubOsProbe(true, "", List.of(), meminfoLines); + assertThat(probe.getTotalMemFromProcMeminfo(), equalTo(0L)); + + // MemTotal line with random valid value + long memTotalInKb = randomLongBetween(1, Long.MAX_VALUE / 1024L); + meminfoLines = Arrays.asList( + "MemTotal: " + memTotalInKb + " kB", + "MemFree: 8467692 kB", + "MemAvailable: 39646240 kB", + "Buffers: 4699504 kB", + "Cached: 23290380 kB", + "SwapCached: 0 kB", + "Active: 43637908 kB", + "Inactive: 8130280 kB" + ); + probe = buildStubOsProbe(true, "", List.of(), meminfoLines); + assertThat(probe.getTotalMemFromProcMeminfo(), equalTo(memTotalInKb * 1024L)); + } + private static List getProcSelfGroupLines(String hierarchy) { return Arrays.asList( "10:freezer:/", @@ -282,12 +326,14 @@ private static OsProbe buildStubOsProbe(final boolean areCgroupStatsAvailable, f * @param areCgroupStatsAvailable whether or not cgroup data is available. Normally OsProbe establishes this for itself. * @param hierarchy a mock value used to generate a cgroup hierarchy. * @param procSelfCgroupLines the lines that will be used as the content of /proc/self/cgroup + * @param procMeminfoLines lines that will be used as the content of /proc/meminfo * @return a test instance */ private static OsProbe buildStubOsProbe( final boolean areCgroupStatsAvailable, final String hierarchy, - List procSelfCgroupLines + List procSelfCgroupLines, + List procMeminfoLines ) { return new OsProbe() { @Override @@ -338,6 +384,20 @@ String readSysFsCgroupMemoryUsageInBytes(String controlGroup) { boolean areCgroupStatsAvailable() { return areCgroupStatsAvailable; } + + @Override + List readProcMeminfo() throws IOException { + return procMeminfoLines; + } }; } + + private static OsProbe buildStubOsProbe( + final boolean areCgroupStatsAvailable, + final String hierarchy, + List procSelfCgroupLines + ) { + return buildStubOsProbe(areCgroupStatsAvailable, hierarchy, procSelfCgroupLines, List.of()); + } + } From 357e7666ec1e65ad5fc50509bcbb6ce04248cbfc Mon Sep 17 00:00:00 2001 From: Dan Hermann Date: Fri, 5 Feb 2021 14:15:27 -0600 Subject: [PATCH 2/2] review comments --- .../org/elasticsearch/monitor/os/OsProbe.java | 5 ++--- .../monitor/os/OsProbeTests.java | 21 +++++++++++++++++++ 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/monitor/os/OsProbe.java b/server/src/main/java/org/elasticsearch/monitor/os/OsProbe.java index a557baeb0d488..40ac26924ce9d 100644 --- a/server/src/main/java/org/elasticsearch/monitor/os/OsProbe.java +++ b/server/src/main/java/org/elasticsearch/monitor/os/OsProbe.java @@ -669,9 +669,8 @@ long getTotalMemFromProcMeminfo() throws IOException { List meminfoLines = readProcMeminfo(); final List memTotalLines = meminfoLines.stream().filter(line -> line.startsWith("MemTotal")).collect(Collectors.toList()); assert memTotalLines.size() <= 1 : memTotalLines; - final Optional maybeMemTotalLine = memTotalLines.size() == 1 ? Optional.of(memTotalLines.get(0)) : Optional.empty(); - if (maybeMemTotalLine.isPresent()) { - final String memTotalLine = maybeMemTotalLine.get(); + if (memTotalLines.size() == 1) { + final String memTotalLine = memTotalLines.get(0); int beginIdx = memTotalLine.indexOf("MemTotal:"); int endIdx = memTotalLine.lastIndexOf(" kB"); if (beginIdx + 9 < endIdx) { diff --git a/server/src/test/java/org/elasticsearch/monitor/os/OsProbeTests.java b/server/src/test/java/org/elasticsearch/monitor/os/OsProbeTests.java index 66f2c20e67b68..cbac14bf7baea 100644 --- a/server/src/test/java/org/elasticsearch/monitor/os/OsProbeTests.java +++ b/server/src/test/java/org/elasticsearch/monitor/os/OsProbeTests.java @@ -282,6 +282,20 @@ public void testGetTotalMemFromProcMeminfo() throws Exception { probe = buildStubOsProbe(true, "", List.of(), meminfoLines); assertThat(probe.getTotalMemFromProcMeminfo(), equalTo(0L)); + // MemTotal line with invalid unit + meminfoLines = Arrays.asList( + "MemTotal: 39646240 MB", + "MemFree: 8467692 kB", + "MemAvailable: 39646240 kB", + "Buffers: 4699504 kB", + "Cached: 23290380 kB", + "SwapCached: 0 kB", + "Active: 43637908 kB", + "Inactive: 8130280 kB" + ); + probe = buildStubOsProbe(true, "", List.of(), meminfoLines); + assertThat(probe.getTotalMemFromProcMeminfo(), equalTo(0L)); + // MemTotal line with random valid value long memTotalInKb = randomLongBetween(1, Long.MAX_VALUE / 1024L); meminfoLines = Arrays.asList( @@ -298,6 +312,13 @@ public void testGetTotalMemFromProcMeminfo() throws Exception { assertThat(probe.getTotalMemFromProcMeminfo(), equalTo(memTotalInKb * 1024L)); } + public void testGetTotalMemoryOnDebian8() throws Exception { + // tests the workaround for JDK bug on debian8: https://github.com/elastic/elasticsearch/issues/67089#issuecomment-756114654 + final OsProbe osProbe = new OsProbe(); + assumeTrue("runs only on Debian 8", osProbe.isDebian8()); + assertThat(osProbe.getTotalPhysicalMemorySize(), greaterThan(0L)); + } + private static List getProcSelfGroupLines(String hierarchy) { return Arrays.asList( "10:freezer:/",