Skip to content

Add Netty ByteBuf Leak Check to REST Test Clusters (#64528) #65864

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Dec 4, 2020
Merged

Add Netty ByteBuf Leak Check to REST Test Clusters (#64528) #65864

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 42 additions & 25 deletions buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -769,6 +769,9 @@ private Map<String, String> getESEnvironment() {
.map(p -> p.replace("${ES_PATH_CONF}", configFile.getParent().toString()))
.collect(Collectors.joining(" "));
}
if (systemProperties.containsKey("io.netty.leakDetection.level") == false) {
systemPropertiesString = systemPropertiesString + " -Dio.netty.leakDetection.level=paranoid";
}
String jvmArgsString = "";
if (jvmArgs.isEmpty() == false) {
jvmArgsString = " " + jvmArgs.stream().peek(argument -> {
Expand Down Expand Up @@ -921,10 +924,6 @@ public synchronized void stop(boolean tailLogs) {
// Test clusters are not reused, don't spend time on a graceful shutdown
stopHandle(esProcess.toHandle(), true);
reaper.unregister(toString());
if (tailLogs) {
logFileContents("Standard output of node", esStdoutFile);
logFileContents("Standard error of node", esStderrFile);
}
esProcess = null;
// Clean up the ports file in case this is started again.
try {
Expand All @@ -937,6 +936,8 @@ public synchronized void stop(boolean tailLogs) {
} catch (IOException e) {
throw new UncheckedIOException(e);
}
logFileContents("Standard output of node", esStdoutFile, tailLogs);
logFileContents("Standard error of node", esStderrFile, tailLogs);
}

@Override
Expand Down Expand Up @@ -989,7 +990,7 @@ private void logProcessInfo(String prefix, ProcessHandle.Info info) {
);
}

private void logFileContents(String description, Path from) {
private void logFileContents(String description, Path from, boolean tailLogs) {
final Map<String, Integer> errorsAndWarnings = new LinkedHashMap<>();
LinkedList<String> ring = new LinkedList<>();
try (LineNumberReader reader = new LineNumberReader(Files.newBufferedReader(from))) {
Expand Down Expand Up @@ -1018,31 +1019,47 @@ private void logFileContents(String description, Path from) {
}
}
} catch (IOException e) {
throw new UncheckedIOException("Failed to tail log " + this, e);
if (tailLogs) {
throw new UncheckedIOException("Failed to tail log " + this, e);
}
return;
}

if (errorsAndWarnings.isEmpty() == false || ring.isEmpty() == false) {
LOGGER.error("\n=== {} `{}` ===", description, this);
}
if (errorsAndWarnings.isEmpty() == false) {
LOGGER.lifecycle("\n» ↓ errors and warnings from " + from + " ↓");
errorsAndWarnings.forEach((message, count) -> {
LOGGER.lifecycle("» " + message.replace("\n", "\n» "));
if (count > 1) {
LOGGER.lifecycle("» ↑ repeated " + count + " times ↑");
}
});
boolean foundNettyLeaks = false;
for (String logLine : errorsAndWarnings.keySet()) {
if (logLine.contains("ResourceLeakDetector]")) {
tailLogs = true;
foundNettyLeaks = true;
break;
}
}
if (tailLogs) {
if (errorsAndWarnings.isEmpty() == false || ring.isEmpty() == false) {
LOGGER.error("\n=== {} `{}` ===", description, this);
}
if (errorsAndWarnings.isEmpty() == false) {
LOGGER.lifecycle("\n» ↓ errors and warnings from " + from + " ↓");
errorsAndWarnings.forEach((message, count) -> {
LOGGER.lifecycle("» " + message.replace("\n", "\n» "));
if (count > 1) {
LOGGER.lifecycle("» ↑ repeated " + count + " times ↑");
}
});
}

ring.removeIf(line -> MESSAGES_WE_DONT_CARE_ABOUT.stream().anyMatch(line::contains));
ring.removeIf(line -> MESSAGES_WE_DONT_CARE_ABOUT.stream().anyMatch(line::contains));

if (ring.isEmpty() == false) {
LOGGER.lifecycle("» ↓ last " + TAIL_LOG_MESSAGES_COUNT + " non error or warning messages from " + from + " ↓");
ring.forEach(message -> {
if (errorsAndWarnings.containsKey(normalizeLogLine(message)) == false) {
LOGGER.lifecycle("» " + message.replace("\n", "\n» "));
}
});
if (ring.isEmpty() == false) {
LOGGER.lifecycle("» ↓ last " + TAIL_LOG_MESSAGES_COUNT + " non error or warning messages from " + from + " ↓");
ring.forEach(message -> {
if (errorsAndWarnings.containsKey(normalizeLogLine(message)) == false) {
LOGGER.lifecycle("» " + message.replace("\n", "\n» "));
}
});
}
}
if (foundNettyLeaks) {
throw new TestClustersException("Found Netty ByteBuf leaks in node logs.");
}
}

Expand Down