Skip to content

Make Crashtracking smoke test more resilient #8685

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 9 commits into from
Apr 18, 2025
Merged

Make Crashtracking smoke test more resilient #8685

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
561f192
Make crash tracking smoke test more resiliant
randomanderson Apr 9, 2025
551f88e
loop through messages until the correct event
randomanderson Apr 9, 2025
3df5bbf
some debug
randomanderson Apr 9, 2025
e4b8c9e
configurable dogstatsd port
randomanderson Apr 10, 2025
b5a1003
Merge branch 'landerson/dd-dogstatsd-port' into landerson/fix-crashtr…
randomanderson Apr 10, 2025
d6250d5
use sleep instead of park nanos
randomanderson Apr 10, 2025
2e0d7ba
spotless
randomanderson Apr 10, 2025
ac863f0
cleanup
randomanderson Apr 11, 2025
0383ef9
Merge remote-tracking branch 'origin/master' into landerson/fix-crash…
randomanderson Apr 14, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions dd-smoke-tests/crashtracking/src/test/java/datadog/smoketest/CrashTelemetryData.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
package datadog.smoketest;

import java.util.List;

public class CrashTelemetryData extends MinimalTelemetryData {
List<LogMessage> payload;

public static class LogMessage {
public String message;
public String level;
public String tags;
}
}
261 changes: 85 additions & 176 deletions dd-smoke-tests/crashtracking/src/test/java/datadog/smoketest/CrashtrackingSmokeTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,22 +3,30 @@
import static org.hamcrest.CoreMatchers.containsString;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assumptions.assumeFalse;

import com.squareup.moshi.JsonAdapter;
import com.squareup.moshi.Moshi;
import datadog.trace.api.Platform;
import java.io.BufferedReader;
import java.io.File;
import java.io.InputStreamReader;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.Comparator;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.stream.Stream;
import okhttp3.mockwebserver.Dispatcher;
import okhttp3.mockwebserver.MockResponse;
import okhttp3.mockwebserver.MockWebServer;
import okhttp3.mockwebserver.RecordedRequest;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
Expand All @@ -29,7 +37,14 @@
* that ships with OS X by default.
*/
public class CrashtrackingSmokeTest {
private static final long DATA_TIMEOUT_MS = 10 * 1000;
private static final OutputThreads OUTPUT = new OutputThreads();
private static final Path LOG_FILE_DIR =
Paths.get(System.getProperty("datadog.smoketest.builddir"), "reports");

private MockWebServer tracingServer;
private TestUDPServer udpServer;
private final BlockingQueue<CrashTelemetryData> crashEvents = new LinkedBlockingQueue<>();

@BeforeAll
static void setupAll() {
Expand All @@ -43,27 +58,61 @@ static void setupAll() {
void setup() throws Exception {
tempDir = Files.createTempDirectory("dd-smoketest-");

crashEvents.clear();

Moshi moshi = new Moshi.Builder().build();
tracingServer = new MockWebServer();
tracingServer.setDispatcher(
new Dispatcher() {
@Override
public MockResponse dispatch(final RecordedRequest request) throws InterruptedException {
public MockResponse dispatch(final RecordedRequest request) {
System.out.println("URL ====== " + request.getPath());

String data = request.getBody().readString(StandardCharsets.UTF_8);

if ("/telemetry/proxy/api/v2/apmtelemetry".equals(request.getPath())) {
try {
JsonAdapter<MinimalTelemetryData> adapter =
moshi.adapter(MinimalTelemetryData.class);
MinimalTelemetryData minimal = adapter.fromJson(data);
if ("logs".equals(minimal.request_type)) {
JsonAdapter<CrashTelemetryData> crashAdapter =
moshi.adapter(CrashTelemetryData.class);
crashEvents.add(crashAdapter.fromJson(data));
}
} catch (IOException e) {
System.out.println("Unable to parse " + e);
}
}

System.out.println(data);

return new MockResponse().setResponseCode(200);
}
});
// tracingServer.start(8126);

udpServer = new TestUDPServer();
udpServer.start();

OUTPUT.clearMessages();
}

@AfterEach
void teardown() throws Exception {
tracingServer.shutdown();
udpServer.close();

try (Stream<Path> fileStream = Files.walk(tempDir)) {
fileStream.sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete);
}
Files.deleteIfExists(tempDir);
}

@AfterAll
static void shutdown() {
OUTPUT.close();
}

private static String javaPath() {
final String separator = FileSystems.getDefault().getSeparator();
return System.getProperty("java.home") + separator + "bin" + separator + "java";
Expand Down Expand Up @@ -108,52 +157,12 @@ void testCrashTracking() throws Exception {
appShadowJar(),
script.toString()));
pb.environment().put("DD_TRACE_AGENT_PORT", String.valueOf(tracingServer.getPort()));
StringBuilder stdoutStr = new StringBuilder();
StringBuilder stderrStr = new StringBuilder();

Process p = pb.start();
Thread stdout =
new Thread(
() -> {
try (BufferedReader br =
new BufferedReader(new InputStreamReader(p.getInputStream()))) {
br.lines()
.forEach(
l -> {
System.out.println(l);
stdoutStr.append(l).append('\n');
});
} catch (Exception e) {
throw new RuntimeException(e);
}
});
Thread stderr =
new Thread(
() -> {
try (BufferedReader br =
new BufferedReader(new InputStreamReader(p.getErrorStream()))) {
br.lines()
.forEach(
l -> {
System.err.println(l);
stderrStr.append(l).append('\n');
});
} catch (Exception e) {
throw new RuntimeException(e);
}
});
stdout.setDaemon(true);
stderr.setDaemon(true);
stdout.start();
stderr.start();
OUTPUT.captureOutput(p, LOG_FILE_DIR.resolve("testProcess.testCrashTracking.log").toFile());

assertNotEquals(0, p.waitFor(), "Application should have crashed");

assertThat(stdoutStr.toString(), containsString(" was uploaded successfully"));
assertThat(
stderrStr.toString(),
containsString(
"com.datadog.crashtracking.CrashUploader - Successfully uploaded the crash files"));
assertCrashData();
}

/*
Expand Down Expand Up @@ -183,52 +192,14 @@ void testCrashTrackingLegacy() throws Exception {
appShadowJar(),
script.toString()));
pb.environment().put("DD_TRACE_AGENT_PORT", String.valueOf(tracingServer.getPort()));
StringBuilder stdoutStr = new StringBuilder();
StringBuilder stderrStr = new StringBuilder();

Process p = pb.start();
Thread stdout =
new Thread(
() -> {
try (BufferedReader br =
new BufferedReader(new InputStreamReader(p.getInputStream()))) {
br.lines()
.forEach(
l -> {
System.out.println(l);
stdoutStr.append(l).append('\n');
});
} catch (Exception e) {
throw new RuntimeException(e);
}
});
Thread stderr =
new Thread(
() -> {
try (BufferedReader br =
new BufferedReader(new InputStreamReader(p.getErrorStream()))) {
br.lines()
.forEach(
l -> {
System.err.println(l);
stderrStr.append(l).append('\n');
});
} catch (Exception e) {
throw new RuntimeException(e);
}
});
stdout.setDaemon(true);
stderr.setDaemon(true);
stdout.start();
stderr.start();
OUTPUT.captureOutput(
p, LOG_FILE_DIR.resolve("testProcess.testCrashTrackingLegacy.log").toFile());

assertNotEquals(0, p.waitFor(), "Application should have crashed");

assertThat(stdoutStr.toString(), containsString(" was uploaded successfully"));
assertThat(
stderrStr.toString(),
containsString(
"com.datadog.crashtracking.CrashUploader - Successfully uploaded the crash files"));
assertCrashData();
}

/*
Expand All @@ -255,51 +226,13 @@ void testOomeTracking() throws Exception {
"-jar",
appShadowJar(),
script.toString()));
StringBuilder stdoutStr = new StringBuilder();
StringBuilder stderrStr = new StringBuilder();
pb.environment().put("DD_DOGSTATSD_PORT", String.valueOf(udpServer.getPort()));

Process p = pb.start();
Thread stdout =
new Thread(
() -> {
try (BufferedReader br =
new BufferedReader(new InputStreamReader(p.getInputStream()))) {
br.lines()
.forEach(
l -> {
System.out.println(l);
stdoutStr.append(l).append('\n');
});
} catch (Exception e) {
throw new RuntimeException(e);
}
});
Thread stderr =
new Thread(
() -> {
try (BufferedReader br =
new BufferedReader(new InputStreamReader(p.getErrorStream()))) {
br.lines()
.forEach(
l -> {
System.err.println(l);
stderrStr.append(l).append('\n');
});
} catch (Exception e) {
throw new RuntimeException(e);
}
});
stdout.setDaemon(true);
stderr.setDaemon(true);
stdout.start();
stderr.start();
OUTPUT.captureOutput(p, LOG_FILE_DIR.resolve("testProcess.testOomeTracking.log").toFile());

assertNotEquals(0, p.waitFor(), "Application should have crashed");

assertThat(
stderrStr.toString(),
containsString("com.datadog.crashtracking.OOMENotifier - OOME event sent"));
assertThat(stdoutStr.toString(), containsString("OOME Event generated successfully"));
assertOOMEvent();
}

@Test
Expand All @@ -326,58 +259,34 @@ void testCombineTracking() throws Exception {
appShadowJar(),
oomeScript.toString()));
pb.environment().put("DD_TRACE_AGENT_PORT", String.valueOf(tracingServer.getPort()));
StringBuilder stdoutStr = new StringBuilder();
StringBuilder stderrStr = new StringBuilder();
pb.environment().put("DD_DOGSTATSD_PORT", String.valueOf(udpServer.getPort()));

Process p = pb.start();
Thread stdout =
new Thread(
() -> {
try (BufferedReader br =
new BufferedReader(new InputStreamReader(p.getInputStream()))) {
br.lines()
.forEach(
l -> {
System.out.println(l);
stdoutStr.append(l).append('\n');
});
} catch (Exception e) {
throw new RuntimeException(e);
}
});
Thread stderr =
new Thread(
() -> {
try (BufferedReader br =
new BufferedReader(new InputStreamReader(p.getErrorStream()))) {
br.lines()
.forEach(
l -> {
System.err.println(l);
stderrStr.append(l).append('\n');
});
} catch (Exception e) {
throw new RuntimeException(e);
}
});
stdout.setDaemon(true);
stderr.setDaemon(true);
stdout.start();
stderr.start();
OUTPUT.captureOutput(p, LOG_FILE_DIR.resolve("testProcess.testCombineTracking.log").toFile());

assertNotEquals(0, p.waitFor(), "Application should have crashed");

// Crash uploader did get triggered
assertThat(stdoutStr.toString(), containsString(" was uploaded successfully"));
assertThat(
stderrStr.toString(),
containsString(
"com.datadog.crashtracking.CrashUploader - Successfully uploaded the crash files"));

// OOME notifier did get triggered
assertThat(
stderrStr.toString(),
containsString("com.datadog.crashtracking.OOMENotifier - OOME event sent"));
assertThat(stdoutStr.toString(), containsString("OOME Event generated successfully"));
assertCrashData();
assertOOMEvent();
}

private void assertCrashData() throws InterruptedException {
CrashTelemetryData crashData = crashEvents.poll(DATA_TIMEOUT_MS, TimeUnit.MILLISECONDS);
assertNotNull(crashData, "Crash data not uploaded");
assertThat(crashData.payload.get(0).message, containsString("OutOfMemory"));
assertThat(crashData.payload.get(0).tags, containsString("severity:crash"));
}

private void assertOOMEvent() throws InterruptedException {
String event;
do {
event = udpServer.getMessages().poll(DATA_TIMEOUT_MS, TimeUnit.MILLISECONDS);
} while (event != null && !event.startsWith("_e"));

assertNotNull(event, "OOM Event not received");

assertThat(event, containsString(":OutOfMemoryError"));
assertThat(event, containsString("t:error"));
assertThat(event, containsString("s:java"));
}
}
5 changes: 5 additions & 0 deletions dd-smoke-tests/crashtracking/src/test/java/datadog/smoketest/MinimalTelemetryData.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
package datadog.smoketest;

public class MinimalTelemetryData {
String request_type;
}
Loading