Skip to content

Commit 561f192

Browse files
randomandersonrandomanderson
committed
Make crash tracking smoke test more resiliant
1 parent 03b9371 commit 561f192

File tree

6 files changed

+408
-302
lines changed

6 files changed

+408
-302
lines changed

‎dd-smoke-tests/crashtracking/src/test/java/datadog/smoketest/CrashTelemetryData.java

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
package datadog.smoketest;
2+
3+
import java.util.List;
4+
5+
public class CrashTelemetryData extends MinimalTelemetryData {
6+
List<LogMessage> payload;
7+
8+
public static class LogMessage {
9+
public String message;
10+
public String level;
11+
public String tags;
12+
}
13+
}

‎dd-smoke-tests/crashtracking/src/test/java/datadog/smoketest/CrashtrackingSmokeTest.java

+86-175
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,33 @@
11
package datadog.smoketest;
22

33
import static org.hamcrest.CoreMatchers.containsString;
4+
import static org.hamcrest.CoreMatchers.startsWith;
45
import static org.hamcrest.MatcherAssert.assertThat;
56
import static org.junit.jupiter.api.Assertions.assertNotEquals;
7+
import static org.junit.jupiter.api.Assertions.assertNotNull;
68
import static org.junit.jupiter.api.Assumptions.assumeFalse;
79

10+
import com.squareup.moshi.JsonAdapter;
11+
import com.squareup.moshi.Moshi;
812
import datadog.trace.api.Platform;
9-
import java.io.BufferedReader;
1013
import java.io.File;
11-
import java.io.InputStreamReader;
14+
import java.io.IOException;
15+
import java.nio.charset.StandardCharsets;
1216
import java.nio.file.FileSystems;
1317
import java.nio.file.Files;
1418
import java.nio.file.Path;
19+
import java.nio.file.Paths;
1520
import java.util.Arrays;
1621
import java.util.Comparator;
22+
import java.util.concurrent.BlockingQueue;
23+
import java.util.concurrent.LinkedBlockingQueue;
24+
import java.util.concurrent.TimeUnit;
1725
import java.util.stream.Stream;
1826
import okhttp3.mockwebserver.Dispatcher;
1927
import okhttp3.mockwebserver.MockResponse;
2028
import okhttp3.mockwebserver.MockWebServer;
2129
import okhttp3.mockwebserver.RecordedRequest;
30+
import org.junit.jupiter.api.AfterAll;
2231
import org.junit.jupiter.api.AfterEach;
2332
import org.junit.jupiter.api.BeforeAll;
2433
import org.junit.jupiter.api.BeforeEach;
@@ -29,41 +38,82 @@
2938
* that ships with OS X by default.
3039
*/
3140
public class CrashtrackingSmokeTest {
41+
private static final long DATA_TIMEOUT_MS = 10 * 1000;
42+
private static Path LOG_FILE_DIR;
3243
private MockWebServer tracingServer;
44+
private TestUDPServer udpServer;
45+
private BlockingQueue<CrashTelemetryData> crashEvents = new LinkedBlockingQueue<>();
3346

3447
@BeforeAll
3548
static void setupAll() {
3649
// Only Hotspot based implementation are supported
3750
assumeFalse(Platform.isJ9());
51+
52+
LOG_FILE_DIR = Paths.get(System.getProperty("datadog.smoketest.builddir"), "reports");
3853
}
3954

4055
private Path tempDir;
56+
private static OutputThreads outputThreads = new OutputThreads();
4157

4258
@BeforeEach
4359
void setup() throws Exception {
4460
tempDir = Files.createTempDirectory("dd-smoketest-");
4561

62+
crashEvents.clear();
63+
64+
Moshi moshi = new Moshi.Builder().build();
4665
tracingServer = new MockWebServer();
4766
tracingServer.setDispatcher(
4867
new Dispatcher() {
4968
@Override
5069
public MockResponse dispatch(final RecordedRequest request) throws InterruptedException {
70+
String data = request.getBody().readString(StandardCharsets.UTF_8);
71+
72+
if ("/telemetry/proxy/api/v2/apmtelemetry".equals(request.getPath())) {
73+
try {
74+
JsonAdapter<MinimalTelemetryData> adapter =
75+
moshi.adapter(MinimalTelemetryData.class);
76+
MinimalTelemetryData minimal = adapter.fromJson(data);
77+
if ("logs".equals(minimal.request_type)) {
78+
JsonAdapter<CrashTelemetryData> crashAdapter =
79+
moshi.adapter(CrashTelemetryData.class);
80+
crashEvents.add(crashAdapter.fromJson(data));
81+
}
82+
} catch (IOException e) {
83+
System.out.println("Unable to parse " + e);
84+
}
85+
}
86+
System.out.println("URL ====== " + request.getPath());
87+
System.out.println(data);
88+
5189
return new MockResponse().setResponseCode(200);
5290
}
5391
});
54-
// tracingServer.start(8126);
92+
93+
udpServer = new TestUDPServer();
94+
udpServer.start();
95+
96+
synchronized (outputThreads.testLogMessages) {
97+
outputThreads.testLogMessages.clear();
98+
}
5599
}
56100

57101
@AfterEach
58102
void teardown() throws Exception {
59103
tracingServer.shutdown();
104+
udpServer.close();
60105

61106
try (Stream<Path> fileStream = Files.walk(tempDir)) {
62107
fileStream.sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete);
63108
}
64109
Files.deleteIfExists(tempDir);
65110
}
66111

112+
@AfterAll
113+
static void shutdown() {
114+
outputThreads.close();
115+
}
116+
67117
private static String javaPath() {
68118
final String separator = FileSystems.getDefault().getSeparator();
69119
return System.getProperty("java.home") + separator + "bin" + separator + "java";
@@ -108,52 +158,13 @@ void testCrashTracking() throws Exception {
108158
appShadowJar(),
109159
script.toString()));
110160
pb.environment().put("DD_TRACE_AGENT_PORT", String.valueOf(tracingServer.getPort()));
111-
StringBuilder stdoutStr = new StringBuilder();
112-
StringBuilder stderrStr = new StringBuilder();
113161

114162
Process p = pb.start();
115-
Thread stdout =
116-
new Thread(
117-
() -> {
118-
try (BufferedReader br =
119-
new BufferedReader(new InputStreamReader(p.getInputStream()))) {
120-
br.lines()
121-
.forEach(
122-
l -> {
123-
System.out.println(l);
124-
stdoutStr.append(l).append('\n');
125-
});
126-
} catch (Exception e) {
127-
throw new RuntimeException(e);
128-
}
129-
});
130-
Thread stderr =
131-
new Thread(
132-
() -> {
133-
try (BufferedReader br =
134-
new BufferedReader(new InputStreamReader(p.getErrorStream()))) {
135-
br.lines()
136-
.forEach(
137-
l -> {
138-
System.err.println(l);
139-
stderrStr.append(l).append('\n');
140-
});
141-
} catch (Exception e) {
142-
throw new RuntimeException(e);
143-
}
144-
});
145-
stdout.setDaemon(true);
146-
stderr.setDaemon(true);
147-
stdout.start();
148-
stderr.start();
163+
outputThreads.captureOutput(
164+
p, LOG_FILE_DIR.resolve("testProcess.testCrashTracking.log").toFile());
149165

150166
assertNotEquals(0, p.waitFor(), "Application should have crashed");
151-
152-
assertThat(stdoutStr.toString(), containsString(" was uploaded successfully"));
153-
assertThat(
154-
stderrStr.toString(),
155-
containsString(
156-
"com.datadog.crashtracking.CrashUploader - Successfully uploaded the crash files"));
167+
assertCrashData();
157168
}
158169

159170
/*
@@ -183,52 +194,14 @@ void testCrashTrackingLegacy() throws Exception {
183194
appShadowJar(),
184195
script.toString()));
185196
pb.environment().put("DD_TRACE_AGENT_PORT", String.valueOf(tracingServer.getPort()));
186-
StringBuilder stdoutStr = new StringBuilder();
187-
StringBuilder stderrStr = new StringBuilder();
188197

189198
Process p = pb.start();
190-
Thread stdout =
191-
new Thread(
192-
() -> {
193-
try (BufferedReader br =
194-
new BufferedReader(new InputStreamReader(p.getInputStream()))) {
195-
br.lines()
196-
.forEach(
197-
l -> {
198-
System.out.println(l);
199-
stdoutStr.append(l).append('\n');
200-
});
201-
} catch (Exception e) {
202-
throw new RuntimeException(e);
203-
}
204-
});
205-
Thread stderr =
206-
new Thread(
207-
() -> {
208-
try (BufferedReader br =
209-
new BufferedReader(new InputStreamReader(p.getErrorStream()))) {
210-
br.lines()
211-
.forEach(
212-
l -> {
213-
System.err.println(l);
214-
stderrStr.append(l).append('\n');
215-
});
216-
} catch (Exception e) {
217-
throw new RuntimeException(e);
218-
}
219-
});
220-
stdout.setDaemon(true);
221-
stderr.setDaemon(true);
222-
stdout.start();
223-
stderr.start();
199+
outputThreads.captureOutput(
200+
p, LOG_FILE_DIR.resolve("testProcess.testCrashTrackingLegacy.log").toFile());
224201

225202
assertNotEquals(0, p.waitFor(), "Application should have crashed");
226203

227-
assertThat(stdoutStr.toString(), containsString(" was uploaded successfully"));
228-
assertThat(
229-
stderrStr.toString(),
230-
containsString(
231-
"com.datadog.crashtracking.CrashUploader - Successfully uploaded the crash files"));
204+
assertCrashData();
232205
}
233206

234207
/*
@@ -255,51 +228,14 @@ void testOomeTracking() throws Exception {
255228
"-jar",
256229
appShadowJar(),
257230
script.toString()));
258-
StringBuilder stdoutStr = new StringBuilder();
259-
StringBuilder stderrStr = new StringBuilder();
260231

261232
Process p = pb.start();
262-
Thread stdout =
263-
new Thread(
264-
() -> {
265-
try (BufferedReader br =
266-
new BufferedReader(new InputStreamReader(p.getInputStream()))) {
267-
br.lines()
268-
.forEach(
269-
l -> {
270-
System.out.println(l);
271-
stdoutStr.append(l).append('\n');
272-
});
273-
} catch (Exception e) {
274-
throw new RuntimeException(e);
275-
}
276-
});
277-
Thread stderr =
278-
new Thread(
279-
() -> {
280-
try (BufferedReader br =
281-
new BufferedReader(new InputStreamReader(p.getErrorStream()))) {
282-
br.lines()
283-
.forEach(
284-
l -> {
285-
System.err.println(l);
286-
stderrStr.append(l).append('\n');
287-
});
288-
} catch (Exception e) {
289-
throw new RuntimeException(e);
290-
}
291-
});
292-
stdout.setDaemon(true);
293-
stderr.setDaemon(true);
294-
stdout.start();
295-
stderr.start();
233+
outputThreads.captureOutput(
234+
p, LOG_FILE_DIR.resolve("testProcess.testOomeTracking.log").toFile());
235+
pb.environment().put("DD_DOGSTATSD_PORT", String.valueOf(udpServer.getPort()));
296236

297237
assertNotEquals(0, p.waitFor(), "Application should have crashed");
298-
299-
assertThat(
300-
stderrStr.toString(),
301-
containsString("com.datadog.crashtracking.OOMENotifier - OOME event sent"));
302-
assertThat(stdoutStr.toString(), containsString("OOME Event generated successfully"));
238+
assertOOMEvent();
303239
}
304240

305241
@Test
@@ -326,58 +262,33 @@ void testCombineTracking() throws Exception {
326262
appShadowJar(),
327263
oomeScript.toString()));
328264
pb.environment().put("DD_TRACE_AGENT_PORT", String.valueOf(tracingServer.getPort()));
329-
StringBuilder stdoutStr = new StringBuilder();
330-
StringBuilder stderrStr = new StringBuilder();
265+
pb.environment().put("DD_DOGSTATSD_PORT", String.valueOf(udpServer.getPort()));
331266

332267
Process p = pb.start();
333-
Thread stdout =
334-
new Thread(
335-
() -> {
336-
try (BufferedReader br =
337-
new BufferedReader(new InputStreamReader(p.getInputStream()))) {
338-
br.lines()
339-
.forEach(
340-
l -> {
341-
System.out.println(l);
342-
stdoutStr.append(l).append('\n');
343-
});
344-
} catch (Exception e) {
345-
throw new RuntimeException(e);
346-
}
347-
});
348-
Thread stderr =
349-
new Thread(
350-
() -> {
351-
try (BufferedReader br =
352-
new BufferedReader(new InputStreamReader(p.getErrorStream()))) {
353-
br.lines()
354-
.forEach(
355-
l -> {
356-
System.err.println(l);
357-
stderrStr.append(l).append('\n');
358-
});
359-
} catch (Exception e) {
360-
throw new RuntimeException(e);
361-
}
362-
});
363-
stdout.setDaemon(true);
364-
stderr.setDaemon(true);
365-
stdout.start();
366-
stderr.start();
268+
outputThreads.captureOutput(
269+
p, LOG_FILE_DIR.resolve("testProcess.testCombineTracking.log").toFile());
367270

368271
assertNotEquals(0, p.waitFor(), "Application should have crashed");
369272

370-
// Crash uploader did get triggered
371-
assertThat(stdoutStr.toString(), containsString(" was uploaded successfully"));
372-
assertThat(
373-
stderrStr.toString(),
374-
containsString(
375-
"com.datadog.crashtracking.CrashUploader - Successfully uploaded the crash files"));
376-
377-
// OOME notifier did get triggered
378-
assertThat(
379-
stderrStr.toString(),
380-
containsString("com.datadog.crashtracking.OOMENotifier - OOME event sent"));
381-
assertThat(stdoutStr.toString(), containsString("OOME Event generated successfully"));
273+
assertCrashData();
274+
assertOOMEvent();
275+
}
276+
277+
private void assertCrashData() throws InterruptedException {
278+
CrashTelemetryData crashData = crashEvents.poll(DATA_TIMEOUT_MS, TimeUnit.MILLISECONDS);
279+
assertNotNull(crashData, "Crash data not uploaded");
280+
assertThat(crashData.payload.get(0).message, containsString("OutOfMemory"));
281+
assertThat(crashData.payload.get(0).tags, containsString("severity:crash"));
282+
}
283+
284+
private void assertOOMEvent() throws InterruptedException {
285+
byte[] data = udpServer.getMessages().poll(DATA_TIMEOUT_MS, TimeUnit.MILLISECONDS);
286+
assertNotNull(data, "OOM Event not received");
287+
String event = new String(data);
288+
289+
assertThat(event, startsWith("_e"));
290+
assertThat(event, containsString(":OutOfMemoryError"));
291+
assertThat(event, containsString("t:error"));
292+
assertThat(event, containsString("s:java"));
382293
}
383294
}

‎dd-smoke-tests/crashtracking/src/test/java/datadog/smoketest/MinimalTelemetryData.java

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
package datadog.smoketest;
2+
3+
public class MinimalTelemetryData {
4+
String request_type;
5+
}

0 commit comments

Comments
 (0)