Skip to content

Commit 9fb8e4b

Browse files
vektory79bsideup
vektory79
authored and
bsideup
committed
More accurate log processing. (#643)
* More accurate log processing. The Docker logs can come in non normalized form. The line breaks can be not at end of line, the frame's boundary can split multibyte unicode symbols in the middle and so on. This patch address this problems and normalize logs so, that they can be processed strictly line by line. To achive this, the extra base class BaseConsumer have been added. That class normalize the incoming in method accept() logs and forward it to method process() for it's child classes. Other *Consumer classes have been reworked to be child to the BaseConsumer class and only do own work. Adititionally, BaseConsumer have new withRemoveAnsiCodes(boolean) method for ability to disable ANSI color codes removing (true by default). * Move normalization logic directly to FrameConsumerResultCallback It's help to not ruin the API of *Consumer classes. * Added some tests for log normalization in class FrameConsumerResultCallback. * Fix Codacity warning * More test for the FrameConsumerResultCallback class * Stabilize tests * Log consumers, that not derive BaseConsumer class, is not receive color codes now. Added record to the changelog. * One more test for FrameConsumerResultCallback class. * Fixes due to the code review recommendations * Fixes due to the code review recommendations (Part 2) * One more use case for FrameConsumerResultCallback class. And unit test for it. If StreamType is STDERR or STDOUT the log always have newline at line end. Therefore preprocessor should trim it to be consistent with RAW type processing. * Roll back previouse change doe to failing tests. Doing newline trimming directly in Slf4jLogConsumer class. * Fixes due to the code review recommendations (Part 3)
1 parent 11db027 commit 9fb8e4b

File tree

7 files changed

+319
-36
lines changed

7 files changed

+319
-36
lines changed

‎CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ All notable changes to this project will be documented in this file.
66
### Fixed
77
- Fixed missing `commons-codec` dependency ([\#642](https://github.com/testcontainers/testcontainers-java/issues/642))
88
- Fixed `HostPortWaitStrategy` throws `NumberFormatException` when port is exposed but not mapped ([\#640](https://github.com/testcontainers/testcontainers-java/issues/640))
9+
- Fixed log processing: multibyte unicode, linebreaks and ASCII color codes. Color codes can be turned on with `withRemoveAnsiCodes(false)` ([PR \#643](https://github.com/testcontainers/testcontainers-java/pull/643))
910

1011
### Changed
1112
- Support multiple HTTP status codes for HttpWaitStrategy ([\#630](https://github.com/testcontainers/testcontainers-java/issues/630))

‎core/src/main/java/org/testcontainers/containers/output/BaseConsumer.java

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
package org.testcontainers.containers.output;
2+
3+
import lombok.Getter;
4+
import lombok.Setter;
5+
6+
import java.util.function.Consumer;
7+
8+
public abstract class BaseConsumer<SELF extends BaseConsumer<SELF>> implements Consumer<OutputFrame> {
9+
@Getter
10+
@Setter
11+
private boolean removeColorCodes = true;
12+
13+
public SELF withRemoveAnsiCodes(boolean removeAnsiCodes) {
14+
this.removeColorCodes = removeAnsiCodes;
15+
return (SELF) this;
16+
}
17+
}

‎core/src/main/java/org/testcontainers/containers/output/FrameConsumerResultCallback.java

Lines changed: 102 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,28 +2,45 @@
22

33

44
import com.github.dockerjava.api.model.Frame;
5+
import com.github.dockerjava.api.model.StreamType;
56
import com.github.dockerjava.core.async.ResultCallbackTemplate;
67
import org.slf4j.Logger;
78
import org.slf4j.LoggerFactory;
89

910
import java.io.IOException;
11+
import java.util.ArrayList;
12+
import java.util.Arrays;
1013
import java.util.HashMap;
1114
import java.util.HashSet;
15+
import java.util.List;
1216
import java.util.Map;
1317
import java.util.concurrent.CountDownLatch;
1418
import java.util.function.Consumer;
19+
import java.util.regex.Pattern;
1520

1621
/**
1722
* This class can be used as a generic callback for docker-java commands that produce Frames.
1823
*/
1924
public class FrameConsumerResultCallback extends ResultCallbackTemplate<FrameConsumerResultCallback, Frame> {
2025

21-
private final static Logger LOGGER = LoggerFactory.getLogger(FrameConsumerResultCallback.class);
26+
private static final Logger LOGGER = LoggerFactory.getLogger(FrameConsumerResultCallback.class);
27+
28+
private static final byte[] EMPTY_LINE = new byte[0];
29+
30+
private static final Pattern ANSI_COLOR_PATTERN = Pattern.compile("\u001B\\[[0-9;]+m");
31+
32+
private static final String LINE_BREAK_REGEX = "((\\r?\\n)|(\\r))";
33+
34+
static final String LINE_BREAK_AT_END_REGEX = LINE_BREAK_REGEX + "$";
2235

2336
private Map<OutputFrame.OutputType, Consumer<OutputFrame>> consumers;
2437

2538
private CountDownLatch completionLatch = new CountDownLatch(1);
2639

40+
private StringBuilder logString = new StringBuilder();
41+
42+
private OutputFrame brokenFrame;
43+
2744
public FrameConsumerResultCallback() {
2845
consumers = new HashMap<>();
2946
}
@@ -45,9 +62,13 @@ public void onNext(Frame frame) {
4562
if (outputFrame != null) {
4663
Consumer<OutputFrame> consumer = consumers.get(outputFrame.getType());
4764
if (consumer == null) {
48-
LOGGER.error("got frame with type " + frame.getStreamType() + ", for which no handler is configured");
49-
} else {
50-
consumer.accept(outputFrame);
65+
LOGGER.error("got frame with type {}, for which no handler is configured", frame.getStreamType());
66+
} else if (outputFrame.getBytes() != null && outputFrame.getBytes().length > 0) {
67+
if (frame.getStreamType() == StreamType.RAW) {
68+
processRawFrame(outputFrame, consumer);
69+
} else {
70+
processOtherFrame(outputFrame, consumer);
71+
}
5172
}
5273
}
5374
}
@@ -63,8 +84,17 @@ public void onError(Throwable throwable) {
6384

6485
@Override
6586
public void close() throws IOException {
87+
OutputFrame lastLine = null;
88+
89+
if (logString.length() > 0) {
90+
lastLine = new OutputFrame(OutputFrame.OutputType.STDOUT, logString.toString().getBytes());
91+
}
92+
6693
// send an END frame to every consumer... but only once per consumer.
6794
for (Consumer<OutputFrame> consumer : new HashSet<>(consumers.values())) {
95+
if (lastLine != null) {
96+
consumer.accept(lastLine);
97+
}
6898
consumer.accept(OutputFrame.END);
6999
}
70100
super.close();
@@ -78,4 +108,72 @@ public void close() throws IOException {
78108
public CountDownLatch getCompletionLatch() {
79109
return completionLatch;
80110
}
111+
112+
private synchronized void processRawFrame(OutputFrame outputFrame, Consumer<OutputFrame> consumer) {
113+
String utf8String = outputFrame.getUtf8String();
114+
byte[] bytes = outputFrame.getBytes();
115+
116+
// Merging the strings by bytes to solve the problem breaking non-latin unicode symbols.
117+
if (brokenFrame != null) {
118+
bytes = merge(brokenFrame.getBytes(), bytes);
119+
utf8String = new String(bytes);
120+
brokenFrame = null;
121+
}
122+
// Logger chunks can break the string in middle of multibyte unicode character.
123+
// Backup the bytes to reconstruct proper char sequence with bytes from next frame.
124+
int lastCharacterType = Character.getType(utf8String.charAt(utf8String.length() - 1));
125+
if (lastCharacterType == Character.OTHER_SYMBOL) {
126+
brokenFrame = new OutputFrame(outputFrame.getType(), bytes);
127+
return;
128+
}
129+
130+
utf8String = processAnsiColorCodes(utf8String, consumer);
131+
normalizeLogLines(utf8String, consumer);
132+
}
133+
134+
private synchronized void processOtherFrame(OutputFrame outputFrame, Consumer<OutputFrame> consumer) {
135+
String utf8String = outputFrame.getUtf8String();
136+
137+
utf8String = processAnsiColorCodes(utf8String, consumer);
138+
consumer.accept(new OutputFrame(outputFrame.getType(), utf8String.getBytes()));
139+
}
140+
141+
private void normalizeLogLines(String utf8String, Consumer<OutputFrame> consumer) {
142+
// Reformat strings to normalize new lines.
143+
List<String> lines = new ArrayList<>(Arrays.asList(utf8String.split(LINE_BREAK_REGEX)));
144+
if (lines.isEmpty()) {
145+
consumer.accept(new OutputFrame(OutputFrame.OutputType.STDOUT, EMPTY_LINE));
146+
return;
147+
}
148+
if (utf8String.startsWith("\n") || utf8String.startsWith("\r")) {
149+
lines.add(0, "");
150+
}
151+
if (utf8String.endsWith("\n") || utf8String.endsWith("\r")) {
152+
lines.add("");
153+
}
154+
for (int i = 0; i < lines.size() - 1; i++) {
155+
String line = lines.get(i);
156+
if (i == 0 && logString.length() > 0) {
157+
line = logString.toString() + line;
158+
logString.setLength(0);
159+
}
160+
consumer.accept(new OutputFrame(OutputFrame.OutputType.STDOUT, line.getBytes()));
161+
}
162+
logString.append(lines.get(lines.size() - 1));
163+
}
164+
165+
private String processAnsiColorCodes(String utf8String, Consumer<OutputFrame> consumer) {
166+
if (!(consumer instanceof BaseConsumer) || ((BaseConsumer) consumer).isRemoveColorCodes()) {
167+
return ANSI_COLOR_PATTERN.matcher(utf8String).replaceAll("");
168+
}
169+
return utf8String;
170+
}
171+
172+
173+
private byte[] merge(byte[] str1, byte[] str2) {
174+
byte[] mergedString = new byte[str1.length + str2.length];
175+
System.arraycopy(str1, 0, mergedString, 0, str1.length);
176+
System.arraycopy(str2, 0, mergedString, str1.length, str2.length);
177+
return mergedString;
178+
}
81179
}

‎core/src/main/java/org/testcontainers/containers/output/Slf4jLogConsumer.java

Lines changed: 14 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -2,18 +2,13 @@
22

33
import org.slf4j.Logger;
44

5-
import java.util.function.Consumer;
6-
import java.util.regex.Pattern;
7-
85
/**
96
* A consumer for container output that logs output to an SLF4J logger.
107
*/
11-
public class Slf4jLogConsumer implements Consumer<OutputFrame> {
8+
public class Slf4jLogConsumer extends BaseConsumer<Slf4jLogConsumer> {
129
private final Logger logger;
1310
private String prefix = "";
1411

15-
private static final Pattern ANSI_CODE_PATTERN = Pattern.compile("\\[\\d[ABCD]");
16-
1712
public Slf4jLogConsumer(Logger logger) {
1813
this.logger = logger;
1914
}
@@ -25,28 +20,19 @@ public Slf4jLogConsumer withPrefix(String prefix) {
2520

2621
@Override
2722
public void accept(OutputFrame outputFrame) {
28-
if (outputFrame != null) {
29-
String utf8String = outputFrame.getUtf8String();
30-
31-
if (utf8String != null) {
32-
OutputFrame.OutputType outputType = outputFrame.getType();
33-
String message = utf8String.trim();
34-
35-
if (ANSI_CODE_PATTERN.matcher(message).matches()) {
36-
return;
37-
}
38-
39-
switch (outputType) {
40-
case END:
41-
break;
42-
case STDOUT:
43-
case STDERR:
44-
logger.info("{}{}: {}", prefix, outputType, message);
45-
break;
46-
default:
47-
throw new IllegalArgumentException("Unexpected outputType " + outputType);
48-
}
49-
}
23+
OutputFrame.OutputType outputType = outputFrame.getType();
24+
25+
String utf8String = outputFrame.getUtf8String();
26+
utf8String = utf8String.replaceAll(FrameConsumerResultCallback.LINE_BREAK_AT_END_REGEX, "");
27+
switch (outputType) {
28+
case END:
29+
break;
30+
case STDOUT:
31+
case STDERR:
32+
logger.info("{}{}: {}", prefix, outputType, utf8String);
33+
break;
34+
default:
35+
throw new IllegalArgumentException("Unexpected outputType " + outputType);
5036
}
5137
}
5238
}

‎core/src/main/java/org/testcontainers/containers/output/ToStringConsumer.java

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,21 +5,26 @@
55
import java.io.ByteArrayOutputStream;
66
import java.io.IOException;
77
import java.nio.charset.Charset;
8-
import java.util.function.Consumer;
98

109
/**
1110
* Created by rnorth on 26/03/2016.
1211
*/
13-
public class ToStringConsumer implements Consumer<OutputFrame> {
12+
public class ToStringConsumer extends BaseConsumer<ToStringConsumer> {
13+
private static final byte[] NEW_LINE = "\n".getBytes();
1414

15+
private boolean firstLine = true;
1516
private ByteArrayOutputStream stringBuffer = new ByteArrayOutputStream();
1617

1718
@Override
1819
public void accept(OutputFrame outputFrame) {
1920
try {
2021
if (outputFrame.getBytes() != null) {
22+
if (!firstLine) {
23+
stringBuffer.write(NEW_LINE);
24+
}
2125
stringBuffer.write(outputFrame.getBytes());
2226
stringBuffer.flush();
27+
firstLine = false;
2328
}
2429
} catch (IOException e) {
2530
throw new RuntimeException(e);

‎core/src/main/java/org/testcontainers/containers/output/WaitingConsumer.java

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,13 @@
66
import java.util.concurrent.LinkedBlockingDeque;
77
import java.util.concurrent.TimeUnit;
88
import java.util.concurrent.TimeoutException;
9-
import java.util.function.Consumer;
109
import java.util.function.Predicate;
1110

1211
/**
1312
* A consumer for container output that buffers lines in a {@link java.util.concurrent.BlockingDeque} and enables tests
1413
* to wait for a matching condition.
1514
*/
16-
public class WaitingConsumer implements Consumer<OutputFrame> {
15+
public class WaitingConsumer extends BaseConsumer<WaitingConsumer> {
1716

1817
private static final Logger LOGGER = LoggerFactory.getLogger(WaitingConsumer.class);
1918

0 commit comments

Comments
 (0)