Skip to content

Commit 3416000

Browse files
authored
[DE-725] Bugfix VST resilience (#529)
* increased wait time in resilience tests * CI: disable tests for draft PRs * refactoring VstCommunication * refactoring VstConnection * auth fail test * unify VST and HTTP communication * test fix * rm disabled consumer thread tests
1 parent 3e80267 commit 3416000

File tree

22 files changed

+420
-550
lines changed

22 files changed

+420
-550
lines changed

Diff for: .github/workflows/resilience.yml

-3
Original file line numberDiff line numberDiff line change
@@ -25,9 +25,6 @@ jobs:
2525
env:
2626
TOXIPROXY_VERSION: v2.7.0
2727

28-
strategy:
29-
fail-fast: false
30-
3128
steps:
3229
- uses: actions/checkout@v2
3330
- name: Set up JDK

Diff for: .github/workflows/test.yml

+7
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ on:
2020
jobs:
2121

2222
test:
23+
if: '! github.event.pull_request.draft'
2324
timeout-minutes: 20
2425
runs-on: ubuntu-latest
2526

@@ -63,6 +64,7 @@ jobs:
6364
run: mvn --no-transfer-progress -am -pl driver test -DargLine="-Duser.language=${{matrix.user-language}}"
6465

6566
test-ssl:
67+
if: '! github.event.pull_request.draft'
6668
timeout-minutes: 10
6769
runs-on: ubuntu-latest
6870

@@ -98,6 +100,7 @@ jobs:
98100

99101
# test encodeURIComponent() and normalize('NFC') comparing to Javascript behavior
100102
test-graalvm:
103+
if: '! github.event.pull_request.draft'
101104
runs-on: ubuntu-latest
102105
steps:
103106
- uses: actions/checkout@v2
@@ -113,6 +116,7 @@ jobs:
113116
run: mvn -e --no-transfer-progress -am -pl driver test -Dtest=graalvm.UnicodeUtilsTest -Dsurefire.failIfNoSpecifiedTests=false
114117

115118
test-jwt:
119+
if: '! github.event.pull_request.draft'
116120
timeout-minutes: 20
117121
runs-on: ubuntu-latest
118122

@@ -160,6 +164,7 @@ jobs:
160164
run: mvn --no-transfer-progress -am -pl driver test -DargLine="-Duser.language=${{matrix.user-language}}"
161165

162166
jackson-test:
167+
if: '! github.event.pull_request.draft'
163168
timeout-minutes: 20
164169
runs-on: ubuntu-latest
165170

@@ -205,6 +210,7 @@ jobs:
205210
run: mvn --no-transfer-progress -am -pl driver test -Dadb.jackson.version=${{matrix.jackson-version}}
206211

207212
integration-tests:
213+
if: '! github.event.pull_request.draft'
208214
timeout-minutes: 20
209215
runs-on: ubuntu-latest
210216

@@ -250,6 +256,7 @@ jobs:
250256
run: mvn --no-transfer-progress -Pplain test
251257

252258
sonar:
259+
if: '! github.event.pull_request.draft'
253260
timeout-minutes: 10
254261
runs-on: ubuntu-latest
255262

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,161 @@
1+
package com.arangodb.internal.net;
2+
3+
import com.arangodb.ArangoDBException;
4+
import com.arangodb.config.HostDescription;
5+
import com.arangodb.internal.InternalRequest;
6+
import com.arangodb.internal.InternalResponse;
7+
import com.arangodb.internal.RequestType;
8+
import com.arangodb.internal.config.ArangoConfig;
9+
import com.arangodb.internal.serde.InternalSerde;
10+
import com.arangodb.internal.util.HostUtils;
11+
import com.arangodb.internal.util.RequestUtils;
12+
import com.arangodb.internal.util.ResponseUtils;
13+
import org.slf4j.Logger;
14+
import org.slf4j.LoggerFactory;
15+
16+
import java.io.Closeable;
17+
import java.io.IOException;
18+
import java.net.ConnectException;
19+
import java.net.SocketTimeoutException;
20+
import java.util.concurrent.CompletableFuture;
21+
import java.util.concurrent.CompletionException;
22+
import java.util.concurrent.TimeoutException;
23+
import java.util.concurrent.atomic.AtomicLong;
24+
25+
public abstract class Communication implements Closeable {
26+
private static final Logger LOGGER = LoggerFactory.getLogger(Communication.class);
27+
protected final HostHandler hostHandler;
28+
protected final InternalSerde serde;
29+
private final AtomicLong reqCount;
30+
31+
32+
protected Communication(final ArangoConfig config, final HostHandler hostHandler) {
33+
this.hostHandler = hostHandler;
34+
serde = config.getInternalSerde();
35+
reqCount = new AtomicLong();
36+
}
37+
38+
protected abstract void connect(final Connection conn) throws IOException;
39+
40+
@Override
41+
public void close() throws IOException {
42+
hostHandler.close();
43+
}
44+
45+
public CompletableFuture<InternalResponse> executeAsync(final InternalRequest request, final HostHandle hostHandle) {
46+
return executeAsync(request, hostHandle, hostHandler.get(hostHandle, RequestUtils.determineAccessType(request)), 0);
47+
}
48+
49+
private CompletableFuture<InternalResponse> executeAsync(final InternalRequest request, final HostHandle hostHandle, final Host host, final int attemptCount) {
50+
long reqId = reqCount.getAndIncrement();
51+
return doExecuteAsync(request, hostHandle, host, attemptCount, host.connection(), reqId);
52+
}
53+
54+
private CompletableFuture<InternalResponse> doExecuteAsync(
55+
final InternalRequest request, final HostHandle hostHandle, final Host host, final int attemptCount, Connection connection, long reqId
56+
) {
57+
if (LOGGER.isDebugEnabled()) {
58+
String body = request.getBody() == null ? "" : serde.toJsonString(request.getBody());
59+
LOGGER.debug("Send Request [id={}]: {} {}", reqId, request, body);
60+
}
61+
final CompletableFuture<InternalResponse> rfuture = new CompletableFuture<>();
62+
try {
63+
connect(connection);
64+
} catch (IOException e) {
65+
handleException(true, e, hostHandle, request, host, reqId, attemptCount, rfuture);
66+
return rfuture;
67+
}
68+
69+
connection.executeAsync(request)
70+
.whenComplete((response, e) -> {
71+
try {
72+
if (e instanceof SocketTimeoutException) {
73+
// SocketTimeoutException exceptions are wrapped and rethrown.
74+
TimeoutException te = new TimeoutException(e.getMessage());
75+
te.initCause(e);
76+
rfuture.completeExceptionally(ArangoDBException.of(te, reqId));
77+
} else if (e instanceof TimeoutException) {
78+
rfuture.completeExceptionally(ArangoDBException.of(e, reqId));
79+
} else if (e instanceof ConnectException) {
80+
handleException(true, e, hostHandle, request, host, reqId, attemptCount, rfuture);
81+
} else if (e != null) {
82+
handleException(isSafe(request), e, hostHandle, request, host, reqId, attemptCount, rfuture);
83+
} else {
84+
if (LOGGER.isDebugEnabled()) {
85+
String body = response.getBody() == null ? "" : serde.toJsonString(response.getBody());
86+
LOGGER.debug("Received Response [id={}]: {} {}", reqId, response, body);
87+
}
88+
ArangoDBException errorEntityEx = ResponseUtils.translateError(serde, response);
89+
if (errorEntityEx instanceof ArangoDBRedirectException) {
90+
if (attemptCount >= 3) {
91+
rfuture.completeExceptionally(errorEntityEx);
92+
} else {
93+
final String location = ((ArangoDBRedirectException) errorEntityEx).getLocation();
94+
final HostDescription redirectHost = HostUtils.createFromLocation(location);
95+
hostHandler.failIfNotMatch(redirectHost, errorEntityEx);
96+
mirror(
97+
executeAsync(request, new HostHandle().setHost(redirectHost), hostHandler.get(hostHandle, RequestUtils.determineAccessType(request)), attemptCount + 1),
98+
rfuture
99+
);
100+
}
101+
} else if (errorEntityEx != null) {
102+
rfuture.completeExceptionally(errorEntityEx);
103+
} else {
104+
hostHandler.success();
105+
rfuture.complete(response);
106+
}
107+
}
108+
} catch (Exception ex) {
109+
rfuture.completeExceptionally(ArangoDBException.of(ex, reqId));
110+
}
111+
});
112+
return rfuture;
113+
}
114+
115+
private void handleException(boolean isSafe, Throwable e, HostHandle hostHandle, InternalRequest request, Host host,
116+
long reqId, int attemptCount, CompletableFuture<InternalResponse> rfuture) {
117+
IOException ioEx = wrapIOEx(e);
118+
hostHandler.fail(ioEx);
119+
if (hostHandle != null && hostHandle.getHost() != null) {
120+
hostHandle.setHost(null);
121+
}
122+
hostHandler.checkNext(hostHandle, RequestUtils.determineAccessType(request));
123+
if (isSafe) {
124+
Host nextHost = hostHandler.get(hostHandle, RequestUtils.determineAccessType(request));
125+
LOGGER.warn("Could not connect to {} while executing request [id={}]",
126+
host.getDescription(), reqId, ioEx);
127+
LOGGER.debug("Try connecting to {}", nextHost.getDescription());
128+
mirror(
129+
executeAsync(request, hostHandle, nextHost, attemptCount),
130+
rfuture
131+
);
132+
} else {
133+
ArangoDBException aEx = ArangoDBException.of(ioEx, reqId);
134+
rfuture.completeExceptionally(aEx);
135+
}
136+
}
137+
138+
private void mirror(CompletableFuture<InternalResponse> up, CompletableFuture<InternalResponse> down) {
139+
up.whenComplete((v, err) -> {
140+
if (err != null) {
141+
down.completeExceptionally(err instanceof CompletionException ? err.getCause() : err);
142+
} else {
143+
down.complete(v);
144+
}
145+
});
146+
}
147+
148+
private static IOException wrapIOEx(Throwable t) {
149+
if (t instanceof IOException) {
150+
return (IOException) t;
151+
} else {
152+
return new IOException(t);
153+
}
154+
}
155+
156+
private boolean isSafe(final InternalRequest request) {
157+
RequestType type = request.getRequestType();
158+
return type == RequestType.GET || type == RequestType.HEAD || type == RequestType.OPTIONS;
159+
}
160+
161+
}

Diff for: core/src/main/java/com/arangodb/internal/net/Connection.java

+6
Original file line numberDiff line numberDiff line change
@@ -20,11 +20,17 @@
2020

2121
package com.arangodb.internal.net;
2222

23+
import com.arangodb.internal.InternalRequest;
24+
import com.arangodb.internal.InternalResponse;
25+
2326
import java.io.Closeable;
27+
import java.util.concurrent.CompletableFuture;
2428

2529
/**
2630
* @author Mark Vollmary
2731
*/
2832
public interface Connection extends Closeable {
2933
void setJwt(String jwt);
34+
35+
CompletableFuture<InternalResponse> executeAsync(InternalRequest request);
3036
}

Diff for: driver/src/test/java/com/arangodb/ConsumerThreadAsyncTest.java

-88
Original file line numberDiff line numberDiff line change
@@ -1,101 +1,13 @@
11
package com.arangodb;
22

33
import com.arangodb.config.ArangoConfigProperties;
4-
import org.junit.jupiter.api.Disabled;
54
import org.junit.jupiter.params.ParameterizedTest;
65
import org.junit.jupiter.params.provider.EnumSource;
76

8-
import java.util.UUID;
97
import java.util.concurrent.ExecutionException;
10-
import java.util.concurrent.ExecutorService;
11-
import java.util.concurrent.Executors;
12-
13-
import static org.assertj.core.api.Assertions.assertThat;
148

159
public class ConsumerThreadAsyncTest extends BaseJunit5 {
1610

17-
private volatile Thread thread;
18-
19-
private void setThread() {
20-
thread = Thread.currentThread();
21-
}
22-
23-
private void sleep() {
24-
try {
25-
Thread.sleep(3_000);
26-
} catch (InterruptedException e) {
27-
throw new RuntimeException(e);
28-
}
29-
}
30-
31-
@ParameterizedTest
32-
@EnumSource(Protocol.class)
33-
@Disabled
34-
void defaultConsumerThread(Protocol protocol) throws ExecutionException, InterruptedException {
35-
ArangoDBAsync adb = new ArangoDB.Builder()
36-
.loadProperties(ArangoConfigProperties.fromFile())
37-
.protocol(protocol)
38-
.build()
39-
.async();
40-
41-
adb.getVersion()
42-
.thenAccept(it -> setThread())
43-
.get();
44-
45-
adb.shutdown();
46-
47-
if (Protocol.VST.equals(protocol)) {
48-
assertThat(thread.getName()).startsWith("adb-vst-");
49-
} else {
50-
assertThat(thread.getName()).startsWith("adb-http-");
51-
}
52-
}
53-
54-
@ParameterizedTest
55-
@EnumSource(Protocol.class)
56-
void customConsumerExecutor(Protocol protocol) throws ExecutionException, InterruptedException {
57-
ExecutorService es = Executors.newCachedThreadPool(r -> {
58-
Thread t = Executors.defaultThreadFactory().newThread(r);
59-
t.setName("custom-" + UUID.randomUUID());
60-
return t;
61-
});
62-
ArangoDBAsync adb = new ArangoDB.Builder()
63-
.loadProperties(ArangoConfigProperties.fromFile())
64-
.protocol(protocol)
65-
.asyncExecutor(es)
66-
.build()
67-
.async();
68-
69-
adb.getVersion()
70-
.thenAccept(it -> setThread())
71-
.get();
72-
73-
adb.shutdown();
74-
es.shutdown();
75-
assertThat(thread.getName()).startsWith("custom-");
76-
}
77-
78-
/**
79-
* Generates warns from Vert.x BlockedThreadChecker
80-
*/
81-
@ParameterizedTest
82-
@EnumSource(Protocol.class)
83-
@Disabled
84-
void sleepOnDefaultConsumerThread(Protocol protocol) throws ExecutionException, InterruptedException {
85-
ArangoDBAsync adb = new ArangoDB.Builder()
86-
.loadProperties(ArangoConfigProperties.fromFile())
87-
.protocol(protocol)
88-
.maxConnections(1)
89-
.build()
90-
.async();
91-
92-
adb.getVersion()
93-
.thenAccept(it -> sleep())
94-
.get();
95-
96-
adb.shutdown();
97-
}
98-
9911
@ParameterizedTest
10012
@EnumSource(Protocol.class)
10113
void nestedRequests(Protocol protocol) throws ExecutionException, InterruptedException {

0 commit comments

Comments
 (0)