Skip to content

Commit 159b9c7

Browse files
rashtaorashtao
authored
Active Failover Asynchronous driver (#381)
* fixed active failover communication in async driver * fixed active failover dirty read test * fixed active failover communication in async driver * fixed async StreamTransactionGraphTest * fixed race conditions at VST MessageStore.clear * enhanced VST connection debug logging * set default acquireHostList=true for tests * fixed race condition in ConnectionPool * added active-failover to GH actions * fixed some active-failover test failures * skipping getLogs test for ArangoDB <= 3.6 (BTS-362) * skipping getLogs test for ArangoDB <= 3.6 (BTS-362) * documented active failover acquireHostList and loadBalancingStrategy configuration * changelog upd
1 parent 339c834 commit 159b9c7

19 files changed

+195
-55
lines changed

‎.github/workflows/maven.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ jobs:
4242
topology:
4343
- single
4444
- cluster
45+
- active-failover
4546

4647
steps:
4748
- uses: actions/checkout@v1

‎ChangeLog.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a
66

77
## [Unreleased]
88

9+
- fixed active failover behavior for the asynchronous driver
10+
911
## [6.10.0] - 2021-03-27
1012

1113
- closing VST connection after 3 consecutive keepAlive failures (#ES-837)

‎docker/clean_active-failover.sh

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
#!/bin/bash
2+
3+
for c in server1 \
4+
server2 \
5+
server3; do
6+
docker rm -f $c
7+
done

‎docker/start_db_active-failover.sh

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
#!/bin/bash
2+
3+
# USAGE:
4+
# export ARANGO_LICENSE_KEY=<arangodb-enterprise-license>
5+
# ./start_active-failover.sh <dockerImage>
6+
7+
# EXAMPLE:
8+
# ./start_active-failover.sh docker.io/arangodb/arangodb:3.7.10
9+
10+
docker pull "$1"
11+
12+
LOCATION=$(pwd)/$(dirname "$0")
13+
14+
docker network create arangodb --subnet 172.28.0.0/16
15+
16+
echo "Averysecretword" >"$LOCATION"/jwtSecret
17+
docker run --rm -v "$LOCATION"/jwtSecret:/jwtSecret "$1" arangodb auth header --auth.jwt-secret /jwtSecret >"$LOCATION"/jwtHeader
18+
AUTHORIZATION_HEADER=$(cat "$LOCATION"/jwtHeader)
19+
20+
echo "Starting containers..."
21+
22+
docker run -d -v "$LOCATION"/jwtSecret:/jwtSecret -e ARANGO_LICENSE_KEY="$ARANGO_LICENSE_KEY" --network arangodb --ip 172.28.3.1 --name server1 "$1" sh -c 'arangodb --starter.address=$(hostname -i) --starter.mode=activefailover --starter.join server1,server2,server3 --auth.jwt-secret /jwtSecret'
23+
docker run -d -v "$LOCATION"/jwtSecret:/jwtSecret -e ARANGO_LICENSE_KEY="$ARANGO_LICENSE_KEY" --network arangodb --ip 172.28.3.2 --name server2 "$1" sh -c 'arangodb --starter.address=$(hostname -i) --starter.mode=activefailover --starter.join server1,server2,server3 --auth.jwt-secret /jwtSecret'
24+
docker run -d -v "$LOCATION"/jwtSecret:/jwtSecret -e ARANGO_LICENSE_KEY="$ARANGO_LICENSE_KEY" --network arangodb --ip 172.28.3.3 --name server3 "$1" sh -c 'arangodb --starter.address=$(hostname -i) --starter.mode=activefailover --starter.join server1,server2,server3 --auth.jwt-secret /jwtSecret'
25+
26+
debug_container() {
27+
running=$(docker inspect -f '{{.State.Running}}' "$1")
28+
29+
if [ "$running" = false ]; then
30+
echo "$1 is not running!"
31+
echo "---"
32+
docker logs "$1"
33+
echo "---"
34+
exit 1
35+
fi
36+
}
37+
38+
debug() {
39+
for c in server1 \
40+
server2 \
41+
server3; do
42+
debug_container $c
43+
done
44+
}
45+
46+
wait_server() {
47+
# shellcheck disable=SC2091
48+
until $(curl --output /dev/null --silent --head --fail -i -H "$AUTHORIZATION_HEADER" "http://$1/_api/version"); do
49+
printf '.'
50+
debug
51+
sleep 1
52+
done
53+
}
54+
55+
echo "Waiting..."
56+
57+
# Wait for agents:
58+
for a in 172.28.3.1:8529 \
59+
172.28.3.2:8529 \
60+
172.28.3.3:8529; do
61+
wait_server $a
62+
done
63+
64+
docker exec server1 arangosh --server.authentication=false --javascript.execute-string='require("org/arangodb/users").update("root", "test")'
65+
docker exec server2 arangosh --server.authentication=false --javascript.execute-string='require("org/arangodb/users").update("root", "test")'
66+
docker exec server3 arangosh --server.authentication=false --javascript.execute-string='require("org/arangodb/users").update("root", "test")'
67+
68+
#rm "$LOCATION"/jwtHeader "$LOCATION"/jwtSecret
69+
70+
echo "Done, your cluster is ready."

‎docker/start_db_active-failover_retry_fail.sh

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
#!/bin/bash
2+
3+
# USAGE:
4+
# export ARANGO_LICENSE_KEY=<arangodb-enterprise-license>
5+
# ./docker/start_db_active-failover_retry_fail.sh <dockerImage>
6+
7+
# EXAMPLE:
8+
# ./docker/start_db_active-failover_retry_fail.sh docker.io/arangodb/arangodb:3.7.10
9+
10+
./docker/start_db_active-failover.sh "$1"
11+
while [ $? -ne 0 ]; do
12+
echo "=== === ==="
13+
echo "active-failover startup failed, retrying ..."
14+
./docker/clean_active-failover.sh
15+
./docker/start_db_active-failover.sh "$1"
16+
done

‎src/main/java/com/arangodb/ArangoDB.java

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -286,6 +286,7 @@ public Builder keepAliveInterval(final Integer keepAliveInterval) {
286286
/**
287287
* Whether or not the driver should acquire a list of available coordinators in an ArangoDB cluster or a single
288288
* server with active failover.
289+
* In case of Active-Failover deployment set to {@code true} to enable automatic master discovery.
289290
*
290291
* <p>
291292
* The host list will be used for failover and load balancing.
@@ -312,6 +313,8 @@ public Builder acquireHostListInterval(final Integer acquireHostListInterval) {
312313

313314
/**
314315
* Sets the load balancing strategy to be used in an ArangoDB cluster setup.
316+
* In case of Active-Failover deployment set to {@link LoadBalancingStrategy#NONE} or not set at all, since that
317+
* would be the default.
315318
*
316319
* @param loadBalancingStrategy the load balancing strategy to be used (default: {@link LoadBalancingStrategy#NONE}
317320
* @return {@link ArangoDB.Builder}

‎src/main/java/com/arangodb/async/ArangoDBAsync.java

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -442,6 +442,7 @@ public Builder keepAliveInterval(final Integer keepAliveInterval) {
442442
/**
443443
* Whether or not the driver should acquire a list of available coordinators in an ArangoDB cluster or a single
444444
* server with active failover.
445+
* In case of Active-Failover deployment set to {@code true} to enable automatic master discovery.
445446
*
446447
* <p>
447448
* The host list will be used for failover and load balancing.
@@ -457,6 +458,8 @@ public Builder acquireHostList(final Boolean acquireHostList) {
457458

458459
/**
459460
* Sets the load balancing strategy to be used in an ArangoDB cluster setup.
461+
* In case of Active-Failover deployment set to {@link LoadBalancingStrategy#NONE} or not set at all, since that
462+
* would be the default.
460463
*
461464
* @param loadBalancingStrategy the load balancing strategy to be used (default: {@link LoadBalancingStrategy#NONE}
462465
* @return {@link ArangoDBAsync.Builder}

‎src/main/java/com/arangodb/async/internal/velocystream/VstCommunicationAsync.java

Lines changed: 30 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,11 @@
2121
package com.arangodb.async.internal.velocystream;
2222

2323
import com.arangodb.ArangoDBException;
24-
import com.arangodb.entity.ErrorEntity;
24+
import com.arangodb.internal.net.ArangoDBRedirectException;
25+
import com.arangodb.internal.net.HostDescription;
26+
import com.arangodb.internal.net.HostHandle;
2527
import com.arangodb.internal.net.HostHandler;
28+
import com.arangodb.internal.util.HostUtils;
2629
import com.arangodb.internal.velocystream.VstCommunication;
2730
import com.arangodb.internal.velocystream.internal.AuthenticationRequest;
2831
import com.arangodb.internal.velocystream.internal.Message;
@@ -58,23 +61,37 @@ protected CompletableFuture<Response> execute(final Request request, final VstCo
5861
final Message message = createMessage(request);
5962
send(message, connection).whenComplete((m, ex) -> {
6063
if (m != null) {
64+
final Response response;
6165
try {
62-
final Response response = createResponse(m);
63-
if (response.getResponseCode() >= 300) {
64-
if (response.getBody() != null) {
65-
final ErrorEntity errorEntity = util.deserialize(response.getBody(), ErrorEntity.class);
66-
rfuture.completeExceptionally(new ArangoDBException(errorEntity));
67-
} else {
68-
rfuture.completeExceptionally(new ArangoDBException(
69-
String.format("Response Code: %s", response.getResponseCode()), response.getResponseCode()));
70-
}
71-
} else {
72-
rfuture.complete(response);
73-
}
66+
response = createResponse(m);
7467
} catch (final VPackParserException e) {
7568
LOGGER.error(e.getMessage(), e);
7669
rfuture.completeExceptionally(e);
70+
return;
7771
}
72+
73+
try {
74+
checkError(response);
75+
} catch (final ArangoDBRedirectException e) {
76+
final String location = e.getLocation();
77+
final HostDescription redirectHost = HostUtils.createFromLocation(location);
78+
hostHandler.closeCurrentOnError();
79+
hostHandler.fail();
80+
execute(request, new HostHandle().setHost(redirectHost))
81+
.whenComplete((v, err) -> {
82+
if (v != null) {
83+
rfuture.complete(v);
84+
} else if (err != null) {
85+
rfuture.completeExceptionally(err);
86+
} else {
87+
rfuture.cancel(true);
88+
}
89+
});
90+
return;
91+
} catch (ArangoDBException e) {
92+
rfuture.completeExceptionally(e);
93+
}
94+
rfuture.complete(response);
7895
} else if (ex != null) {
7996
LOGGER.error(ex.getMessage(), ex);
8097
rfuture.completeExceptionally(ex);

‎src/main/java/com/arangodb/internal/net/ConnectionPoolImpl.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,7 @@ public synchronized Connection connection() {
7979
}
8080

8181
@Override
82-
public void close() throws IOException {
82+
public synchronized void close() throws IOException {
8383
for (final Connection connection : connections) {
8484
connection.close();
8585
}

‎src/main/java/com/arangodb/internal/velocystream/VstCommunication.java

Lines changed: 3 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -23,12 +23,9 @@
2323
import com.arangodb.ArangoDBException;
2424
import com.arangodb.internal.ArangoDefaults;
2525
import com.arangodb.internal.net.AccessType;
26-
import com.arangodb.internal.net.ArangoDBRedirectException;
2726
import com.arangodb.internal.net.Host;
28-
import com.arangodb.internal.net.HostDescription;
2927
import com.arangodb.internal.net.HostHandle;
3028
import com.arangodb.internal.net.HostHandler;
31-
import com.arangodb.internal.util.HostUtils;
3229
import com.arangodb.internal.util.RequestUtils;
3330
import com.arangodb.internal.util.ResponseUtils;
3431
import com.arangodb.internal.velocystream.internal.Chunk;
@@ -64,7 +61,7 @@ public abstract class VstCommunication<R, C extends VstConnection> implements Cl
6461
protected final String password;
6562

6663
protected final Integer chunksize;
67-
private final HostHandler hostHandler;
64+
protected final HostHandler hostHandler;
6865

6966
protected VstCommunication(final Integer timeout, final String user, final String password, final Boolean useSsl,
7067
final SSLContext sslContext, final ArangoSerialization util, final Integer chunksize,
@@ -134,20 +131,8 @@ public void close() throws IOException {
134131
}
135132

136133
public R execute(final Request request, final HostHandle hostHandle) throws ArangoDBException {
137-
try {
138-
final C connection = connect(hostHandle, RequestUtils.determineAccessType(request));
139-
return execute(request, connection);
140-
} catch (final ArangoDBException e) {
141-
if (e instanceof ArangoDBRedirectException) {
142-
final String location = ((ArangoDBRedirectException) e).getLocation();
143-
final HostDescription redirectHost = HostUtils.createFromLocation(location);
144-
hostHandler.closeCurrentOnError();
145-
hostHandler.fail();
146-
return execute(request, new HostHandle().setHost(redirectHost));
147-
} else {
148-
throw e;
149-
}
150-
}
134+
final C connection = connect(hostHandle, RequestUtils.determineAccessType(request));
135+
return execute(request, connection);
151136
}
152137

153138
protected abstract R execute(final Request request, C connection) throws ArangoDBException;

‎src/main/java/com/arangodb/internal/velocystream/VstCommunicationSync.java

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,11 @@
2121
package com.arangodb.internal.velocystream;
2222

2323
import com.arangodb.ArangoDBException;
24+
import com.arangodb.internal.net.ArangoDBRedirectException;
25+
import com.arangodb.internal.net.HostDescription;
26+
import com.arangodb.internal.net.HostHandle;
2427
import com.arangodb.internal.net.HostHandler;
28+
import com.arangodb.internal.util.HostUtils;
2529
import com.arangodb.internal.velocystream.internal.AuthenticationRequest;
2630
import com.arangodb.internal.velocystream.internal.Message;
2731
import com.arangodb.internal.velocystream.internal.VstConnectionSync;
@@ -127,6 +131,12 @@ protected Response execute(final Request request, final VstConnectionSync connec
127131
return response;
128132
} catch (final VPackParserException e) {
129133
throw new ArangoDBException(e);
134+
} catch (final ArangoDBRedirectException e) {
135+
final String location = e.getLocation();
136+
final HostDescription redirectHost = HostUtils.createFromLocation(location);
137+
hostHandler.closeCurrentOnError();
138+
hostHandler.fail();
139+
return execute(request, new HostHandle().setHost(redirectHost));
130140
}
131141
}
132142

‎src/main/java/com/arangodb/internal/velocystream/internal/MessageStore.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,7 @@ public void cancel(final long messageId) {
8282
}
8383
}
8484

85-
public void clear(final Exception e) {
85+
public synchronized void clear(final Exception e) {
8686
if (!task.isEmpty()) {
8787
LOGGER.error(e.getMessage(), e);
8888
}
@@ -96,7 +96,7 @@ public void clear(final Exception e) {
9696
task.clear();
9797
}
9898

99-
public void clear() {
99+
public synchronized void clear() {
100100
for (final Entry<Long, FutureTask<Message>> entry : task.entrySet()) {
101101
if (LOGGER.isDebugEnabled()) {
102102
LOGGER.debug(String.format("Cancel Message (id=%s).", entry.getKey()));

0 commit comments

Comments
 (0)