Skip to content

Commit e95d218

Browse files
ShaneHarveyShaneHarvey
authored
PYTHON-2395 Consider connection pool health during server selection (#515)
Change the driver to maintain a count of in-progress operations to each server (per client). When selecting a mongos server, the driver now picks 2 suitable servers at random and selects the server with fewer in-progress operations. Previously, the driver selected a mongos server at random. The new behavior is intended to route operations away from unhealthy or slow servers in highly concurrent single client workloads. PYTHON-2460 Only reset Pool.active_sockets to 0 after a fork()
1 parent ac07e0f commit e95d218

13 files changed

+632
-31
lines changed

‎pymongo/pool.py

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1111,15 +1111,20 @@ def __init__(self, address, options, handshake=True):
11111111
if self.enabled_for_cmap:
11121112
self.opts.event_listeners.publish_pool_created(
11131113
self.address, self.opts.non_default_options)
1114+
# Similar to active_sockets but includes threads in the wait queue.
1115+
self.operation_count = 0
11141116

11151117
def _reset(self, close):
11161118
with self.lock:
11171119
if self.closed:
11181120
return
11191121
self.generation += 1
1120-
self.pid = os.getpid()
1122+
newpid = os.getpid()
1123+
if self.pid != newpid:
1124+
self.pid = newpid
1125+
self.active_sockets = 0
1126+
self.operation_count = 0
11211127
sockets, self.sockets = self.sockets, collections.deque()
1122-
self.active_sockets = 0
11231128
if close:
11241129
self.closed = True
11251130

@@ -1300,6 +1305,9 @@ def _get_socket(self, all_credentials):
13001305
'Attempted to check out a connection from closed connection '
13011306
'pool')
13021307

1308+
with self.lock:
1309+
self.operation_count += 1
1310+
13031311
# Get a free socket or create one.
13041312
if self.opts.wait_queue_timeout:
13051313
deadline = _time() + self.opts.wait_queue_timeout
@@ -1396,6 +1404,7 @@ def return_socket(self, sock_info):
13961404
self._socket_semaphore.release()
13971405
with self.lock:
13981406
self.active_sockets -= 1
1407+
self.operation_count -= 1
13991408

14001409
def _perished(self, sock_info):
14011410
"""Return True and close the connection if it is "perished".

‎pymongo/topology.py

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -238,9 +238,15 @@ def select_server(self,
238238
server_selection_timeout=None,
239239
address=None):
240240
"""Like select_servers, but choose a random server if several match."""
241-
return random.choice(self.select_servers(selector,
242-
server_selection_timeout,
243-
address))
241+
servers = self.select_servers(
242+
selector, server_selection_timeout, address)
243+
if len(servers) == 1:
244+
return servers[0]
245+
server1, server2 = random.sample(servers, 2)
246+
if server1.pool.operation_count <= server2.pool.operation_count:
247+
return server1
248+
else:
249+
return server2
244250

245251
def select_server_by_address(self, address,
246252
server_selection_timeout=None):

‎test/server_selection/in_window/equilibrium.json

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
{
2+
"description": "When in equilibrium selection is evenly distributed",
3+
"topology_description": {
4+
"type": "Sharded",
5+
"servers": [
6+
{
7+
"address": "a:27017",
8+
"avg_rtt_ms": 35,
9+
"type": "Mongos"
10+
},
11+
{
12+
"address": "b:27017",
13+
"avg_rtt_ms": 35,
14+
"type": "Mongos"
15+
},
16+
{
17+
"address": "c:27017",
18+
"avg_rtt_ms": 35,
19+
"type": "Mongos"
20+
}
21+
]
22+
},
23+
"mocked_topology_state": [
24+
{
25+
"address": "a:27017",
26+
"operation_count": 5
27+
},
28+
{
29+
"address": "b:27017",
30+
"operation_count": 5
31+
},
32+
{
33+
"address": "c:27017",
34+
"operation_count": 5
35+
}
36+
],
37+
"iterations": 2000,
38+
"outcome": {
39+
"tolerance": 0.05,
40+
"expected_frequencies": {
41+
"a:27017": 0.33,
42+
"b:27017": 0.33,
43+
"c:27017": 0.33
44+
}
45+
}
46+
}

‎test/server_selection/in_window/many-choices.json

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
{
2+
"description": "Selections from many choices occur at correct frequencies",
3+
"topology_description": {
4+
"type": "Sharded",
5+
"servers": [
6+
{
7+
"address": "a:27017",
8+
"avg_rtt_ms": 35,
9+
"type": "Mongos"
10+
},
11+
{
12+
"address": "b:27017",
13+
"avg_rtt_ms": 35,
14+
"type": "Mongos"
15+
},
16+
{
17+
"address": "c:27017",
18+
"avg_rtt_ms": 35,
19+
"type": "Mongos"
20+
},
21+
{
22+
"address": "d:27017",
23+
"avg_rtt_ms": 35,
24+
"type": "Mongos"
25+
},
26+
{
27+
"address": "e:27017",
28+
"avg_rtt_ms": 35,
29+
"type": "Mongos"
30+
},
31+
{
32+
"address": "f:27017",
33+
"avg_rtt_ms": 35,
34+
"type": "Mongos"
35+
},
36+
{
37+
"address": "g:27017",
38+
"avg_rtt_ms": 35,
39+
"type": "Mongos"
40+
},
41+
{
42+
"address": "h:27017",
43+
"avg_rtt_ms": 35,
44+
"type": "Mongos"
45+
},
46+
{
47+
"address": "i:27017",
48+
"avg_rtt_ms": 35,
49+
"type": "Mongos"
50+
}
51+
]
52+
},
53+
"mocked_topology_state": [
54+
{
55+
"address": "a:27017",
56+
"operation_count": 0
57+
},
58+
{
59+
"address": "b:27017",
60+
"operation_count": 5
61+
},
62+
{
63+
"address": "c:27017",
64+
"operation_count": 5
65+
},
66+
{
67+
"address": "d:27017",
68+
"operation_count": 10
69+
},
70+
{
71+
"address": "e:27017",
72+
"operation_count": 10
73+
},
74+
{
75+
"address": "f:27017",
76+
"operation_count": 20
77+
},
78+
{
79+
"address": "g:27017",
80+
"operation_count": 20
81+
},
82+
{
83+
"address": "h:27017",
84+
"operation_count": 50
85+
},
86+
{
87+
"address": "i:27017",
88+
"operation_count": 60
89+
}
90+
],
91+
"iterations": 10000,
92+
"outcome": {
93+
"tolerance": 0.03,
94+
"expected_frequencies": {
95+
"a:27017": 0.22,
96+
"b:27017": 0.18,
97+
"c:27017": 0.18,
98+
"d:27017": 0.125,
99+
"e:27017": 0.125,
100+
"f:27017": 0.074,
101+
"g:27017": 0.074,
102+
"h:27017": 0.0277,
103+
"i:27017": 0
104+
}
105+
}
106+
}

‎test/server_selection/in_window/one-least-two-tied.json

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
{
2+
"description": "Least operations gets most selections, two tied share the rest",
3+
"topology_description": {
4+
"type": "Sharded",
5+
"servers": [
6+
{
7+
"address": "a:27017",
8+
"avg_rtt_ms": 35,
9+
"type": "Mongos"
10+
},
11+
{
12+
"address": "b:27017",
13+
"avg_rtt_ms": 35,
14+
"type": "Mongos"
15+
},
16+
{
17+
"address": "c:27017",
18+
"avg_rtt_ms": 35,
19+
"type": "Mongos"
20+
}
21+
]
22+
},
23+
"mocked_topology_state": [
24+
{
25+
"address": "a:27017",
26+
"operation_count": 16
27+
},
28+
{
29+
"address": "b:27017",
30+
"operation_count": 10
31+
},
32+
{
33+
"address": "c:27017",
34+
"operation_count": 16
35+
}
36+
],
37+
"iterations": 2000,
38+
"outcome": {
39+
"tolerance": 0.05,
40+
"expected_frequencies": {
41+
"a:27017": 0.165,
42+
"b:27017": 0.66,
43+
"c:27017": 0.165
44+
}
45+
}
46+
}

‎test/server_selection/in_window/rs-equilibrium.json

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
{
2+
"description": "When in equilibrium selection is evenly distributed (replica set)",
3+
"topology_description": {
4+
"type": "ReplicaSetWithPrimary",
5+
"servers": [
6+
{
7+
"address": "a:27017",
8+
"avg_rtt_ms": 35,
9+
"type": "RSPrimary"
10+
},
11+
{
12+
"address": "b:27017",
13+
"avg_rtt_ms": 35,
14+
"type": "RSSecondary"
15+
},
16+
{
17+
"address": "c:27017",
18+
"avg_rtt_ms": 35,
19+
"type": "RSSecondary"
20+
}
21+
]
22+
},
23+
"mocked_topology_state": [
24+
{
25+
"address": "a:27017",
26+
"operation_count": 6
27+
},
28+
{
29+
"address": "b:27017",
30+
"operation_count": 6
31+
},
32+
{
33+
"address": "c:27017",
34+
"operation_count": 6
35+
}
36+
],
37+
"iterations": 2000,
38+
"outcome": {
39+
"tolerance": 0.05,
40+
"expected_frequencies": {
41+
"a:27017": 0.33,
42+
"b:27017": 0.33,
43+
"c:27017": 0.33
44+
}
45+
}
46+
}

‎test/server_selection/in_window/rs-three-choices.json

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
{
2+
"description": "Selections from three servers occur at proper distributions (replica set)",
3+
"topology_description": {
4+
"type": "ReplicaSetWithPrimary",
5+
"servers": [
6+
{
7+
"address": "a:27017",
8+
"avg_rtt_ms": 35,
9+
"type": "RSPrimary"
10+
},
11+
{
12+
"address": "b:27017",
13+
"avg_rtt_ms": 35,
14+
"type": "RSSecondary"
15+
},
16+
{
17+
"address": "c:27017",
18+
"avg_rtt_ms": 35,
19+
"type": "RSSecondary"
20+
}
21+
]
22+
},
23+
"mocked_topology_state": [
24+
{
25+
"address": "a:27017",
26+
"operation_count": 3
27+
},
28+
{
29+
"address": "b:27017",
30+
"operation_count": 6
31+
},
32+
{
33+
"address": "c:27017",
34+
"operation_count": 20
35+
}
36+
],
37+
"iterations": 2000,
38+
"outcome": {
39+
"tolerance": 0.05,
40+
"expected_frequencies": {
41+
"a:27017": 0.66,
42+
"b:27017": 0.33,
43+
"c:27017": 0
44+
}
45+
}
46+
}

0 commit comments

Comments
 (0)