Skip to content

Commit c071e85

Browse files
author
Venkatesh Duggirala
committed
Bug#27610678 SEMI-SYNC REPLICATION DOESN'T WORK FOR MINUTES AFTER RESTART REPLICATION
Problem: ======== When semisync master is busy in reading acknowledgements from one semisync slave, adding another semisync slave to the master is getting delayed. Analysis: ========= In Ack_receiver::run(), server is holding m_mutex while it is reading acknowledgements from the ack receiver socket. The same mutex is required to add/remove a semisync slave to the Master. So if server is busy in reading the acknowledgements from the ack receiver socket, then add/remove a semisync will be delayed until m_mutex is released by Ack_receiver::run(). Fix: ==== Adjust acknowledgment reading logic in Ack_receiver::run() function in such a way that m_mutex is not held while reading for the acknowledgments from the ack receiver socket.
1 parent fadc507 commit c071e85

8 files changed

+415
-26
lines changed
Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
include/rpl_init.inc [topology=1->2, 1->3, 1->4, 1->5, 1->6, 1->7, 1->8, 1->9]
2+
Warnings:
3+
Note #### Sending passwords in plain text without SSL/TLS is extremely insecure.
4+
Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information.
5+
Warnings:
6+
Note #### Sending passwords in plain text without SSL/TLS is extremely insecure.
7+
Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information.
8+
Warnings:
9+
Note #### Sending passwords in plain text without SSL/TLS is extremely insecure.
10+
Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information.
11+
Warnings:
12+
Note #### Sending passwords in plain text without SSL/TLS is extremely insecure.
13+
Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information.
14+
Warnings:
15+
Note #### Sending passwords in plain text without SSL/TLS is extremely insecure.
16+
Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information.
17+
Warnings:
18+
Note #### Sending passwords in plain text without SSL/TLS is extremely insecure.
19+
Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information.
20+
Warnings:
21+
Note #### Sending passwords in plain text without SSL/TLS is extremely insecure.
22+
Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information.
23+
Warnings:
24+
Note #### Sending passwords in plain text without SSL/TLS is extremely insecure.
25+
Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information.
26+
#
27+
# 1. Initial data.
28+
#
29+
CREATE TABLE t1(c1 INT);
30+
include/rpl_sync.inc
31+
#
32+
# 2. Install semisync on all the 9 servers.
33+
#
34+
[connection server_1]
35+
CALL mtr.add_suppression("Semi-sync master failed on net_flush.*");
36+
include/install_semisync_master.inc
37+
[connection server_2]
38+
CALL mtr.add_suppression("Semi-sync slave net_flush.*");
39+
include/install_semisync_slave.inc
40+
[connection server_3]
41+
CALL mtr.add_suppression("Semi-sync slave net_flush.*");
42+
include/install_semisync_slave.inc
43+
[connection server_4]
44+
CALL mtr.add_suppression("Semi-sync slave net_flush.*");
45+
include/install_semisync_slave.inc
46+
[connection server_5]
47+
CALL mtr.add_suppression("Semi-sync slave net_flush.*");
48+
include/install_semisync_slave.inc
49+
[connection server_6]
50+
CALL mtr.add_suppression("Semi-sync slave net_flush.*");
51+
include/install_semisync_slave.inc
52+
[connection server_7]
53+
CALL mtr.add_suppression("Semi-sync slave net_flush.*");
54+
include/install_semisync_slave.inc
55+
[connection server_8]
56+
CALL mtr.add_suppression("Semi-sync slave net_flush.*");
57+
include/install_semisync_slave.inc
58+
[connection server_9]
59+
CALL mtr.add_suppression("Semi-sync slave net_flush.*");
60+
include/install_semisync_slave.inc
61+
#
62+
# 3. Make Master aware of 8 semisync slaves.
63+
#
64+
[connection server1_1]
65+
SET GLOBAL rpl_semi_sync_master_wait_for_slave_count = 8;
66+
#
67+
# 4. Insert 5000 tuples in background using 10 parallel connections.
68+
#
69+
#
70+
# 5. While the insert operations are going in the background, keep
71+
# adding/removing semisync slaves randomly.
72+
#
73+
#
74+
# 6. Add all 8 semisync slaves back to Master
75+
# (if they were disconnected, in above step).
76+
#
77+
#
78+
# 7. Make sure the data is synced on Master without any issues.
79+
#
80+
[connection server_1]
81+
include/rpl_sync.inc
82+
#
83+
# 8. Check on all servers semisync is enabled after the
84+
# the experiment and then uninstall semisync pluging
85+
# from it.
86+
#
87+
[connection server_9]
88+
include/assert.inc [ should be 1]
89+
include/uninstall_semisync_slave.inc
90+
[connection server_8]
91+
include/assert.inc [ should be 1]
92+
include/uninstall_semisync_slave.inc
93+
[connection server_7]
94+
include/assert.inc [ should be 1]
95+
include/uninstall_semisync_slave.inc
96+
[connection server_6]
97+
include/assert.inc [ should be 1]
98+
include/uninstall_semisync_slave.inc
99+
[connection server_5]
100+
include/assert.inc [ should be 1]
101+
include/uninstall_semisync_slave.inc
102+
[connection server_4]
103+
include/assert.inc [ should be 1]
104+
include/uninstall_semisync_slave.inc
105+
[connection server_3]
106+
include/assert.inc [ should be 1]
107+
include/uninstall_semisync_slave.inc
108+
[connection server_2]
109+
include/assert.inc [ should be 1]
110+
include/uninstall_semisync_slave.inc
111+
[connection server_1]
112+
include/assert.inc [ should be 1]
113+
include/uninstall_semisync_master.inc
114+
#
115+
# 9. Cleanup time.
116+
#
117+
[connection server_1]
118+
DROP TABLE t1;
119+
include/rpl_end.inc
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
$SEMISYNC_PLUGIN_OPT
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
$SEMISYNC_PLUGIN_OPT
Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
!include ../my.cnf
2+
3+
[mysqld.1]
4+
log-slave-updates
5+
server_id=1
6+
7+
[mysqld.2]
8+
log-slave-updates
9+
server_id=2
10+
11+
[mysqld.3]
12+
log-slave-updates
13+
server_id=3
14+
15+
[mysqld.4]
16+
log-slave-updates
17+
server_id=4
18+
19+
[mysqld.5]
20+
log-slave-updates
21+
server_id=5
22+
23+
[mysqld.6]
24+
log-slave-updates
25+
server_id=6
26+
27+
[mysqld.7]
28+
log-slave-updates
29+
server_id=7
30+
31+
[mysqld.8]
32+
log-slave-updates
33+
server_id=8
34+
35+
[mysqld.9]
36+
log-slave-updates
37+
server_id=9
38+
39+
[ENV]
40+
SERVER_MYPORT_3= @mysqld.3.port
41+
SERVER_MYSOCK_3= @mysqld.3.socket
42+
43+
SERVER_MYPORT_4= @mysqld.4.port
44+
SERVER_MYSOCK_4= @mysqld.4.socket
45+
46+
SERVER_MYPORT_5= @mysqld.5.port
47+
SERVER_MYSOCK_5= @mysqld.5.socket
48+
49+
SERVER_MYPORT_6= @mysqld.6.port
50+
SERVER_MYSOCK_6= @mysqld.6.socket
51+
52+
SERVER_MYPORT_7= @mysqld.7.port
53+
SERVER_MYSOCK_7= @mysqld.7.socket
54+
55+
SERVER_MYPORT_8= @mysqld.8.port
56+
SERVER_MYSOCK_8= @mysqld.8.socket
57+
58+
SERVER_MYPORT_9= @mysqld.9.port
59+
SERVER_MYSOCK_9= @mysqld.9.socket
Lines changed: 190 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,190 @@
1+
#==== Purpose ====
2+
#
3+
# This test script purpose is to check semisync replication
4+
# behaviour when there are semisync slaves getting added and
5+
# removed while dml changes are happening.
6+
#
7+
#==== Implementation ====
8+
#
9+
# 1) Install semisync on one Master and 8 Slaves.
10+
# 2) Using mysqlslap, keep inserting data in background on Master.
11+
# 3) While the DMLs are going on, keep adding/removing semisync slaves.
12+
# 4) At the end of the step-3, check that the data is synced without any
13+
# issues.
14+
# 5) Check semisync is still enabled on Master and all slaves.
15+
#
16+
#==== References ====
17+
#
18+
# Bug#27610678 SEMI-SYNC REPLICATION DOESN'T WORK FOR MINUTES AFTER RESTART
19+
# REPLICATION
20+
#
21+
#========
22+
#
23+
# 9 servers + 5000 inserts involved in test. Hence marking it as big-test.
24+
#
25+
--source include/big_test.inc
26+
--source include/not_group_replication_plugin.inc
27+
--source include/have_binlog_format_statement.inc
28+
--source include/have_debug.inc
29+
30+
#
31+
# One Master and 8 Slaves in the semi sync topology.
32+
#
33+
--let rpl_topology=1->2, 1->3, 1->4, 1->5, 1->6, 1->7, 1->8, 1->9
34+
--source include/rpl_init.inc
35+
36+
--echo #
37+
--echo # 1. Initial data.
38+
--echo #
39+
CREATE TABLE t1(c1 INT);
40+
--source include/rpl_sync.inc
41+
42+
--echo #
43+
--echo # 2. Install semisync on all the 9 servers.
44+
--echo #
45+
--let $i= 1
46+
while ($i < 10)
47+
{
48+
--let $rpl_connection_name= server_$i
49+
--source include/rpl_connection.inc
50+
if ( $i == 1)
51+
{
52+
--connect(server1_1, localhost,root,,,$MASTER_MYPORT)
53+
CALL mtr.add_suppression("Semi-sync master failed on net_flush.*");
54+
--source include/install_semisync_master.inc
55+
}
56+
if ( $i != 1 )
57+
{
58+
CALL mtr.add_suppression("Semi-sync slave net_flush.*");
59+
--source include/install_semisync_slave.inc
60+
}
61+
--inc $i
62+
}
63+
64+
--echo #
65+
--echo # 3. Make Master aware of 8 semisync slaves.
66+
--echo #
67+
--let $rpl_connection_name= server1_1
68+
--source include/rpl_connection.inc
69+
--let $slaves = 8
70+
--eval SET GLOBAL rpl_semi_sync_master_wait_for_slave_count = $slaves
71+
72+
--echo #
73+
--echo # 4. Insert 5000 tuples in background using 10 parallel connections.
74+
--echo #
75+
--exec_in_background $MYSQL_SLAP --create-schema=test --delimiter=";" --iterations=500 --query="INSERT INTO t1 values (1)" --concurrency=10 --silent 2>&1
76+
77+
--echo #
78+
--echo # 5. While the insert operations are going in the background, keep
79+
--echo # adding/removing semisync slaves randomly.
80+
--echo #
81+
--let $rpl_connection_silent = 1
82+
--let $include_silent= 1
83+
--disable_result_log
84+
--disable_query_log
85+
--let $rpl_debug= 0
86+
--let $iter= 1
87+
while ( $iter < 50 )
88+
{
89+
# Random server_id generator ( server_id will between [2-9]
90+
--let $server_id=`SELECT FLOOR(2 + ((RAND() * 100) % 8))`
91+
--let $rpl_connection_name= server_$server_id
92+
--source include/rpl_connection.inc
93+
--let $service_state= `SELECT service_state FROM performance_schema.replication_connection_status`
94+
if ($service_state == 'ON')
95+
{
96+
# If the replication state is ON, bring it down.
97+
--source include/stop_slave.inc
98+
--let $rpl_connection_name= server1_1
99+
--source include/rpl_connection.inc
100+
# Adjust rpl_semi_sync_master_wait_for_slave_count variable.
101+
--dec $slaves
102+
--eval SET GLOBAL rpl_semi_sync_master_wait_for_slave_count = $slaves
103+
}
104+
if ($service_state == 'OFF')
105+
{
106+
# If the replication state is OFF, bring it up.
107+
--source include/start_slave.inc
108+
--let $rpl_connection_name= server1_1
109+
--source include/rpl_connection.inc
110+
# Adjust rpl_semi_sync_master_wait_for_slave_count variable.
111+
--inc $slaves
112+
--eval SET GLOBAL rpl_semi_sync_master_wait_for_slave_count = $slaves
113+
}
114+
--inc $iter
115+
# Do it slowly while insertions on the master are happening in the background.
116+
sleep 1;
117+
}
118+
119+
--echo #
120+
--echo # 6. Add all 8 semisync slaves back to Master
121+
--echo # (if they were disconnected, in above step).
122+
--echo #
123+
--let $iter = 2
124+
while ($iter <= 9)
125+
{
126+
--let $rpl_connection_name= server_$iter
127+
--source include/rpl_connection.inc
128+
--let $service_state= `SELECT service_state FROM performance_schema.replication_connection_status`
129+
if ($service_state == 'OFF')
130+
{
131+
--source include/start_slave.inc
132+
--let $rpl_connection_name= server1_1
133+
--source include/rpl_connection.inc
134+
# Adjust rpl_semi_sync_master_wait_for_slave_count variable.
135+
--inc $slaves
136+
--eval SET GLOBAL rpl_semi_sync_master_wait_for_slave_count = $slaves
137+
}
138+
--inc $iter
139+
}
140+
--enable_result_log
141+
--enable_query_log
142+
--let $rpl_connection_silent = 0
143+
--let $include_silent= 0
144+
145+
146+
--echo #
147+
--echo # 7. Make sure the data is synced on Master without any issues.
148+
--echo #
149+
--let $rpl_connection_name= server_1
150+
--source include/rpl_connection.inc
151+
--let $wait_timeout= 60
152+
--let $wait_condition= SELECT count(*) = 5000 FROM t1
153+
--source include/wait_condition.inc
154+
--let $slave_timeout=600
155+
--source include/rpl_sync.inc
156+
157+
--echo #
158+
--echo # 8. Check on all servers semisync is enabled after the
159+
--echo # the experiment and then uninstall semisync pluging
160+
--echo # from it.
161+
--echo #
162+
--let $i= 9
163+
while ($i >= 1)
164+
{
165+
--let $rpl_connection_name= server_$i
166+
--source include/rpl_connection.inc
167+
if ( $i == 1)
168+
{
169+
--let $assert_variable_name= rpl_semi_sync_master_enabled
170+
--let $assert_variable_value= 1
171+
--source include/assert_variable.inc
172+
--source include/uninstall_semisync_master.inc
173+
}
174+
if ( $i != 1 )
175+
{
176+
--let $assert_variable_name= rpl_semi_sync_slave_enabled
177+
--let $assert_variable_value= 1
178+
--source include/assert_variable.inc
179+
--source include/uninstall_semisync_slave.inc
180+
}
181+
--dec $i
182+
}
183+
184+
--echo #
185+
--echo # 9. Cleanup time.
186+
--echo #
187+
--let $rpl_connection_name= server_1
188+
--source include/rpl_connection.inc
189+
DROP TABLE t1;
190+
--source include/rpl_end.inc

0 commit comments

Comments
 (0)