BUG#19641482 RPL.RPL_MTS_LOGICAL_CLOCK_RECOVERY FAILS WTH "SERVER FAILED

             TO DISSAPPEAR"

Problem
-------

The test case is failing to make the slave server to "disappear".

Analysis
--------

The "crash_in_a_worker" debug sync point is relying on the fact that the
workload will be parallelized and reach MTS worker #2, but on slow
systems the parallelization will not happen and the server will fail to
"disappear".

Fix
---

Ensure that the workload will be distributed by at all the workers
even on slow systems.

Author: Joao Gramacho

mysql-test/suite/rpl/r/rpl_mts_logical_clock_recovery.result

@@ -3,23 +3,42 @@ Warnings:
 Note	####	Sending passwords in plain text without SSL/TLS is extremely insecure.
 Note	####	Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information.
 [connection master]
-SET GLOBAL DEBUG ='+d,set_commit_parent_100';
+# Adding debug point 'set_commit_parent_100' to @@GLOBAL.debug
 CREATE TABLE t1 (a int primary key) engine=innodb;
 CREATE TABLE t2 (a int primary key) engine=innodb;
 CREATE TABLE t3 (a int primary key) engine=innodb;
 CREATE TABLE t4 (a int primary key) engine=innodb;
 include/sync_slave_sql_with_master.inc
-include/stop_slave.inc
-SET GLOBAL DEBUG ='+d,crash_in_a_worker';
-SET GLOBAL DEBUG ='-d,set_commit_parent_100';
-SET GLOBAL DEBUG ='+d,set_commit_parent_150';
-INSERT INTO t1 values (1),(2),(3);
+include/stop_slave_sql.inc
+# Adding debug point 'crash_in_a_worker' to @@GLOBAL.debug
+# Removing debug point 'set_commit_parent_100' from @@GLOBAL.debug
+# Adding debug point 'set_commit_parent_150' to @@GLOBAL.debug
+INSERT INTO t1 values (0);
 BEGIN;
 INSERT INTO t2 values (1),(2),(3);
 UPDATE t2 SET a= a+1 WHERE a=3;
 COMMIT;
+BEGIN;
 INSERT INTO t3 values (1),(2),(3);
+UPDATE t3 SET a= a+1 WHERE a=3;
+COMMIT;
+BEGIN;
 INSERT INTO t4 values (1),(2),(3);
+UPDATE t4 SET a= a+1 WHERE a=3;
+COMMIT;
+BEGIN;
+INSERT INTO t1 values (1),(2),(3);
+UPDATE t1 SET a= a+1 WHERE a=3;
+COMMIT;
+# Removing debug point 'set_commit_parent_150' from @@GLOBAL.debug
+include/sync_slave_io_with_master.inc
+[connection slave1]
+BEGIN;
+INSERT INTO t1 (a) VALUES (4);
+INSERT INTO t2 (a) VALUES (4);
+INSERT INTO t3 (a) VALUES (4);
+INSERT INTO t4 (a) VALUES (4);
+[connection slave]
 START SLAVE;
 include/rpl_start_server.inc [server_number=2]
 # Reconnecting to the slave server

mysql-test/suite/rpl/t/rpl_mts_logical_clock_recovery.test

@@ -4,46 +4,79 @@
 -- source include/have_debug.inc
 -- source include/not_valgrind.inc
 -- source include/master-slave.inc
--- let $saved_debug=`SELECT @@global.DEBUG`
-SET GLOBAL DEBUG ='+d,set_commit_parent_100';
+
+--let $debug_point=set_commit_parent_100
+--source include/add_debug_point.inc
 
 # set up the tables and replicate it to the slave.
 CREATE TABLE t1 (a int primary key) engine=innodb;
 CREATE TABLE t2 (a int primary key) engine=innodb;
 CREATE TABLE t3 (a int primary key) engine=innodb;
 CREATE TABLE t4 (a int primary key) engine=innodb;
 -- source include/sync_slave_sql_with_master.inc
--- source include/stop_slave.inc
--- let $saved_debug_slave=`SELECT @@global.DEBUG`
-SET GLOBAL DEBUG ='+d,crash_in_a_worker';
+-- source include/stop_slave_sql.inc
+--let $debug_point=crash_in_a_worker
+--source include/add_debug_point.inc
 
 -- connection master
-
-SET GLOBAL DEBUG ='-d,set_commit_parent_100';
-SET GLOBAL DEBUG ='+d,set_commit_parent_150';
+--let $debug_point=set_commit_parent_100
+--source include/remove_debug_point.inc
+--let $debug_point=set_commit_parent_150
+--source include/add_debug_point.inc
 
 # execute transactions
 
-# TRX1
-INSERT INTO t1 values (1),(2),(3);
+# trx1 should be applied and its worker released for receiving more work.
+# trx2 to trx5 are prepared in a way that they will be scheduled on the slave
+# but will not be applied. In order to have all four transactions scheduled,
+# the coordinator will have to distribute work by all four workers. The test
+# case expects the slave to crash after some workload be distributed to the
+# worker #2.
 
-# trx2 will fail on the slave.
+# trx1
+INSERT INTO t1 values (0);
+
+# trx2 should not complete on the slave.
 BEGIN;
 INSERT INTO t2 values (1),(2),(3);
 UPDATE t2 SET a= a+1 WHERE a=3;
 COMMIT;
 
-# trx3
+# trx3 should not complete on the slave.
+BEGIN;
 INSERT INTO t3 values (1),(2),(3);
+UPDATE t3 SET a= a+1 WHERE a=3;
+COMMIT;
 
-# trx4
+# trx4 should not complete on the slave.
+BEGIN;
 INSERT INTO t4 values (1),(2),(3);
+UPDATE t4 SET a= a+1 WHERE a=3;
+COMMIT;
 
---disable_query_log ONCE
---disable_result_log ONCE
-SET GLOBAL DEBUG ='$saved_debug';
+# trx5 should not complete on the slave.
+BEGIN;
+INSERT INTO t1 values (1),(2),(3);
+UPDATE t1 SET a= a+1 WHERE a=3;
+COMMIT;
+
+--let $debug_point=set_commit_parent_150
+--source include/remove_debug_point.inc
+
+# Make sure the workload is already on the relay log before starting the
+# applier to avoid issues with slow systems.
+--source include/sync_slave_io_with_master.inc
 
---connection slave
+# This should prevent trx2 to trx5 from committing and will make all
+# four MTS workers to be busy regardless of server speed.
+--source include/rpl_connection_slave1.inc
+BEGIN;
+INSERT INTO t1 (a) VALUES (4);
+INSERT INTO t2 (a) VALUES (4);
+INSERT INTO t3 (a) VALUES (4);
+INSERT INTO t4 (a) VALUES (4);
+
+--source include/rpl_connection_slave.inc
 # make sure mtr expects a crash & restarts
 -- exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.2.expect
 START SLAVE;
@@ -53,12 +86,6 @@ START SLAVE;
 -- enable_reconnect
 -- echo # Reconnecting to the slave server
 -- source include/wait_until_connected_again.inc
--- connection slave
--- disable_query_log ONCE
--- disable_result_log ONCE
--- disable_reconnect
--- eval SET GLOBAL DEBUG ='$saved_debug_slave';
-
 -- connection slave
 START SLAVE UNTIL SQL_AFTER_MTS_GAPS;