Bug#30930132 SUBSCRIPTION REPORTS SENT OUT TOO EARLY BY SUMA DURING NODE RESTART

Problem:
During a node restart the suma block of the restarting node sends
subscription reports to the new subscriber and, if requested, informs
it about all other subscribers. Currently, these reports are sent too
early for the subscribers as they won't be connected to the restarting
node yet. Due to this they miss out these subscription reports during
a node restart.

The MySQL Server depends on these subscription reports to maintain a list
of other MySQL Servers(participants) connected to the cluster. During
schema distribution, the MySQL Server logs the schema change and waits
for these participants to finish applying the changes before returning.
If all the data nodes connected in a cluster are restarted one by one,
the MySQL Servers will miss out on all the subscription reports and thus
end up losing all details about the connected participants.

Solution:
SUMA block changed in order to, during the node restart, wait until all
the subscribers are connected to send out the reports.
Also, the reporting mechanism is optimized to send to subscribers only
the necessary information.

New API leve test added to test_event
New MTR test ndb_participants added

Change-Id: I879fe0fa535f4afd503d85f926f55388ce9f6227

Author: vinc13e

mysql-test/suite/ndb/r/ndb_participants.result

@@ -0,0 +1,38 @@
+# Execute a DDL and verify the participant list in the servers
+create table t1 (a int) engine ndb;
+include/assert_grep.inc [Found two participants in Server 1]
+drop table t1;
+include/assert_grep.inc [Found two participants in Server 2]
+# Restart all nodes one by one
+Restart data node 1
+Node 1 is being restarted
+
+# Verify the participants again to confirm that they were not lost after node 1 stopped
+create table t1 (a int) engine ndb;
+include/assert_grep.inc [Found two participants in Server 1]
+drop table t1;
+include/assert_grep.inc [Found two participants in Server 2]
+Database node 1 is being started.
+
+Node 1 restarted
+# Verify the participants again to confirm that they were not lost after node 1 restart
+create table t1 (a int) engine ndb;
+include/assert_grep.inc [Found two participants in Server 1]
+drop table t1;
+include/assert_grep.inc [Found two participants in Server 2]
+Restart data node 2
+Node 2 is being restarted
+
+# Verify the participants again to confirm that they were not lost after node 2 stopped
+create table t1 (a int) engine ndb;
+include/assert_grep.inc [Found two participants in Server 1]
+drop table t1;
+include/assert_grep.inc [Found two participants in Server 2]
+Database node 2 is being started.
+
+Node 1 restarted
+# Verify the participants again to confirm that they were not lost after node 2 restart
+create table t1 (a int) engine ndb;
+include/assert_grep.inc [Found two participants in Server 1]
+drop table t1;
+include/assert_grep.inc [Found two participants in Server 2]

mysql-test/suite/ndb/t/ndb_participants.test

@@ -0,0 +1,117 @@
+--source include/have_ndb.inc
+--source include/have_multi_ndb.inc
+
+--echo # Execute a DDL and verify the participant list in the servers
+
+--connection server1
+create table t1 (a int) engine ndb;
+
+--let $assert_text= Found two participants in Server 1
+--let $assert_file= $MYSQLTEST_VARDIR/log/mysqld.1.1.err
+# MySQL Server prints the connected participants in the format :
+# [NDB] Participants [<participant1 node id>,<participant2 node id>..]
+--let $assert_select= \[NDB\] Participants: \[[0-9]+,[0-9]+\]
+--let $assert_count= 1
+--source include/assert_grep.inc
+
+--connection server2
+drop table t1;
+
+--let $assert_text= Found two participants in Server 2
+--let $assert_file= $MYSQLTEST_VARDIR/log/mysqld.2.1.err
+--let $assert_select= \[NDB\] Participants: \[[0-9]+,[0-9]+\]
+--let $assert_count= 1
+--source include/assert_grep.inc
+
+--echo # Restart all nodes one by one
+--echo Restart data node 1
+--exec $NDB_MGM -e "1 RESTART -n"
+--exec $NDB_WAITER --nowait-nodes=2 --not-started  > /dev/null
+
+--echo # Verify the participants again to confirm that they were not lost after node 1 stopped
+--connection server1
+create table t1 (a int) engine ndb;
+
+--let $assert_text= Found two participants in Server 1
+--let $assert_file= $MYSQLTEST_VARDIR/log/mysqld.1.1.err
+--let $assert_select= \[NDB\] Participants: \[[0-9]+,[0-9]+\]
+--let $assert_count= 2
+--source include/assert_grep.inc
+
+--connection server2
+drop table t1;
+
+--let $assert_text= Found two participants in Server 2
+--let $assert_file= $MYSQLTEST_VARDIR/log/mysqld.2.1.err
+--let $assert_select= \[NDB\] Participants: \[[0-9]+,[0-9]+\]
+--let $assert_count= 2
+--source include/assert_grep.inc
+
+--exec $NDB_MGM -e "1 START"
+--exec $NDB_WAITER  > /dev/null
+--echo Node 1 restarted
+
+--echo # Verify the participants again to confirm that they were not lost after node 1 restart
+--connection server1
+create table t1 (a int) engine ndb;
+
+--let $assert_text= Found two participants in Server 1
+--let $assert_file= $MYSQLTEST_VARDIR/log/mysqld.1.1.err
+--let $assert_select= \[NDB\] Participants: \[[0-9]+,[0-9]+\]
+--let $assert_count= 3
+--source include/assert_grep.inc
+
+--connection server2
+drop table t1;
+
+--let $assert_text= Found two participants in Server 2
+--let $assert_file= $MYSQLTEST_VARDIR/log/mysqld.2.1.err
+--let $assert_select= \[NDB\] Participants: \[[0-9]+,[0-9]+\]
+--let $assert_count= 3
+--source include/assert_grep.inc
+
+--echo Restart data node 2
+--exec $NDB_MGM -e "2 RESTART -n"
+--exec $NDB_WAITER --nowait-nodes=1 --not-started  > /dev/null
+
+--echo # Verify the participants again to confirm that they were not lost after node 2 stopped
+--connection server1
+create table t1 (a int) engine ndb;
+
+--let $assert_text= Found two participants in Server 1
+--let $assert_file= $MYSQLTEST_VARDIR/log/mysqld.1.1.err
+--let $assert_select= \[NDB\] Participants: \[[0-9]+,[0-9]+\]
+--let $assert_count= 4
+--source include/assert_grep.inc
+
+--connection server2
+drop table t1;
+
+--let $assert_text= Found two participants in Server 2
+--let $assert_file= $MYSQLTEST_VARDIR/log/mysqld.2.1.err
+--let $assert_select= \[NDB\] Participants: \[[0-9]+,[0-9]+\]
+--let $assert_count= 4
+--source include/assert_grep.inc
+
+--exec $NDB_MGM -e "2 START"
+--exec $NDB_WAITER  > /dev/null
+--echo Node 1 restarted
+
+--echo # Verify the participants again to confirm that they were not lost after node 2 restart
+--connection server1
+create table t1 (a int) engine ndb;
+
+--let $assert_text= Found two participants in Server 1
+--let $assert_file= $MYSQLTEST_VARDIR/log/mysqld.1.1.err
+--let $assert_select= \[NDB\] Participants: \[[0-9]+,[0-9]+\]
+--let $assert_count= 5
+--source include/assert_grep.inc
+
+--connection server2
+drop table t1;
+
+--let $assert_text= Found two participants in Server 2
+--let $assert_file= $MYSQLTEST_VARDIR/log/mysqld.2.1.err
+--let $assert_select= \[NDB\] Participants: \[[0-9]+,[0-9]+\]
+--let $assert_count= 5
+--source include/assert_grep.inc

storage/ndb/include/kernel/signaldata/SumaImpl.hpp

@@ -593,6 +593,7 @@ struct SumaContinueB
     ,WAIT_SCAN_TAB_REQ = 10
     ,WAIT_GET_FRAGMENT = 11
     ,SEND_SUB_GCP_COMPLETE_REP = 12
+    ,REPORT_SUBSCRIPTION_SET = 13
   };
 };
 

storage/ndb/src/kernel/blocks/ERROR_codes.txt

@@ -34,7 +34,7 @@ Next TRPMAN 9501
 Next BACKUP 10055
 Next PGMAN 11010
 Next DBTUX 12010
-Next SUMA 13058
+Next SUMA 13060
 Next LGMAN 15002
 Next TSMAN 16002
 Next DBSPJ 17000
@@ -855,6 +855,8 @@ SUMA:
 13051: Delay the DROP_TRIG_IMPL_REQ
 13057: Changing the ndbrequire from checking whether the gci is higher than
        the known gci instead of the next gci
+13058: Delay subscription reporting during node restart + cause API failure
+13059: Delay subscription reporting during node restart
 
 LGMAN:
 -----