Bug#36313793 --ndb-log-apply-status fails to work with no replica changes [3/3]

Problem:

The NDB table, ndb_apply_status, tracks the replica applier state. If
replicated changes were applied but are not visible in the Binlog
Injector (log-replica-updates=OFF) - because NDB data nodes filtered
those - then the epoch, from the Binlog Injector point-of-view, is
deemed to be empty.  But, ndb_apply_status did have an update because
those changes were replicated in the cluster. This breaks the flow of
apply status information in a chain/circular replication deployment.

Analysis:

Running some chain/circular replication scenarios, it is possible to
observe, in the binlog injector handling of data events, that the
ndb_apply_status table updates are written into the binlog
transaction, thus not filtered (system tables do not have
skip-log-replica-updates data node filter).

If only ndb_apply_status updates are written, then it's considered an
empty epoch, as there's no actual table data to write to the
binlog. Therefore, the binlog transaction will be rolled back, in case
logging of empty epochs is off. But, due to the nature of the
`skip-log-replica-updates` filter in the NDB data nodes, some data
might have been replicated even if not received by the binlog thread.

Solution:

The usage of ndb-log-apply-status=ON with circular replication is a
complex case. Requires the understanding that even for a server that
has skip-log-replica-updates, ndb_apply_status updates are still
logged (and thus continue to propagate through the chain).

With the above, this patch does not enable the replica updates filter
in the NDB data nodes for the cases that ndb-log-apply-status or
ndb-log-orig are ON. If log-replica-updates is OFF (filter might be
enabled and in operation) then ndb-log-apply-status and ndb-log-orig
cannot be toggled.

Tests adapted to this new behavior.

Change-Id: I48944c536142b87d361e73eacf0f7a699616fa66

Author: dtcpatricio

mysql-test/suite/ndb_rpl/my.cnf

@@ -25,7 +25,6 @@ ndb-wait-connected=30
 ndb-wait-setup=120
 ndb-cluster-connection-pool=3
 replica-allow-batching
-ndb-log-orig
 ndb-log-bin=1
 
 # Enable load data local functionality

mysql-test/suite/ndb_rpl/r/ndb_rpl_circular.result

@@ -7,7 +7,9 @@ Note	####	Sending passwords in plain text without SSL/TLS is extremely insecure.
 Note	####	Storing MySQL user name or password information in the connection metadata repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START REPLICA; see the 'START REPLICA Syntax' in the MySQL Manual for more information.
 include/rpl/connect.inc [creating master]
 include/rpl/connect.inc [creating slave]
+# create the table on the "slave"
 CREATE TABLE t1 (a int key, b int) ENGINE=ndb;
+# now we should have a table on the master as well
 SHOW TABLES;
 Tables_in_test
 t1
@@ -70,70 +72,4 @@ master-bin.000001	#	Write_rows	#	#	table_id: # flags: STMT_END_F
 master-bin.000001	#	Query	#	#	COMMIT
 STOP REPLICA;
 DROP TABLE t1;
-START REPLICA;
-create table t1 (a int primary key, b int) engine=ndb;
-show variables like '%log_orig%';
-Variable_name	Value
-ndb_log_orig	ON
-'Master' has only slave's serverid entry 
-select server_id, log_name from mysql.ndb_apply_status order by server_id;
-server_id	log_name
-2	slave-bin.000001
-set global ndb_log_apply_status=On;
-show variables like 'ndb_log_apply_status';
-Variable_name	Value
-ndb_log_apply_status	ON
-'Slave' has only Master's serverid entry
-select server_id, log_name from mysql.ndb_apply_status order by server_id;
-server_id	log_name
-1	master-bin.000001
-'Slave' has following ndb_binlog_index entries
-select inserts, updates, deletes, schemaops, orig_server_id from mysql.ndb_binlog_index order by position,orig_epoch;
-inserts	updates	deletes	schemaops	orig_server_id
-1	0	0	0	1
-1	0	0	0	2
-set global ndb_log_apply_status=On;
-show variables like 'ndb_log_apply_status';
-Variable_name	Value
-ndb_log_apply_status	ON
-STOP REPLICA;
-insert into t1 values (1,1);
-'Slave' still has only Master's serverid entry
-select server_id, log_name from mysql.ndb_apply_status order by server_id;
-server_id	log_name
-1	master-bin.000001
-'Slave' has following ndb_binlog_index entries
-select inserts, updates, deletes, schemaops, orig_server_id from mysql.ndb_binlog_index order by position,orig_epoch;
-inserts	updates	deletes	schemaops	orig_server_id
-1	0	0	0	1
-1	0	0	0	2
-0	0	0	0	2
-1	0	0	0	1
-'Master' still has only Slave's serverid entry
-select server_id, log_name from mysql.ndb_apply_status order by server_id;
-server_id	log_name
-2	slave-bin.000001
-START REPLICA;
-'Master' now has own serverid entry as well.
-select server_id, log_name from mysql.ndb_apply_status order by server_id;
-server_id	log_name
-1	master-bin.000001
-2	slave-bin.000001
-'Slave' still only has 'Master''s serverid entry
-select server_id, log_name from mysql.ndb_apply_status order by server_id;
-server_id	log_name
-1	master-bin.000001
-Now create event originating at Slave
-insert into t1 values (2,2);
-'Slave' now also has its own serverid entry
-select server_id, log_name from mysql.ndb_apply_status order by server_id;
-server_id	log_name
-1	master-bin.000001
-2	slave-bin.000001
-STOP REPLICA;
-set global ndb_log_apply_status=off;
-set global ndb_log_apply_status=off;
-STOP REPLICA;
-drop table t1;
-drop table t1;
 include/rpl/deinit.inc

mysql-test/suite/ndb_rpl/r/ndb_rpl_circular_apply_status.result

@@ -0,0 +1,93 @@
+include/rpl/init.inc [topology=1->2->1]
+Warnings:
+Note	####	Sending passwords in plain text without SSL/TLS is extremely insecure.
+Note	####	Storing MySQL user name or password information in the connection metadata repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START REPLICA; see the 'START REPLICA Syntax' in the MySQL Manual for more information.
+Warnings:
+Note	####	Sending passwords in plain text without SSL/TLS is extremely insecure.
+Note	####	Storing MySQL user name or password information in the connection metadata repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START REPLICA; see the 'START REPLICA Syntax' in the MySQL Manual for more information.
+include/rpl/connect.inc [creating source]
+include/rpl/connect.inc [creating replica]
+# Validating servers setup
+[connection source]
+SELECT @@log_replica_updates;
+@@log_replica_updates
+0
+SELECT @@ndb_log_orig;
+@@ndb_log_orig
+1
+SELECT @@ndb_log_apply_status;
+@@ndb_log_apply_status
+1
+# Since server has log-replica-updates=OFF, it cannot set following variables
+SET GLOBAL ndb_log_orig=ON;
+SHOW WARNINGS;
+Level	Code	Message
+Warning	1231	Variable 'ndb_log_orig' can't be changed when 'log_replica_updates' is OFF
+Error	1231	Variable 'ndb_log_orig' can't be set to the value of 'ON'
+SET GLOBAL ndb_log_orig=OFF;
+SHOW WARNINGS;
+Level	Code	Message
+Warning	1231	Variable 'ndb_log_orig' can't be changed when 'log_replica_updates' is OFF
+Error	1231	Variable 'ndb_log_orig' can't be set to the value of 'OFF'
+SET GLOBAL ndb_log_apply_status=ON;
+SHOW WARNINGS;
+Level	Code	Message
+Warning	1231	Variable 'ndb_log_apply_status' can't be changed when 'log_replica_updates' is OFF
+Error	1231	Variable 'ndb_log_apply_status' can't be set to the value of 'ON'
+SET GLOBAL ndb_log_apply_status=OFF;
+SHOW WARNINGS;
+Level	Code	Message
+Warning	1231	Variable 'ndb_log_apply_status' can't be changed when 'log_replica_updates' is OFF
+Error	1231	Variable 'ndb_log_apply_status' can't be set to the value of 'OFF'
+[connection replica]
+SELECT @@log_replica_updates;
+@@log_replica_updates
+1
+SELECT @@ndb_log_orig;
+@@ndb_log_orig
+1
+SELECT @@ndb_log_apply_status;
+@@ndb_log_apply_status
+1
+# Since server has log-replica-updates=ON, it can toggle following variables
+SET GLOBAL ndb_log_orig=OFF;
+SET GLOBAL ndb_log_orig=ON;
+SET GLOBAL ndb_log_apply_status=OFF;
+SET GLOBAL ndb_log_apply_status=ON;
+# Now create table
+[connection source]
+CREATE TABLE t1 (a INT PRIMARY KEY, b INT) ENGINE = NDB;
+include/rpl/stop_replica.inc
+INSERT INTO t1 VALUES (1,1);
+[connection replica]
+# Replica has only source's serverid entry
+select server_id, log_name from mysql.ndb_apply_status order by server_id;
+server_id	log_name
+1	master-bin.000001
+# Replica has following ndb_binlog_index entries
+select inserts, updates, deletes, schemaops, orig_server_id from mysql.ndb_binlog_index order by position,orig_epoch;
+inserts	updates	deletes	schemaops	orig_server_id
+0	0	0	0	2
+1	0	0	0	1
+[connection source]
+include/rpl/start_replica.inc
+# Source now has own serverid entry as well
+select server_id, log_name from mysql.ndb_apply_status order by server_id;
+server_id	log_name
+1	master-bin.000001
+2	slave-bin.000001
+[connection replica]
+# Replica still only has source's serverid entry
+select server_id, log_name from mysql.ndb_apply_status order by server_id;
+server_id	log_name
+1	master-bin.000001
+# Now create event originating at replica
+INSERT INTO t1 VALUES (2,2);
+# Replica now also has its own serverid entry
+select server_id, log_name from mysql.ndb_apply_status order by server_id;
+server_id	log_name
+1	master-bin.000001
+2	slave-bin.000001
+# Cleanup
+DROP TABLE t1;
+include/rpl/deinit.inc

mysql-test/suite/ndb_rpl/r/ndb_rpl_slave_lsu_anyval.result

@@ -19,8 +19,6 @@ CREATE TABLE bug_45756_slave_logged_1 (a int) engine = NDB;
 INSERT INTO bug_45756_slave_logged_1 VALUES (1);
 DROP TABLE bug_45756_slave_logged_1;
 CREATE TABLE bug_45756_slave_logged_2 (a int) engine = NDB;
-Warnings:
-Warning	1296	Replica updates are only filtered in NDB when using --server-id-bits=32
 INSERT INTO bug_45756_slave_logged_2 VALUES (1);
 DROP TABLE bug_45756_slave_logged_2;
 CREATE TABLE bug_45756_slave_logged_3 (a int) engine = NDB;
@@ -34,8 +32,6 @@ DROP TABLE bug_45756_slave_not_logged_1;
 SET SQL_LOG_BIN= 1;
 SET SQL_LOG_BIN= 0;
 CREATE TABLE bug_45756_slave_not_logged_2 (a int) engine = NDB;
-Warnings:
-Warning	1296	Replica updates are only filtered in NDB when using --server-id-bits=32
 INSERT INTO bug_45756_slave_not_logged_2 VALUES (1);
 DROP TABLE bug_45756_slave_not_logged_2;
 SET SQL_LOG_BIN= 1;
@@ -46,23 +42,17 @@ DROP TABLE bug_45756_slave_not_logged_3;
 SET SQL_LOG_BIN= 1;
 *** Generating data to be replicated ***
 CREATE TABLE bug45756_master_logged_1 (a int) engine = NDB;
-Warnings:
-Warning	1296	Replica updates are only filtered in NDB when using --server-id-bits=32
 INSERT INTO bug45756_master_logged_1 VALUES (1);
 DROP TABLE bug45756_master_logged_1;
 CREATE TABLE bug45756_master_logged_2 (a int) engine = NDB;
 INSERT INTO bug45756_master_logged_2 VALUES (1);
 DROP TABLE bug45756_master_logged_2;
 CREATE TABLE bug45756_master_logged_3 (a int) engine = NDB;
-Warnings:
-Warning	1296	Replica updates are only filtered in NDB when using --server-id-bits=32
 INSERT INTO bug45756_master_logged_3 VALUES (1);
 DROP TABLE bug45756_master_logged_3;
 *** Generating changes not to be replicated ***
 SET SQL_LOG_BIN= 0;
 CREATE TABLE bug45756_master_not_logged_1 (a int) engine = NDB;
-Warnings:
-Warning	1296	Replica updates are only filtered in NDB when using --server-id-bits=32
 INSERT INTO bug45756_master_not_logged_1 VALUES (1);
 DROP TABLE bug45756_master_not_logged_1;
 SET SQL_LOG_BIN= 1;
@@ -73,8 +63,6 @@ DROP TABLE bug45756_master_not_logged_2;
 SET SQL_LOG_BIN= 1;
 SET SQL_LOG_BIN= 0;
 CREATE TABLE bug45756_master_not_logged_3 (a int) engine = NDB;
-Warnings:
-Warning	1296	Replica updates are only filtered in NDB when using --server-id-bits=32
 INSERT INTO bug45756_master_not_logged_3 VALUES (1);
 DROP TABLE bug45756_master_not_logged_3;
 SET SQL_LOG_BIN= 1;
@@ -430,8 +418,6 @@ CREATE TABLE bug_45756_slave_logged_1 (a int) engine = NDB;
 INSERT INTO bug_45756_slave_logged_1 VALUES (1);
 DROP TABLE bug_45756_slave_logged_1;
 CREATE TABLE bug_45756_slave_logged_2 (a int) engine = NDB;
-Warnings:
-Warning	1296	Replica updates are only filtered in NDB when using --server-id-bits=32
 INSERT INTO bug_45756_slave_logged_2 VALUES (1);
 DROP TABLE bug_45756_slave_logged_2;
 CREATE TABLE bug_45756_slave_logged_3 (a int) engine = NDB;
@@ -445,8 +431,6 @@ DROP TABLE bug_45756_slave_not_logged_1;
 SET SQL_LOG_BIN= 1;
 SET SQL_LOG_BIN= 0;
 CREATE TABLE bug_45756_slave_not_logged_2 (a int) engine = NDB;
-Warnings:
-Warning	1296	Replica updates are only filtered in NDB when using --server-id-bits=32
 INSERT INTO bug_45756_slave_not_logged_2 VALUES (1);
 DROP TABLE bug_45756_slave_not_logged_2;
 SET SQL_LOG_BIN= 1;
@@ -457,23 +441,17 @@ DROP TABLE bug_45756_slave_not_logged_3;
 SET SQL_LOG_BIN= 1;
 *** Generating data to be replicated ***
 CREATE TABLE bug45756_master_logged_1 (a int) engine = NDB;
-Warnings:
-Warning	1296	Replica updates are only filtered in NDB when using --server-id-bits=32
 INSERT INTO bug45756_master_logged_1 VALUES (1);
 DROP TABLE bug45756_master_logged_1;
 CREATE TABLE bug45756_master_logged_2 (a int) engine = NDB;
 INSERT INTO bug45756_master_logged_2 VALUES (1);
 DROP TABLE bug45756_master_logged_2;
 CREATE TABLE bug45756_master_logged_3 (a int) engine = NDB;
-Warnings:
-Warning	1296	Replica updates are only filtered in NDB when using --server-id-bits=32
 INSERT INTO bug45756_master_logged_3 VALUES (1);
 DROP TABLE bug45756_master_logged_3;
 *** Generating changes not to be replicated ***
 SET SQL_LOG_BIN= 0;
 CREATE TABLE bug45756_master_not_logged_1 (a int) engine = NDB;
-Warnings:
-Warning	1296	Replica updates are only filtered in NDB when using --server-id-bits=32
 INSERT INTO bug45756_master_not_logged_1 VALUES (1);
 DROP TABLE bug45756_master_not_logged_1;
 SET SQL_LOG_BIN= 1;
@@ -484,8 +462,6 @@ DROP TABLE bug45756_master_not_logged_2;
 SET SQL_LOG_BIN= 1;
 SET SQL_LOG_BIN= 0;
 CREATE TABLE bug45756_master_not_logged_3 (a int) engine = NDB;
-Warnings:
-Warning	1296	Replica updates are only filtered in NDB when using --server-id-bits=32
 INSERT INTO bug45756_master_not_logged_3 VALUES (1);
 DROP TABLE bug45756_master_not_logged_3;
 SET SQL_LOG_BIN= 1;
@@ -841,8 +817,6 @@ CREATE TABLE bug_45756_slave_logged_1 (a int) engine = NDB;
 INSERT INTO bug_45756_slave_logged_1 VALUES (1);
 DROP TABLE bug_45756_slave_logged_1;
 CREATE TABLE bug_45756_slave_logged_2 (a int) engine = NDB;
-Warnings:
-Warning	1296	Replica updates are only filtered in NDB when using --server-id-bits=32
 INSERT INTO bug_45756_slave_logged_2 VALUES (1);
 DROP TABLE bug_45756_slave_logged_2;
 CREATE TABLE bug_45756_slave_logged_3 (a int) engine = NDB;
@@ -856,8 +830,6 @@ DROP TABLE bug_45756_slave_not_logged_1;
 SET SQL_LOG_BIN= 1;
 SET SQL_LOG_BIN= 0;
 CREATE TABLE bug_45756_slave_not_logged_2 (a int) engine = NDB;
-Warnings:
-Warning	1296	Replica updates are only filtered in NDB when using --server-id-bits=32
 INSERT INTO bug_45756_slave_not_logged_2 VALUES (1);
 DROP TABLE bug_45756_slave_not_logged_2;
 SET SQL_LOG_BIN= 1;
@@ -868,23 +840,17 @@ DROP TABLE bug_45756_slave_not_logged_3;
 SET SQL_LOG_BIN= 1;
 *** Generating data to be replicated ***
 CREATE TABLE bug45756_master_logged_1 (a int) engine = NDB;
-Warnings:
-Warning	1296	Replica updates are only filtered in NDB when using --server-id-bits=32
 INSERT INTO bug45756_master_logged_1 VALUES (1);
 DROP TABLE bug45756_master_logged_1;
 CREATE TABLE bug45756_master_logged_2 (a int) engine = NDB;
 INSERT INTO bug45756_master_logged_2 VALUES (1);
 DROP TABLE bug45756_master_logged_2;
 CREATE TABLE bug45756_master_logged_3 (a int) engine = NDB;
-Warnings:
-Warning	1296	Replica updates are only filtered in NDB when using --server-id-bits=32
 INSERT INTO bug45756_master_logged_3 VALUES (1);
 DROP TABLE bug45756_master_logged_3;
 *** Generating changes not to be replicated ***
 SET SQL_LOG_BIN= 0;
 CREATE TABLE bug45756_master_not_logged_1 (a int) engine = NDB;
-Warnings:
-Warning	1296	Replica updates are only filtered in NDB when using --server-id-bits=32
 INSERT INTO bug45756_master_not_logged_1 VALUES (1);
 DROP TABLE bug45756_master_not_logged_1;
 SET SQL_LOG_BIN= 1;
@@ -895,8 +861,6 @@ DROP TABLE bug45756_master_not_logged_2;
 SET SQL_LOG_BIN= 1;
 SET SQL_LOG_BIN= 0;
 CREATE TABLE bug45756_master_not_logged_3 (a int) engine = NDB;
-Warnings:
-Warning	1296	Replica updates are only filtered in NDB when using --server-id-bits=32
 INSERT INTO bug45756_master_not_logged_3 VALUES (1);
 DROP TABLE bug45756_master_not_logged_3;
 SET SQL_LOG_BIN= 1;

mysql-test/suite/ndb_rpl/t/ndb_rpl_circular.cnf

@@ -0,0 +1,4 @@
+!include suite/ndb_rpl/my.cnf
+
+[mysqld]
+ndb-log-orig=1