WL#15713: GR default change: set group_replication_exit_state_action=OFFLINE_MODE

Change the default of group_replication_exit_state_action from
READ_ONLY to OFFLINE_MODE.

Change-Id: I89723ce8e9cb1ada406acd28410d9d2f6c31eff3

Author: nacarvalho

mysql-test/include/stop_group_replication.inc

@@ -9,6 +9,8 @@
 # --let $group_replication_group_name= UUID
 # [--let $skip_assert_read_only= 1]
 # [--let $skip_reset_read_mode= 1]
+# [--let $skip_assert_offline_mode= 1]
+# [--let $skip_reset_offline_mode= 1]
 # [--let $$wait_timeout= NUMBER]
 # [--let $rpl_debug= 1]
 # --source include/stop_group_replication.inc
@@ -25,6 +27,13 @@
 #     Skip the disabling of read_only vars
 #     See include/assert_and_disable_read_only.inc
 #
+#   $skip_assert_offline_mode
+#     Skip assert that offline_mode is enabled when the member is in ERROR state,
+#     and the consequent disable of both variables
+#
+#   $skip_reset_offline_mode
+#     Skip reset offline_mode variable.
+#
 #   $wait_timeout
 #     See include/wait_condition.inc
 #
@@ -44,6 +53,21 @@ if (!$rpl_debug)
   --disable_query_log
 }
 
+--let $_is_member_in_error_state=
+--let $_is_member_auto_rejoin_running= 0
+if (!$skip_assert_offline_mode)
+{
+  # Save member state before it changes to OFFLINE.
+  --let $_this_member_id= query_get_value(SELECT @@SERVER_UUID, @@SERVER_UUID, 1)
+  --let $_is_member_in_error_state= `SELECT member_state IN ("ERROR") FROM performance_schema.replication_group_members WHERE member_id="$_this_member_id"`
+
+  if ($_is_member_in_error_state)
+  {
+    # Check if auto-rejoin is running before STOP.
+    --let $_is_member_auto_rejoin_running= `SELECT COUNT(*) FROM performance_schema.events_stages_current WHERE EVENT_NAME LIKE '%auto-rejoin%'`
+  }
+}
+
 STOP GROUP_REPLICATION;
 
 --let $wait_condition= SELECT COUNT(*)=1 FROM performance_schema.replication_connection_status connection_status WHERE connection_status.channel_name="group_replication_applier" AND connection_status.service_state="OFF"
@@ -70,5 +94,21 @@ if (!$skip_assert_read_only)
 }
 --let $skip_assert_read_only=
 
+if (!$skip_assert_offline_mode)
+{
+  if ($_is_member_in_error_state)
+  {
+    if ($_is_member_auto_rejoin_running == 0)
+    {
+      --let $_member_exit_state_action= `SELECT @@GLOBAL.group_replication_exit_state_action`
+      if ($_member_exit_state_action == "OFFLINE_MODE")
+      {
+        --source include/assert_and_disable_offline_mode.inc
+      }
+    }
+  }
+}
+--let $skip_assert_offline_mode=
+
 --let $include_filename= stop_group_replication.inc
 --source include/end_include_file.inc

mysql-test/suite/group_replication/include/gr_exit_state_action_on_member_expel.inc

@@ -46,6 +46,7 @@ DROP TABLE pid_table;
 # Supress log errors since they are expected.
 call mtr.add_suppression("Member was expelled from the group due to network failures, changing member status to ERROR.");
 call mtr.add_suppression("The server was automatically set into read only mode after an error was detected.");
+call mtr.add_suppression("The server was automatically set into offline mode after an error was detected.");
 call mtr.add_suppression("Skipping leave operation: concurrent attempt to leave the group is on-going.");
 call mtr.add_suppression("The plugin encountered a critical error and will abort: Member was expelled from the group due to network failures.");
 call mtr.add_suppression("Error, maximum number of retries exceeded when waiting for the internal server session state to be operating");
@@ -59,6 +60,8 @@ SET SESSION sql_log_bin = 1;
 --let $local_address_server1= `SELECT @@GLOBAL.group_replication_local_address`
 --let $group_seeds_server1= `SELECT @@GLOBAL.group_replication_group_seeds`
 
+# Set the exit state action sysvar to READ_ONLY
+SET @@GLOBAL.group_replication_exit_state_action = READ_ONLY;
 
 --echo
 --echo #########################################################################

mysql-test/suite/group_replication/r/gr_abort_query_on_plugin_error.result

@@ -38,6 +38,7 @@ call mtr.add_suppression("The member is already leaving or joining a group.");
 call mtr.add_suppression("Error leaving the group");
 call mtr.add_suppression("Skipping leave operation: concurrent attempt to leave the group is on-going.");
 call mtr.add_suppression("The server was automatically set into read only mode after an error was detected.");
+call mtr.add_suppression("The server was automatically set into offline mode after an error was detected.");
 set session sql_log_bin=1;
 [connection server1]
 INSERT INTO test.t1 (b) VALUES (0);

mysql-test/suite/group_replication/r/gr_acf_error_state_handling.result

@@ -14,6 +14,7 @@ call mtr.add_suppression("Error at event handling! Got error: 1");
 call mtr.add_suppression("The member is leaving a group without being on one");
 call mtr.add_suppression("Due to a plugin error, some transactions were unable to be certified and will now rollback.");
 call mtr.add_suppression("The server was automatically set into read only mode after an error was detected.");
+call mtr.add_suppression("The server was automatically set into offline mode after an error was detected.");
 call mtr.add_suppression("Error while waiting for conflict detection procedure to finish on session .*");
 call mtr.add_suppression("Run function 'before_commit' in plugin 'group_replication' failed");
 call mtr.add_suppression("Skipping leave operation: concurrent attempt to leave the group is on-going.");
@@ -26,6 +27,7 @@ CALL mtr.add_suppression("Could not execute Write_rows event on table test.t1");
 CALL mtr.add_suppression("The applier thread execution was aborted. Unable to process more transactions, this member will now leave the group.");
 CALL mtr.add_suppression("Fatal error during execution on the Applier process of Group Replication. The server will now leave the group.");
 CALL mtr.add_suppression("The server was automatically set into read only mode after an error was detected.");
+CALL mtr.add_suppression("The server was automatically set into offline mode after an error was detected.");
 CALL mtr.add_suppression("Skipping leave operation: concurrent attempt to leave the group is on-going.");
 CALL mtr.add_suppression("The replica coordinator and worker threads are stopped");
 SET sql_log_bin=1;

mysql-test/suite/group_replication/r/gr_acf_receiver_failover_channel_non_existent_on_the_group.result

@@ -14,6 +14,7 @@ SET SESSION sql_log_bin= 0;
 call mtr.add_suppression("Unable to set SOURCE_CONNECTION_AUTO_FAILOVER on a non-existent or misconfigured replication channel 'ch1', please create the channel and rejoin the server to the group.");
 call mtr.add_suppression("A message sent through the Group Replication message deliver service was not delivered successfully. The server will now leave the group.*");
 call mtr.add_suppression("The server was automatically set into read only mode after an error was detected.");
+call mtr.add_suppression("The server was automatically set into offline mode after an error was detected.");
 call mtr.add_suppression("Skipping leave operation: concurrent attempt to leave the group is on-going.");
 SET SESSION sql_log_bin= 1;
 include/start_group_replication.inc

mysql-test/suite/group_replication/r/gr_acf_start_failover_channels_error_on_bootstrap.result

@@ -27,6 +27,7 @@ Note	1760	Storing MySQL user name or password information in the connection meta
 SET SESSION sql_log_bin = 0;
 call mtr.add_suppression("The member action \"mysql_start_failover_channels_if_primary\" for event \"AFTER_PRIMARY_ELECTION\" with priority \"10\" failed. Please check previous messages in the error log for hints about what could have caused this failure.");
 call mtr.add_suppression("The server was automatically set into read only mode after an error was detected.");
+call mtr.add_suppression("The server was automatically set into offline mode after an error was detected.");
 call mtr.add_suppression("Skipping leave operation: concurrent attempt to leave the group is on-going.");
 call mtr.add_suppression("Error while sending message. Context: primary election process.");
 SET SESSION sql_log_bin = 1;

mysql-test/suite/group_replication/r/gr_applier_and_shutdown.result

@@ -14,6 +14,7 @@ call mtr.add_suppression("Skipping leave operation: concurrent attempt to leave
 call mtr.add_suppression("Fatal error during execution on the Applier process of Group Replication. The server will now leave the group.");
 call mtr.add_suppression("Skipping leave operation: member already left the group.");
 call mtr.add_suppression("The server was automatically set into read only mode after an error was detected.");
+call mtr.add_suppression("The server was automatically set into offline mode after an error was detected.");
 call mtr.add_suppression("Error, maximum number of retries exceeded when waiting for the internal server session state to be operating");
 call mtr.add_suppression("Failed to establish an internal server connection to execute plugin operations");
 SET SESSION sql_log_bin = 1;

mysql-test/suite/group_replication/r/gr_applier_early_failure.result

@@ -54,6 +54,7 @@ call mtr.add_suppression("Unable to initialize the Group Replication applier mod
 call mtr.add_suppression("The applier thread execution was aborted. Unable to process more transactions, this member will now leave the group.");
 call mtr.add_suppression("Fatal error during execution on the Applier process of Group Replication. The server will now leave the group.");
 call mtr.add_suppression("The server was automatically set into read only mode after an error was detected.");
+call mtr.add_suppression("The server was automatically set into offline mode after an error was detected.");
 call mtr.add_suppression("Unable to confirm whether the server has left the group or not. Check *.*");
 call mtr.add_suppression("Skipping leave operation: concurrent attempt to leave the group is on-going.");
 call mtr.add_suppression("Replica SQL for channel 'group_replication_applier': Could not execute Write_rows event on table");

mysql-test/suite/group_replication/r/gr_applier_error.result

@@ -24,6 +24,7 @@ CALL mtr.add_suppression("Could not execute Write_rows event on table test.t1");
 CALL mtr.add_suppression("The applier thread execution was aborted. Unable to process more transactions, this member will now leave the group.");
 CALL mtr.add_suppression("Fatal error during execution on the Applier process of Group Replication. The server will now leave the group.");
 CALL mtr.add_suppression("The server was automatically set into read only mode after an error was detected.");
+CALL mtr.add_suppression("The server was automatically set into offline mode after an error was detected.");
 CALL mtr.add_suppression("Skipping leave operation: concurrent attempt to leave the group is on-going.");
 CALL mtr.add_suppression("The replica coordinator and worker threads are stopped");
 SET sql_log_bin=1;

mysql-test/suite/group_replication/r/gr_applier_privileged_user.result

@@ -13,6 +13,7 @@ CALL mtr.add_suppression(".*Worker .* failed executing transaction .* INSERT com
 CALL mtr.add_suppression(".*Plugin group_replication reported: 'The applier thread execution was aborted. Unable to process more transactions, this member will now leave the group.'.*");
 CALL mtr.add_suppression(".*Plugin group_replication reported: 'Fatal error during execution on the Applier process of Group Replication. The server will now leave the group.'.*");
 CALL mtr.add_suppression(".*Plugin group_replication reported: 'The server was automatically set into read only mode after an error was detected.'.*");
+CALL mtr.add_suppression("The server was automatically set into offline mode after an error was detected.");
 CALL mtr.add_suppression(".*Plugin group_replication reported: 'Skipping leave operation: concurrent attempt to leave the group is on-going.'.*");
 CALL mtr.add_suppression(".*The replica coordinator and worker threads are stopped.*");
 SET @@sql_log_bin = 1;

mysql-test/suite/group_replication/r/gr_applier_thread_error_detection.result

@@ -15,6 +15,7 @@ call mtr.add_suppression("Replica SQL for channel 'group_replication_applier': .
 call mtr.add_suppression("Fatal error during execution on the Applier process of Group Replication. The server will now leave the group.");
 call mtr.add_suppression("The member is leaving a group without being on one");
 call mtr.add_suppression("The server was automatically set into read only mode after an error was detected.");
+call mtr.add_suppression("The server was automatically set into offline mode after an error was detected.");
 call mtr.add_suppression("The member is already leaving or joining a group.");
 call mtr.add_suppression("Error leaving the group");
 call mtr.add_suppression("Skipping leave operation: concurrent attempt to leave the group is on-going.");

mysql-test/suite/group_replication/r/gr_applier_thread_failure_detection.result

@@ -43,6 +43,7 @@ call mtr.add_suppression("Replica SQL for channel 'group_replication_applier': C
 call mtr.add_suppression("The applier thread execution was aborted. Unable to process more transactions, this member will now leave the group.");
 call mtr.add_suppression("Fatal error during execution on the Applier process of Group Replication. The server will now leave the group.");
 call mtr.add_suppression("The server was automatically set into read only mode after an error was detected.");
+call mtr.add_suppression("The server was automatically set into offline mode after an error was detected.");
 call mtr.add_suppression("Replica SQL for channel 'group_replication_applier': Worker .* failed executing transaction .*");
 call mtr.add_suppression("Replica SQL for channel 'group_replication_applier': ... The replica coordinator and worker threads are stopped,.*");
 SET SESSION sql_log_bin= 1;

mysql-test/suite/group_replication/r/gr_autorejoin_failure_during_mid_join.result

@@ -10,6 +10,7 @@ call mtr.add_suppression("Member was expelled from the group due to network fail
 call mtr.add_suppression("Started auto-rejoin procedure attempt*");
 call mtr.add_suppression("Auto-rejoin procedure attempt*");
 call mtr.add_suppression("The server was automatically set into read only mode after an error was detected.");
+call mtr.add_suppression("The server was automatically set into offline mode after an error was detected.");
 call mtr.add_suppression("Unable to confirm whether the server has left the group or not. Check performance_schema.replication_group_members to check group membership information.");
 call mtr.add_suppression("Timeout while waiting for a view change event during the leave step before a auto-rejoin attempt.");
 call mtr.add_suppression("Timeout while waiting for a view change event during the auto-rejoin procedure");

mysql-test/suite/group_replication/r/gr_autorejoin_majority_loss_rejoin_failed_readonly.result

@@ -25,6 +25,8 @@ SET @debug_saved = @@GLOBAL.DEBUG;
 SET @@GLOBAL.DEBUG='+d,group_replication_rejoin_short_retry';
 SET @@GLOBAL.DEBUG='+d,group_replication_stop_before_rejoin_loop';
 SET @@GLOBAL.DEBUG='+d,group_replication_stop_before_rejoin';
+SET @saved_exit_state_action = @@GLOBAL.group_replication_exit_state_action;
+SET @@GLOBAL.group_replication_exit_state_action = READ_ONLY;
 include/start_and_bootstrap_group_replication.inc
 [connection server2]
 include/start_group_replication.inc
@@ -80,5 +82,6 @@ include/stop_group_replication.inc
 include/start_and_bootstrap_group_replication.inc
 SET @@GLOBAL.DEBUG = @debug_saved;
 SET @@GLOBAL.group_replication_autorejoin_tries = default;
+SET @@GLOBAL.group_replication_exit_state_action = @saved_exit_state_action;
 include/gr_end_autorejoin_monitoring.inc
 include/group_replication_end.inc

mysql-test/suite/group_replication/r/gr_autorejoin_member_expel_rejoin_failed_readonly.result

@@ -14,13 +14,16 @@ call mtr.add_suppression("Started auto-rejoin procedure attempt*");
 call mtr.add_suppression("Auto-rejoin procedure attempt*");
 call mtr.add_suppression("Unable to confirm whether the server has left the group or not. Check performance_schema.replication_group_members to check group membership information.");
 call mtr.add_suppression("The server was automatically set into read only mode after an error was detected.");
+call mtr.add_suppression("The server was automatically set into offline mode after an error was detected.");
 SET sql_log_bin = 1;
 include/gr_autorejoin_monitoring.inc
 SET @debug_saved = @@GLOBAL.DEBUG;
 SET @@GLOBAL.DEBUG='+d,group_replication_rejoin_short_retry';
 SET @@GLOBAL.DEBUG='+d,group_replication_fail_rejoin';
 SET @@GLOBAL.DEBUG='+d,group_replication_stop_before_rejoin_loop';
 SET @@GLOBAL.DEBUG='+d,group_replication_stop_before_rejoin';
+SET @saved_exit_state_action = @@GLOBAL.group_replication_exit_state_action;
+SET @@GLOBAL.group_replication_exit_state_action = READ_ONLY;
 include/start_and_bootstrap_group_replication.inc
 [connection server2]
 include/start_group_replication.inc
@@ -94,5 +97,6 @@ include/rpl_gr_wait_for_number_of_members.inc
 [connection server1]
 SET @@GLOBAL.DEBUG = @debug_saved;
 SET @@GLOBAL.group_replication_autorejoin_tries = default;
+SET @@GLOBAL.group_replication_exit_state_action = @saved_exit_state_action;
 include/gr_end_autorejoin_monitoring.inc
 include/group_replication_end.inc

mysql-test/suite/group_replication/r/gr_autorejoin_member_expel_rejoin_parallel_group_mode_change.result

@@ -14,6 +14,7 @@ call mtr.add_suppression("Started auto-rejoin procedure attempt*");
 call mtr.add_suppression("Auto-rejoin procedure attempt*");
 call mtr.add_suppression("The member configuration is not compatible with the group configuration. Variables such as group_replication_single_primary_mode or group_replication_enforce_update_everywhere_checks.*");
 call mtr.add_suppression("The server was automatically set into read only mode after an error was detected.");
+call mtr.add_suppression("The server was automatically set into offline mode after an error was detected.");
 SET sql_log_bin = 1;
 include/gr_autorejoin_monitoring.inc
 SET @debug_saved = @@GLOBAL.DEBUG;

mysql-test/suite/group_replication/r/gr_autorejoin_monitoring_test.result

@@ -12,6 +12,7 @@ Note	####	Storing MySQL user name or password information in the connection meta
 SET sql_log_bin = 0;
 call mtr.add_suppression("Member was expelled from the group due to network failures, changing member status to ERROR.");
 call mtr.add_suppression("The server was automatically set into read only mode after an error was detected.");
+call mtr.add_suppression("The server was automatically set into offline mode after an error was detected.");
 call mtr.add_suppression("\\[GCS\\] read failed");
 call mtr.add_suppression("Started auto-rejoin procedure attempt*");
 call mtr.add_suppression("Auto-rejoin procedure attempt*");

mysql-test/suite/group_replication/r/gr_autorejoin_option.result

@@ -12,6 +12,7 @@ Note	####	Storing MySQL user name or password information in the connection meta
 SET sql_log_bin = 0;
 call mtr.add_suppression("Member was expelled from the group due to network failures, changing member status to ERROR.");
 call mtr.add_suppression("The server was automatically set into read only mode after an error was detected.");
+call mtr.add_suppression("The server was automatically set into offline mode after an error was detected.");
 call mtr.add_suppression("\\[GCS\\] The member has left the group but the new view will not be installed*");
 call mtr.add_suppression("Started auto-rejoin procedure attempt*");
 call mtr.add_suppression("Auto-rejoin procedure attempt*");

mysql-test/suite/group_replication/r/gr_clone_applier_stop.result

@@ -115,6 +115,7 @@ call mtr.add_suppression("Fatal error during execution on the Applier process of
 call mtr.add_suppression("Skipping leave operation: concurrent attempt to leave the group is on-going.");
 call mtr.add_suppression("Unable to confirm whether the server has left the group or not. Check performance_schema.replication_group_members to check group membership information.");
 call mtr.add_suppression("The server was automatically set into read only mode after an error was detected.");
+call mtr.add_suppression("The server was automatically set into offline mode after an error was detected.");
 call mtr.add_suppression("Unable to initialize the Group Replication applier module.");
 call mtr.add_suppression("There was a previous plugin error while the member joined the group. The member will now exit the group.");
 call mtr.add_suppression("This member will start distributed recovery using clone. It is due to the number of missing transactions being higher than the configured threshold of 1.");

mysql-test/suite/group_replication/r/gr_clone_integration_ddl_error.result

@@ -79,6 +79,7 @@ call mtr.add_suppression("Internal query: CLONE INSTANCE FROM \'root\'@\'127.0.0
 call mtr.add_suppression("There was an issue when cloning from another server: Error number: 3862 Error message: Clone Donor Error: 3633 : Concurrent DDL is performed during clone operation. Please try again.");
 call mtr.add_suppression("Due to a critical cloning error or lack of donors, distributed recovery cannot be executed. The member will now leave the group.");
 call mtr.add_suppression("Skipping leave operation: concurrent attempt to leave the group is on-going.");
+call mtr.add_suppression("The server was automatically set into offline mode after an error was detected.");
 set session sql_log_bin=1;
 [connection server1]
 UNINSTALL PLUGIN clone;

mysql-test/suite/group_replication/r/gr_clone_integration_different_recovery_user.result

@@ -59,6 +59,7 @@ call mtr.add_suppression("The server was automatically set into read only mode a
 call mtr.add_suppression("Skipping leave operation: concurrent attempt to leave the group is on-going.");
 call mtr.add_suppression("Clone removing all user data for provisioning: Started");
 call mtr.add_suppression("Clone removing all user data for provisioning: Finished");
+call mtr.add_suppression("The server was automatically set into offline mode after an error was detected.");
 SET SESSION sql_log_bin=1;
 include/clean_monitoring_process.inc
 include/group_replication_end.inc