|
| 1 | +%% This Source Code Form is subject to the terms of the Mozilla Public |
| 2 | +%% License, v. 2.0. If a copy of the MPL was not distributed with this |
| 3 | +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. |
| 4 | +%% |
| 5 | +%% Copyright (c) 2018-2023 VMware, Inc. or its affiliates. All rights reserved. |
| 6 | + |
| 7 | +-module(member_evaluation_SUITE). |
| 8 | + |
| 9 | +-include_lib("common_test/include/ct.hrl"). |
| 10 | +-include_lib("eunit/include/eunit.hrl"). |
| 11 | +-include_lib("amqp_client/include/amqp_client.hrl"). |
| 12 | +-include_lib("rabbitmq_ct_helpers/include/rabbit_assert.hrl"). |
| 13 | + |
| 14 | +-compile([nowarn_export_all, export_all]). |
| 15 | + |
| 16 | + |
| 17 | +all() -> |
| 18 | + [ |
| 19 | + {group, unclustered} |
| 20 | + ]. |
| 21 | + |
| 22 | +groups() -> |
| 23 | + [ |
| 24 | + {unclustered, [], |
| 25 | + [ |
| 26 | + {quorum_queue_3, [], [auto_grow, auto_shrink]} |
| 27 | + ]} |
| 28 | + ]. |
| 29 | + |
| 30 | +%% ------------------------------------------------------------------- |
| 31 | +%% Testsuite setup/teardown. |
| 32 | +%% ------------------------------------------------------------------- |
| 33 | + |
| 34 | +init_per_suite(Config0) -> |
| 35 | + rabbit_ct_helpers:log_environment(), |
| 36 | + Config1 = rabbit_ct_helpers:merge_app_env( |
| 37 | + Config0, {rabbit, [{quorum_tick_interval, 1000}, |
| 38 | + {member_eval_default_timeout, 5000}, |
| 39 | + {member_eval_short_timeout, 2000}, |
| 40 | + {member_eval_target_group_size, 3}]}), |
| 41 | + rabbit_ct_helpers:run_setup_steps(Config1, []). |
| 42 | + |
| 43 | +end_per_suite(Config) -> |
| 44 | + rabbit_ct_helpers:run_teardown_steps(Config). |
| 45 | +init_per_group(unclustered, Config) -> |
| 46 | + rabbit_ct_helpers:set_config(Config, [{rmq_nodes_clustered, false}]); |
| 47 | +init_per_group(Group, Config) -> |
| 48 | + ClusterSize = 3, |
| 49 | + Config1 = rabbit_ct_helpers:set_config(Config, |
| 50 | + [{rmq_nodes_count, ClusterSize}, |
| 51 | + {rmq_nodename_suffix, Group}, |
| 52 | + {tcp_ports_base}]), |
| 53 | + Config1b = rabbit_ct_helpers:set_config(Config1, [{net_ticktime, 10}]), |
| 54 | + rabbit_ct_helpers:run_steps(Config1b, |
| 55 | + [fun merge_app_env/1 ] ++ |
| 56 | + rabbit_ct_broker_helpers:setup_steps()). |
| 57 | + |
| 58 | +end_per_group(unclustered, Config) -> |
| 59 | + Config; |
| 60 | +end_per_group(_, Config) -> |
| 61 | + rabbit_ct_helpers:run_steps(Config, |
| 62 | + rabbit_ct_broker_helpers:teardown_steps()). |
| 63 | + |
| 64 | +init_per_testcase(Testcase, Config) -> |
| 65 | + Config1 = rabbit_ct_helpers:testcase_started(Config, Testcase), |
| 66 | + rabbit_ct_broker_helpers:rpc(Config, 0, ?MODULE, delete_queues, []), |
| 67 | + Q = rabbit_data_coercion:to_binary(Testcase), |
| 68 | + Config2 = rabbit_ct_helpers:set_config(Config1, |
| 69 | + [{queue_name, Q}, |
| 70 | + {alt_queue_name, <<Q/binary, "_alt">>}, |
| 71 | + {alt_2_queue_name, <<Q/binary, "_alt_2">>} |
| 72 | + ]), |
| 73 | + rabbit_ct_helpers:run_steps(Config2, rabbit_ct_client_helpers:setup_steps()). |
| 74 | + |
| 75 | +merge_app_env(Config) -> |
| 76 | + rabbit_ct_helpers:merge_app_env( |
| 77 | + rabbit_ct_helpers:merge_app_env(Config, |
| 78 | + {rabbit, [{core_metrics_gc_interval, 100}]}), |
| 79 | + {ra, [{min_wal_roll_over_interval, 30000}]}). |
| 80 | + |
| 81 | +end_per_testcase(Testcase, Config) -> |
| 82 | + [Server0, Server1, Server2] = |
| 83 | + rabbit_ct_broker_helpers:get_node_configs(Config, nodename), |
| 84 | + reset_nodes([Server1, Server2], Server0), |
| 85 | + Config1 = rabbit_ct_helpers:run_steps( |
| 86 | + Config, |
| 87 | + rabbit_ct_client_helpers:teardown_steps()), |
| 88 | + rabbit_ct_helpers:testcase_finished(Config1, Testcase). |
| 89 | + |
| 90 | +reset_nodes([], _Leader) -> |
| 91 | + ok; |
| 92 | +reset_nodes([Node| Nodes], Leader) -> |
| 93 | + ok = rabbit_control_helper:command(stop_app, Node), |
| 94 | + ok = rabbit_control_helper:command(forget_cluster_node, Leader, [atom_to_list(Node)]), |
| 95 | + ok = rabbit_control_helper:command(reset, Node), |
| 96 | + ok = rabbit_control_helper:command(start_app, Node), |
| 97 | + reset_nodes(Nodes, Leader). |
| 98 | + |
| 99 | + |
| 100 | +%% ------------------------------------------------------------------- |
| 101 | +%% Testcases. |
| 102 | +%% ------------------------------------------------------------------- |
| 103 | + |
| 104 | +auto_grow(Config) -> |
| 105 | + [Server0, Server1, Server2] = |
| 106 | + rabbit_ct_broker_helpers:get_node_configs(Config, nodename), |
| 107 | + Ch = rabbit_ct_client_helpers:open_channel(Config, Server0), |
| 108 | + |
| 109 | + QQ = ?config(queue_name, Config), |
| 110 | + ?assertEqual({'queue.declare_ok', QQ, 0, 0}, |
| 111 | + declare(Ch, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), |
| 112 | + |
| 113 | + %% There is only one node in the cluster at the moment |
| 114 | + {ok, Members, _} = ra:members({quorum_queue_utils:ra_name(QQ), Server0}), |
| 115 | + ?assertEqual(1, length(Members)), |
| 116 | + |
| 117 | + add_server_to_cluster(Server1, Server0), |
| 118 | + %% With 2 nodes in the cluster, target group size is not reached, so no |
| 119 | + %% new members should be available. We sleep a while so the periodic check |
| 120 | + %% runs |
| 121 | + timer:sleep(4000), |
| 122 | + {ok, Members, _} = ra:members({quorum_queue_utils:ra_name(QQ), Server0}), |
| 123 | + ?assertEqual(1, length(Members)), |
| 124 | + |
| 125 | + add_server_to_cluster(Server2, Server0), |
| 126 | + %% With 3 nodes in the cluster, target size is met so eventually it should |
| 127 | + %% be 3 members |
| 128 | + wait_until(fun() -> |
| 129 | + {ok, M, _} = ra:members({quorum_queue_utils:ra_name(QQ), Server0}), |
| 130 | + 3 =:= length(M) |
| 131 | + end). |
| 132 | + |
| 133 | +auto_shrink(Config) -> |
| 134 | + [Server0, Server1, Server2] = |
| 135 | + rabbit_ct_broker_helpers:get_node_configs(Config, nodename), |
| 136 | + Ch = rabbit_ct_client_helpers:open_channel(Config, Server0), |
| 137 | + add_server_to_cluster(Server1, Server0), |
| 138 | + add_server_to_cluster(Server2, Server0), |
| 139 | + |
| 140 | + QQ = ?config(queue_name, Config), |
| 141 | + ?assertEqual({'queue.declare_ok', QQ, 0, 0}, |
| 142 | + declare(Ch, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), |
| 143 | + |
| 144 | + wait_until(fun() -> |
| 145 | + {ok, M, _} = ra:members({quorum_queue_utils:ra_name(QQ), |
| 146 | + Server0}), |
| 147 | + 3 =:= length(M) |
| 148 | + end), |
| 149 | + ok = rabbit_control_helper:command(stop_app, Server2), |
| 150 | + ok = rabbit_ct_broker_helpers:rpc(Config, 0, rabbit_db_cluster, forget_member, |
| 151 | + [Server2, false]), |
| 152 | + %% with one node 'forgotten', eventually the membership will shrink to 2 |
| 153 | + wait_until(fun() -> |
| 154 | + {ok, M, _} = ra:members({quorum_queue_utils:ra_name(QQ), |
| 155 | + Server0}), |
| 156 | + 2 =:= length(M) |
| 157 | + end). |
| 158 | + |
| 159 | + |
| 160 | + |
| 161 | +add_server_to_cluster(Server, Leader) -> |
| 162 | + ok = rabbit_control_helper:command(stop_app, Server), |
| 163 | + ok = rabbit_control_helper:command(join_cluster, Server, [atom_to_list(Leader)], []), |
| 164 | + rabbit_control_helper:command(start_app, Server). |
| 165 | + |
| 166 | +declare(Ch, Q) -> |
| 167 | + declare(Ch, Q, []). |
| 168 | + |
| 169 | +declare(Ch, Q, Args) -> |
| 170 | + amqp_channel:call(Ch, #'queue.declare'{queue = Q, |
| 171 | + durable = true, |
| 172 | + auto_delete = false, |
| 173 | + arguments = Args}). |
| 174 | + |
| 175 | +wait_until(Condition) -> |
| 176 | + wait_until(Condition, 60). |
| 177 | + |
| 178 | +wait_until(Condition, 0) -> |
| 179 | + ?assertEqual(true, Condition()); |
| 180 | +wait_until(Condition, N) -> |
| 181 | + case Condition() of |
| 182 | + true -> |
| 183 | + ok; |
| 184 | + _ -> |
| 185 | + timer:sleep(500), |
| 186 | + wait_until(Condition, N - 1) |
| 187 | + end. |
| 188 | + |
| 189 | + |
| 190 | +delete_queues() -> |
| 191 | + [rabbit_amqqueue:delete(Q, false, false, <<"dummy">>) |
| 192 | + || Q <- rabbit_amqqueue:list()]. |
0 commit comments