Merge pull request #11926 from rabbitmq/mergify/bp/v3.13.x/pr-11924

Reduce Khepri test flakes (backport #11917) (backport #11924)

Author: michaelklishin

deps/rabbitmq_ct_helpers/BUILD.bazel

@@ -45,6 +45,7 @@ rabbitmq_app(
         "//deps/rabbit_common:erlang_app",
         "@meck//:erlang_app",
         "@proper//:erlang_app",
+        "@ra//:erlang_app",
     ],
 )
 

deps/rabbitmq_ct_helpers/src/rabbit_ct_broker_helpers.erl

@@ -173,7 +173,8 @@
     test_writer/1,
     user/1,
 
-    configured_metadata_store/1
+    configured_metadata_store/1,
+    await_metadata_store_consistent/2
   ]).
 
 %% Internal functions exported to be used by rpc:call/4.
@@ -990,6 +991,30 @@ enable_khepri_metadata_store(Config, FFs0) ->
                         end
                 end, Config, FFs).
 
+%% Waits until the metadata store replica on Node is up to date with the leader.
+await_metadata_store_consistent(Config, Node) ->
+    case configured_metadata_store(Config) of
+        mnesia ->
+            ok;
+        {khepri, _} ->
+            RaClusterName = rabbit_khepri:get_ra_cluster_name(),
+            Leader = rpc(Config, Node, ra_leaderboard, lookup_leader, [RaClusterName]),
+            LastAppliedLeader = ra_last_applied(Leader),
+
+            NodeName = get_node_config(Config, Node, nodename),
+            ServerId = {RaClusterName, NodeName},
+            rabbit_ct_helpers:eventually(
+              ?_assert(
+                 begin
+                     LastApplied = ra_last_applied(ServerId),
+                     is_integer(LastApplied) andalso LastApplied >= LastAppliedLeader
+                 end))
+    end.
+
+ra_last_applied(ServerId) ->
+    #{last_applied := LastApplied} = ra:key_metrics(ServerId),
+    LastApplied.
+
 rewrite_node_config_file(Config, Node) ->
     NodeConfig = get_node_config(Config, Node),
     I = if

deps/rabbitmq_mqtt/test/shared_SUITE.erl

@@ -28,7 +28,8 @@
          get_node_config/3,
          drain_node/2,
          revive_node/2,
-         is_feature_flag_enabled/2
+         is_feature_flag_enabled/2,
+         await_metadata_store_consistent/2
         ]).
 -import(rabbit_ct_helpers,
         [eventually/3,
@@ -1245,6 +1246,7 @@ rabbit_mqtt_qos0_queue_kill_node(Config) ->
     SubscriberId = <<"subscriber">>,
     Sub0 = connect(SubscriberId, Config, 0, []),
     {ok, _, [0]} = emqtt:subscribe(Sub0, Topic1, qos0),
+    ok = await_metadata_store_consistent(Config, 2),
     ok = emqtt:publish(Pub, Topic1, <<"m0">>, qos0),
     ok = expect_publishes(Sub0, Topic1, [<<"m0">>]),