Recover bindings for all durable queues including failed to recover

If a queue fails to recover it may still be restarted by the supervisor
and eventually start. After that some bindings may be in rabbit_durable_route
but not rabbit_route. This can cause binding not found errors.

If bindings are recovered for failed queues, the behaviour will be
the same as for the crashed queues. (which is currently broken
but needs to be fixed separately)

This is a commit for version 3.7.x

Addresses #1873
[#163919158]

Author: hairyhum

src/rabbit_amqqueue.erl

@@ -74,7 +74,7 @@
                            {'absent', rabbit_types:amqqueue(),absent_reason()}.
 -type not_found_or_absent() ::
         'not_found' | {'absent', rabbit_types:amqqueue(), absent_reason()}.
--spec recover(rabbit_types:vhost()) -> [rabbit_types:amqqueue()].
+-spec recover(rabbit_types:vhost()) -> {[rabbit_types:amqqueue()], [rabbit_types:amqqueue()]}.
 -spec stop(rabbit_types:vhost()) -> 'ok'.
 -spec start([rabbit_types:amqqueue()]) -> 'ok'.
 -spec declare
@@ -237,7 +237,11 @@ recover(VHost) ->
         BQ:start(VHost, [QName || #amqqueue{name = QName} <- Queues]),
     case rabbit_amqqueue_sup_sup:start_for_vhost(VHost) of
         {ok, _}         ->
-            recover_durable_queues(lists:zip(Queues, OrderedRecoveryTerms));
+            Recovered = recover_durable_queues(lists:zip(Queues, OrderedRecoveryTerms)),
+            RecoveredNames = [Name || #amqqueue{name = Name} <- Recovered],
+            FailedQueues = [Q || Q = #amqqueue{name = Name} <- Queues,
+                              not lists:member(Name, RecoveredNames)],
+            {Recovered, FailedQueues};
         {error, Reason} ->
             rabbit_log:error("Failed to start queue supervisor for vhost '~s': ~s", [VHost, Reason]),
             throw({error, Reason})

src/rabbit_vhost.erl

@@ -71,10 +71,11 @@ recover(VHost) ->
     VHostStubFile = filename:join(VHostDir, ".vhost"),
     ok = rabbit_file:ensure_dir(VHostStubFile),
     ok = file:write_file(VHostStubFile, VHost),
-    Qs = rabbit_amqqueue:recover(VHost),
-    ok = rabbit_binding:recover(rabbit_exchange:recover(VHost),
-                                [QName || #amqqueue{name = QName} <- Qs]),
-    ok = rabbit_amqqueue:start(Qs),
+    {RecoveredQs, FailedQs} = rabbit_amqqueue:recover(VHost),
+    ok = rabbit_binding:recover(
+            rabbit_exchange:recover(VHost),
+            [QName || #amqqueue{name = QName} <- RecoveredQs ++ FailedQs]),
+    ok = rabbit_amqqueue:start(RecoveredQs),
     %% Start queue mirrors.
     ok = rabbit_mirror_queue_misc:on_vhost_up(VHost),
     ok.

test/backing_queue_SUITE.erl

@@ -730,7 +730,8 @@ bq_queue_recover1(Config) ->
     after 10000 -> exit(timeout_waiting_for_queue_death)
     end,
     rabbit_amqqueue:stop(?VHOST),
-    rabbit_amqqueue:start(rabbit_amqqueue:recover(?VHOST)),
+    {Ok, []} = rabbit_amqqueue:recover(?VHOST),
+    rabbit_amqqueue:start(Ok),
     {ok, Limiter} = rabbit_limiter:start_link(no_id),
     rabbit_amqqueue:with_or_die(
       QName,