rabbit_peer_discovery: Allow backends to select the node to join themselves

[Why]
Before, the backend would always return a list of nodes and the
subsystem would select one based on their uptimes, the nodes they are
already clustered with, and the readiness of their database.

This works well in general but has some limitations. For instance with
the Consul backend, the discoverability of nodes depends on when each
one registered and in which order. Therefore, the node with the highest
uptime might not be the first that registers. In this case, the one that
registers first will only discover itself and boot as a standalone node.
However, the one with the highest uptime that registered after will
discover both nodes. It will then select itself as the node to join
because it has the highest uptime. In the end both nodes form distinct
clusters.

Another example is the Kubernetes backend. The current solution works
fine but it could be optimized: the backend knows we always want to join
the first node ("$node-0") regardless of the order in which they are
started because picking the first node alphabetically is fine.

Therefore we want to let the backend selects the node to join if it
wants.

[How]
The `list_nodes()` callback can now return the following term:

    {ok, {SelectedNode :: node(), NodeType}}

If the subsystem sees this return value, it will consider that the
returned node is the one to join. It will still query properties because
we want to make sure the node's database is ready before joining it.

(cherry picked from commit 3147ab7)

Author: dumbbell

deps/rabbit/src/rabbit_peer_discovery.erl

@@ -186,8 +186,22 @@ sync_desired_cluster(Backend, RetriesLeft, RetryDelay) ->
                #{domain => ?RMQLOG_DOMAIN_PEER_DISC}),
             ok;
         {ok, {DiscoveredNodes, NodeType}} ->
-            NodesAndProps = query_node_props(DiscoveredNodes),
-            case can_use_discovered_nodes(DiscoveredNodes, NodesAndProps) of
+            NodeAlreadySelected = is_atom(DiscoveredNodes),
+            NodesAndProps = case NodeAlreadySelected of
+                                true ->
+                                    ?LOG_DEBUG(
+                                       "Peer discovery: node '~ts' already "
+                                       "selected by backend",
+                                       [DiscoveredNodes],
+                                       #{domain => ?RMQLOG_DOMAIN_PEER_DISC}),
+                                    query_node_props([DiscoveredNodes]);
+                                false ->
+                                    query_node_props(DiscoveredNodes)
+                            end,
+            CanUse = (
+              NodeAlreadySelected orelse
+              can_use_discovered_nodes(DiscoveredNodes, NodesAndProps)),
+            case CanUse of
                 true ->
                     case select_node_to_join(NodesAndProps) of
                         SelectedNode when SelectedNode =/= false ->
@@ -249,8 +263,9 @@ retry_sync_desired_cluster(_Backend, 0, _RetryDelay) ->
     ok.
 
 -spec discover_cluster_nodes() -> {ok, Discovery} when
-      Discovery :: {DiscoveredNodes, NodeType},
+      Discovery :: {DiscoveredNodes | SelectedNode, NodeType},
       DiscoveredNodes :: [node()],
+      SelectedNode :: node(),
       NodeType :: rabbit_types:node_type().
 %% @doc Queries the peer discovery backend to discover nodes.
 %%
@@ -262,10 +277,11 @@ discover_cluster_nodes() ->
 
 -spec discover_cluster_nodes(Backend) -> Ret when
       Backend :: backend(),
-      Discovery :: {DiscoveredNodes, NodeType},
+      Ret :: {ok, Discovery} | {error, Reason},
+      Discovery :: {DiscoveredNodes | SelectedNode, NodeType},
       DiscoveredNodes :: [node()],
+      SelectedNode :: node(),
       NodeType :: rabbit_types:node_type(),
-      Ret :: {ok, Discovery} | {error, Reason},
       Reason :: any().
 %% @private
 
@@ -295,7 +311,7 @@ discover_cluster_nodes(Backend) ->
 
 -spec check_discovered_nodes_list_validity(DiscoveredNodes, NodeType) ->
     Ret when
-      DiscoveredNodes :: [node()],
+      DiscoveredNodes :: [node()] | node(),
       NodeType :: rabbit_types:node_type(),
       Ret :: ok.
 %% @private
@@ -310,6 +326,12 @@ check_discovered_nodes_list_validity(DiscoveredNodes, NodeType)
         [] -> ok;
         _  -> e({invalid_cluster_node_names, BadNodenames})
     end;
+check_discovered_nodes_list_validity(SelectedNode, NodeType)
+  when NodeType =:= disc orelse NodeType =:= disk orelse NodeType =:= ram ->
+    case is_atom(SelectedNode) of
+        true  -> ok;
+        false -> e({invalid_cluster_node_names, SelectedNode})
+    end;
 check_discovered_nodes_list_validity(DiscoveredNodes, BadNodeType)
   when is_list(DiscoveredNodes) ->
     e({invalid_cluster_node_type, BadNodeType}).
@@ -836,7 +858,7 @@ can_use_discovered_nodes(_DiscoveredNodes, []) ->
     false.
 
 -spec select_node_to_join(NodesAndProps) -> SelectedNode when
-      NodesAndProps :: [node_and_props()],
+      NodesAndProps :: nonempty_list(node_and_props()),
       SelectedNode :: node() | false.
 %% @doc Selects the node to join among the sorted list of nodes.
 %%
@@ -1140,10 +1162,10 @@ unlock(Backend, Data) ->
                 {Nodes :: [node()],
                  NodeType :: rabbit_types:node_type()} |
                 {ok, Nodes :: [node()]} |
-                {ok, {Nodes :: [node()],
+                {ok, {Nodes :: [node()] | node(),
                       NodeType :: rabbit_types:node_type()}} |
                 {error, Reason :: string()}) ->
-    {ok, {Nodes :: [node()], NodeType :: rabbit_types:node_type()}} |
+    {ok, {Nodes :: [node()] | node(), NodeType :: rabbit_types:node_type()}} |
     {error, Reason :: string()}.
 
 normalize(Nodes) when is_list(Nodes) ->
@@ -1154,6 +1176,9 @@ normalize({ok, Nodes}) when is_list(Nodes) ->
   {ok, {Nodes, disc}};
 normalize({ok, {Nodes, NodeType}}) when is_list(Nodes) andalso is_atom(NodeType) ->
   {ok, {Nodes, NodeType}};
+normalize({ok, {Node, NodeType}})
+  when is_atom(Node) andalso is_atom(NodeType) ->
+  {ok, {Node, NodeType}};
 normalize({error, Reason}) ->
   {error, Reason}.
 

deps/rabbit_common/src/rabbit_peer_discovery_backend.erl

@@ -41,7 +41,7 @@
 
 -callback init() -> ok | {error, Reason :: string()}.
 
--callback list_nodes() -> {ok, {Nodes :: list(), NodeType :: rabbit_types:node_type()}} |
+-callback list_nodes() -> {ok, {Nodes :: [node()] | node(), NodeType :: rabbit_types:node_type()}} |
                           {error, Reason :: string()}.
 
 -callback supports_registration() -> boolean().