Refactor definition import to allow for arbitrary sources

The classic local filesystem source is still supported
using the same traditional configuration key, load_definitions.

Configuration schema follows peer discovery in spirit:

 * definitions.import_backend configures the mechanism to use,
   which can be a module provided by a plugin
 * definitions.* keys can be defined by plugins and contain any
   keys a specific mechanism needs

For example, the classic local filesystem source can now be
configured like this:

``` ini
definitions.import_backend = local_filesystem
definitions.local.path = /path/to/definitions.d/definition.json
```

``` ini
definitions.import_backend = https
definitions.https.url = https://hostname/path/to/definitions.json
```

HTTPS may require additional configuration keys related to TLS/x.509
peer verification. Such extra keys will be added as the need for them
becomes evident.

References #3249

(cherry picked from commit f3a5235)

Author: michaelklishin

deps/rabbit/priv/schema/rabbit.schema

@@ -121,15 +121,44 @@ end}.
 %% Definition import
 %%
 
-%% Load definitions from a JSON file or directory of files. See
+%% Original key for definition loading from a JSON file or directory of files. See
 %% https://www.rabbitmq.com/management.html#load-definitions
-%%
-%% {load_definitions, "/path/to/schema.json"},
-%% {load_definitions, "/path/to/schemas"},
 {mapping, "load_definitions", "rabbit.load_definitions",
     [{datatype, string},
      {validators, ["file_accessible"]}]}.
 
+%% Newer syntax for definition loading from a JSON file or directory of files. See
+%% https://www.rabbitmq.com/management.html#load-definitions
+{mapping, "definitions.local.path", "rabbit.definitions.local_path",
+    [{datatype, string},
+     {validators, ["file_accessible"]}]}.
+
+%% Extensive mechanism for loading definitions from a remote source
+{mapping, "definitions.import_backend", "rabbit.definitions.import_backend", [
+    {datatype, atom}
+]}.
+
+{translation, "rabbit.definitions.import_backend",
+fun(Conf) ->
+    case cuttlefish:conf_get("definitions.import_backend", Conf, rabbit_definitions_import_local_filesystem) of
+        %% short aliases for known backends
+        local_filesystem -> rabbit_definitions_import_local_filesystem;
+        local            -> rabbit_definitions_import_local_filesystem;
+        https            -> rabbit_definitions_import_https;
+        http             -> rabbit_definitions_import_https;
+        %% accept both rabbitmq_ and rabbit_ (typical core module prefix)
+        rabbitmq_definitions_import_local_filesystem -> rabbit_definitions_import_local_filesystem;
+        rabbitmq_definitions_import_local_filesystem -> rabbit_definitions_import_https;
+        %% any other value is used as is
+        Module           -> Module
+    end
+end}.
+
+%% Load definitions from a remote URL over HTTPS. See
+%% https://www.rabbitmq.com/management.html#load-definitions
+{mapping, "definitions.https.url", "rabbit.definitions.url",
+    [{datatype, string}]}.
+
 %%
 %% Security / AAA
 %% ==============

deps/rabbit/src/rabbit_definitions.erl

@@ -10,8 +10,13 @@
 
 -export([boot/0]).
 %% automatic import on boot
--export([maybe_load_definitions/0, maybe_load_definitions/2, maybe_load_definitions_from/2,
-         has_configured_definitions_to_load/0]).
+-export([
+    maybe_load_definitions/0,
+    maybe_load_definitions/2,
+    maybe_load_definitions_from/2,
+
+    has_configured_definitions_to_load/0
+]).
 %% import
 -export([import_raw/1, import_raw/2, import_parsed/1, import_parsed/2,
          apply_defs/2, apply_defs/3, apply_defs/4, apply_defs/5]).
@@ -25,7 +30,7 @@
 ]).
 -export([decode/1, decode/2, args/1]).
 
--import(rabbit_misc, [pget/2]).
+-import(rabbit_misc, [pget/2, pget/3]).
 -import(rabbit_data_coercion, [to_binary/1]).
 
 %%
@@ -59,9 +64,10 @@ boot() ->
     rabbit_sup:start_supervisor_child(definition_import_pool_sup, worker_pool_sup, [PoolSize, ?IMPORT_WORK_POOL]).
 
 maybe_load_definitions() ->
-    %% Note that management.load_definitions is handled in the plugin for backwards compatibility.
-    %% This executes the "core" version of load_definitions.
-    maybe_load_definitions(rabbit, load_definitions).
+    %% Classic source: local file or data directory
+    maybe_load_definitions_from_local_filesystem(rabbit, load_definitions),
+    %% Extensible sources
+    maybe_load_definitions_from_pluggable_source(rabbit, definitions).
 
 -spec import_raw(Body :: binary() | iolist()) -> ok | {error, term()}.
 import_raw(Body) ->
@@ -126,63 +132,76 @@ all_definitions() ->
         exchanges         => Xs
     }.
 
+-spec has_configured_definitions_to_load() -> boolean().
+has_configured_definitions_to_load() ->
+    has_configured_definitions_to_load_via_classic_option() or has_configured_definitions_to_load_via_modern_option().
+
+%% Retained for backwards compatibility, implicitly assumes the local filesystem source
+maybe_load_definitions(App, Key) ->
+    maybe_load_definitions_from_local_filesystem(App, Key).
+
+maybe_load_definitions_from(IsDir, Path) ->
+    rabbit_definitions_import_local_filesystem:load(IsDir, Path).
+
 %%
 %% Implementation
 %%
 
--spec has_configured_definitions_to_load() -> boolean().
-has_configured_definitions_to_load() ->
+-spec has_configured_definitions_to_load_via_modern_option() -> boolean().
+has_configured_definitions_to_load_via_modern_option() ->
+    case application:get_env(rabbit, definitions) of
+        undefined  -> false;
+        {ok, none} -> false;
+        {ok, []}   -> false;
+        {ok, _Options} -> true
+    end.
+
+has_configured_definitions_to_load_via_classic_option() ->
     case application:get_env(rabbit, load_definitions) of
         undefined   -> false;
         {ok, none}  -> false;
         {ok, _Path} -> true
     end.
 
-maybe_load_definitions(App, Key) ->
+maybe_load_definitions_from_local_filesystem(App, Key) ->
     case application:get_env(App, Key) of
-        undefined  ->
-            rabbit_log:debug("No definition file configured to import via load_definitions"),
-            ok;
-        {ok, none} ->
-            rabbit_log:debug("No definition file configured to import via load_definitions"),
-            ok;
-        {ok, FileOrDir} ->
-            rabbit_log:debug("Will import definitions file from load_definitions"),
-            IsDir = filelib:is_dir(FileOrDir),
-            maybe_load_definitions_from(IsDir, FileOrDir)
+        undefined  -> ok;
+        {ok, none} -> ok;
+        {ok, Path} ->
+            IsDir = filelib:is_dir(Path),
+            rabbit_definitions_import_local_filesystem:load(IsDir, Path)
     end.
 
-maybe_load_definitions_from(true, Dir) ->
-    rabbit_log:info("Applying definitions from directory ~s", [Dir]),
-    load_definitions_from_files(file:list_dir(Dir), Dir);
-maybe_load_definitions_from(false, File) ->
-    load_definitions_from_file(File).
-
-load_definitions_from_files({ok, Filenames0}, Dir) ->
-    Filenames1 = lists:sort(Filenames0),
-    Filenames2 = [filename:join(Dir, F) || F <- Filenames1],
-    load_definitions_from_filenames(Filenames2);
-load_definitions_from_files({error, E}, Dir) ->
-    rabbit_log:error("Could not read definitions from directory ~s, Error: ~p", [Dir, E]),
-    {error, {could_not_read_defs, E}}.
-
-load_definitions_from_filenames([]) ->
-    ok;
-load_definitions_from_filenames([File|Rest]) ->
-    case load_definitions_from_file(File) of
-        ok         -> load_definitions_from_filenames(Rest);
-        {error, E} -> {error, {failed_to_import_definitions, File, E}}
+maybe_load_definitions_from_pluggable_source(App, Key) ->
+    case application:get_env(App, Key) of
+        undefined  -> ok;
+        {ok, none} -> ok;
+        {ok, []}   -> ok;
+        {ok, Proplist} ->
+            case pget(import_backend, Proplist, undefined) of
+                undefined  ->
+                    {error, "definition import source is configured but definitions.import_backend is not set"};
+                ModOrAlias ->
+                    Mod = normalize_backend_module(ModOrAlias),
+                    rabbit_log:debug("Will use module ~s to import definitions", [Mod]),
+                    Mod:load(Proplist)
+            end
     end.
 
-load_definitions_from_file(File) ->
-    case file:read_file(File) of
-        {ok, Body} ->
-            rabbit_log:info("Applying definitions from file at '~s'", [File]),
-            import_raw(Body);
-        {error, E} ->
-            rabbit_log:error("Could not read definitions from file at '~s', error: ~p", [File, E]),
-            {error, {could_not_read_defs, {File, E}}}
-    end.
+normalize_backend_module(local_filesystem) ->
+    rabbit_definitions_import_local_filesystem;
+normalize_backend_module(local) ->
+    rabbit_definitions_import_local_filesystem;
+normalize_backend_module(https) ->
+    rabbit_definitions_import_https;
+normalize_backend_module(http)  ->
+    rabbit_definitions_import_https;
+normalize_backend_module(rabbitmq_definitions_import_local_filesystem) ->
+    rabbit_definitions_import_local_filesystem;
+normalize_backend_module(rabbitmq_definitions_import_https) ->
+    rabbit_definitions_import_https;
+normalize_backend_module(Other) ->
+    Other.
 
 decode(Keys, Body) ->
     case decode(Body) of

deps/rabbit/src/rabbit_definitions_import_https.erl

@@ -0,0 +1,72 @@
+%% This Source Code Form is subject to the terms of the Mozilla Public
+%% License, v. 2.0. If a copy of the MPL was not distributed with this
+%% file, You can obtain one at https://mozilla.org/MPL/2.0/.
+%%
+%% Copyright (c) 2007-2021 VMware, Inc. or its affiliates.  All rights reserved.
+%%
+
+-module(rabbit_definitions_import_https).
+-include_lib("rabbit_common/include/rabbit.hrl").
+
+-export([
+    is_enabled/0,
+    load/1
+]).
+
+
+
+-import(rabbit_misc, [pget/2]).
+-import(rabbit_data_coercion, [to_binary/1]).
+-import(rabbit_definitions, [import_raw/1]).
+
+%%
+%% API
+%%
+
+-spec is_enabled() -> boolean().
+is_enabled() ->
+    case application:get_env(rabbit, definitions) of
+        undefined   -> false;
+        {ok, none}  -> false;
+        {ok, []}    -> false;
+        {ok, Proplist} ->
+            case proplists:get_value(import_backend, Proplist, undefined) of
+                undefined -> false;
+                ?MODULE   -> true;
+                _         -> false
+            end
+    end.
+
+load(Proplist) ->
+    URL = pget(url, Proplist),
+    %% TODO
+    HTTPOptions = [],
+    load_from_url(URL, HTTPOptions).
+
+
+%%
+%% Implementation
+%%
+
+load_from_url(URL, HTTPOptions0) ->
+    inets:start(),
+    Options = [
+        {body_format, binary}
+    ],
+    HTTPOptions = HTTPOptions0 ++ [
+        {autoredirect, true}
+    ],
+    rabbit_log:info("Applying definitions from remote URL"),
+    case httpc:request(get, {URL, []}, lists:usort(HTTPOptions), Options) of
+        %% 2XX
+        {ok, {{_, Code, _}, _Headers, Body}} when Code div 100 == 2 ->
+            rabbit_log:debug("Requested definitions from remote URL '~s', response code: ~b", [URL, Code]),
+            rabbit_log:debug("Requested definitions from remote URL '~s', body: ~p", [URL, Body]),
+            import_raw(Body);
+        {ok, {{_, Code, _}, _Headers, _Body}} when Code >= 400 ->
+            rabbit_log:debug("Requested definitions from remote URL '~s', response code: ~b", [URL, Code]),
+            {error, {could_not_read_defs, {URL, rabbit_misc:format("URL request failed with response code ~b", [Code])}}};
+        {error, Reason} ->
+            rabbit_log:error("Requested definitions from remote URL '~s', error: ~p", [URL, Reason]),
+            {error, {could_not_read_defs, {URL, Reason}}}
+    end.