Merge branch 'master' into rabbitmq-server-567

Author: Daniil Fedotov

LICENSE

@@ -1,8 +1,5 @@
 This package, the RabbitMQ server is licensed under the MPL. For the
 MPL, please see LICENSE-MPL-RabbitMQ.
 
-The files `mochinum.erl' and `mochiweb_util.erl` are (c) 2007 Mochi Media, Inc and
-licensed under a MIT license, see LICENSE-MIT-Mochi.
-
 If you have any questions regarding licensing, please contact us at
 [email protected].

LICENSE-MIT-Mochi

@@ -2,7 +2,7 @@ PROJECT = rabbit
 VERSION ?= $(call get_app_version,src/$(PROJECT).app.src)
 
 DEPS = ranch lager rabbit_common rabbitmq_cli
-TEST_DEPS = rabbitmq_ct_helpers amqp_client meck proper
+TEST_DEPS = rabbitmq_ct_helpers rabbitmq_ct_client_helpers amqp_client meck proper
 
 dep_rabbitmq_cli = git_rmq rabbitmq-cli $(current_rmq_ref) $(base_rmq_ref) rabbitmq-cli-integration
 

Makefile

@@ -380,6 +380,21 @@
 ##
 # queue_index_embed_msgs_below = 4kb
 
+## Whether or not to enable background periodic GC of all
+## Erlang processes in "waiting" state.
+##
+## Disabling background GC may reduce latency for client operations,
+## keeping it enabled may reduce median RAM usage.
+##
+# background_gc_enabled = true
+
+## Target (desired) interval (in milliseconds) at which we run background GC.
+## The actual interval will vary depending on how long it takes to execute
+## the operation (can be higher than this interval). Values less than
+## 30000 milliseconds are not recommended.
+##
+# background_gc_target_interval = 60000
+
 ## ----------------------------------------------------------------------------
 ## Advanced Erlang Networking/Clustering Options.
 ##
@@ -615,7 +630,7 @@
 
 ## File rotation config. No rotation by defualt.
 ## DO NOT SET rotation date to ''. Leave unset if require "" value
-# log.file.rotation.date = $D0 
+# log.file.rotation.date = $D0
 # log.file.rotation.size = 0
 
 

docs/rabbitmq.conf.example

@@ -331,7 +331,22 @@
    %% Size in bytes below which to embed messages in the queue index. See
    %% http://www.rabbitmq.com/persistence-conf.html
    %%
-   %% {queue_index_embed_msgs_below, 4096}
+   %% {queue_index_embed_msgs_below, 4096},
+
+   %% Whether or not to enable background periodic GC of all
+   %% Erlang processes in "waiting" state.
+   %%
+   %% Disabling background GC may reduce latency for client operations,
+   %% keeping it enabled may reduce median RAM usage.
+   %%
+   %% {background_gc_enabled, true},
+   %%
+   %% Target (desired) interval (in milliseconds) at which we run background GC.
+   %% The actual interval will vary depending on how long it takes to execute
+   %% the operation (can be higher than this interval). Values less than
+   %% 30000 milliseconds are not recommended.
+   %%
+   %% {background_gc_target_interval, 60000}
 
   ]},
 

docs/rabbitmq.config.example

@@ -910,6 +910,20 @@ end}.
 {mapping, "queue_index_embed_msgs_below", "rabbit.queue_index_embed_msgs_below",
     [{datatype, bytesize}]}.
 
+%% Whether or not to enable background GC.
+%%
+%% {background_gc_enabled, true}
+
+{mapping, "background_gc_enabled", "rabbit.background_gc_enabled",
+    [{datatype, {enum, [true, false]}}]}.
+
+%% Interval (in milliseconds) at which we run background GC.
+%%
+%% {background_gc_target_interval, 60000}
+
+{mapping, "background_gc_target_interval", "rabbit.background_gc_target_interval",
+    [{datatype, integer}]}.
+
 % ==========================
 % Lager section
 % ==========================

priv/schema/rabbitmq.schema

@@ -29,6 +29,7 @@ dep_rabbitmq_clusterer                = git_rmq rabbitmq-clusterer $(current_rmq
 dep_rabbitmq_cli                      = git_rmq rabbitmq-cli $(current_rmq_ref) $(base_rmq_ref) master
 dep_rabbitmq_codegen                  = git_rmq rabbitmq-codegen $(current_rmq_ref) $(base_rmq_ref) master
 dep_rabbitmq_consistent_hash_exchange = git_rmq rabbitmq-consistent-hash-exchange $(current_rmq_ref) $(base_rmq_ref) master
+dep_rabbitmq_ct_client_helpers        = git_rmq rabbitmq-ct-client-helpers $(current_rmq_ref) $(base_rmq_ref) master
 dep_rabbitmq_ct_helpers               = git_rmq rabbitmq-ct-helpers $(current_rmq_ref) $(base_rmq_ref) master
 dep_rabbitmq_delayed_message_exchange = git_rmq rabbitmq-delayed-message-exchange $(current_rmq_ref) $(base_rmq_ref) master
 dep_rabbitmq_dotnet_client            = git_rmq rabbitmq-dotnet-client $(current_rmq_ref) $(base_rmq_ref) master
@@ -72,12 +73,16 @@ dep_toke                              = git_rmq toke $(current_rmq_ref) $(base_r
 
 dep_rabbitmq_public_umbrella          = git_rmq rabbitmq-public-umbrella $(current_rmq_ref) $(base_rmq_ref) master
 
-# FIXME: As of 2015-11-20, we depend on Ranch 1.2.1, but erlang.mk
-# defaults to Ranch 1.1.0. All projects depending indirectly on Ranch
-# needs to add "ranch" as a BUILD_DEPS. The list of projects needing
-# this workaround are:
-#     o  rabbitmq-web-stomp
-dep_ranch = git https://github.com/ninenines/ranch 1.2.1
+# Third-party dependencies version pinning.
+#
+# We do that in this file, which is copied in all projects, to ensure
+# all projects use the same versions. It avoids conflicts and makes it
+# possible to work with rabbitmq-public-umbrella.
+
+dep_cowboy_commit = 1.0.3
+dep_mochiweb = git git://github.com/basho/mochiweb.git v2.9.0p2
+dep_ranch_commit = 1.2.1
+dep_webmachine_commit = 1.10.8p2
 
 RABBITMQ_COMPONENTS = amqp_client \
 		      rabbit \
@@ -92,6 +97,7 @@ RABBITMQ_COMPONENTS = amqp_client \
 		      rabbitmq_cli \
 		      rabbitmq_codegen \
 		      rabbitmq_consistent_hash_exchange \
+		      rabbitmq_ct_client_helpers \
 		      rabbitmq_ct_helpers \
 		      rabbitmq_delayed_message_exchange \
 		      rabbitmq_dotnet_client \

rabbitmq-components.mk

@@ -44,4 +44,15 @@ SCHEMA_DIR=${SYS_PREFIX}/var/lib/rabbitmq/schema
 
 PLUGINS_DIR="${RABBITMQ_HOME}/plugins"
 
+# RABBIT_HOME can contain a version number, so default plugins
+# directory can be hard to find if we want to package some plugin
+# separately. When RABBITMQ_HOME points to a standard location where
+# it's usally being installed by package managers, we add
+# "/usr/lib/rabbitmq/plugins" to plugin search path.
+case "$RABBITMQ_HOME" in
+    /usr/lib/rabbitmq/*)
+        PLUGINS_DIR="/usr/lib/rabbitmq/plugins:$PLUGINS_DIR"
+        ;;
+esac
+
 CONF_ENV_FILE=${SYS_PREFIX}/etc/rabbitmq/rabbitmq-env.conf

scripts/rabbitmq-defaults

@@ -57,6 +57,20 @@ rmq_realpath() {
     fi
 }
 
+path_contains_existing_directory() {
+    local path="${1:?}"
+    local dir
+    local rc
+    local IFS="
+    "
+    for dir in $(echo "$path" | tr ':' '\n'); do
+        if [ -d "$dir" ]; then
+            return 0
+        fi
+    done
+    return 1
+}
+
 RABBITMQ_HOME="$(rmq_realpath "${RABBITMQ_SCRIPTS_DIR}/..")"
 ESCRIPT_DIR="${RABBITMQ_HOME}/escript"
 
@@ -108,8 +122,9 @@ fi
 rmq_normalize_path() {
     local path=$1
 
-    # Remove redundant slashes and strip a trailing slash
-    echo "$path" | sed -e 's#/\{2,\}#/#g' -e 's#/$##'
+    # Remove redundant slashes and strip a trailing slash for a
+    # PATH-like vars - ':' is the delimiter
+    echo "$path" | sed -e 's#/\{2,\}#/#g' -e 's#/$##' -e 's#/:#:#g'
 }
 
 rmq_normalize_path_var() {
@@ -265,9 +280,8 @@ if [ "${RABBITMQ_DEV_ENV}" ]; then
             RABBITMQ_ENABLED_PLUGINS_FILE="${enabled_plugins_file}"
         fi
     fi
-    
-    
-    if [ -d "${RABBITMQ_PLUGINS_DIR}" ]; then
+
+    if path_contains_existing_directory "${RABBITMQ_PLUGINS_DIR}" ; then
         # RabbitMQ was started with "make run-broker" from its own
         # source tree. Take rabbit_common from the plugins directory.
         ERL_LIBS="${RABBITMQ_PLUGINS_DIR}:${ERL_LIBS}"
@@ -291,7 +305,7 @@ if [ "${RABBITMQ_DEV_ENV}" ]; then
         ERL_LIBS="${DEPS_DIR_norm}:${ERL_LIBS}"
     fi
 else
-    if [ -d "${RABBITMQ_PLUGINS_DIR}" ]; then
+    if path_contains_existing_directory "${RABBITMQ_PLUGINS_DIR}" ; then
         # RabbitMQ was started from its install directory. Take
         # rabbit_common from the plugins directory.
         ERL_LIBS="${RABBITMQ_PLUGINS_DIR}:${ERL_LIBS}"

scripts/rabbitmq-env

@@ -25,7 +25,6 @@
          terminate/2, code_change/3]).
 
 -define(MAX_RATIO, 0.01).
--define(IDEAL_INTERVAL, 60000).
 -define(MAX_INTERVAL, 240000).
 
 -record(state, {last_interval}).
@@ -45,7 +44,9 @@ run() -> gen_server2:cast(?MODULE, run).
 
 %%----------------------------------------------------------------------------
 
-init([]) -> {ok, interval_gc(#state{last_interval = ?IDEAL_INTERVAL})}.
+init([]) ->
+    {ok, IdealInterval} = application:get_env(rabbit, background_gc_target_interval),
+    {ok, interval_gc(#state{last_interval = IdealInterval})}.
 
 handle_call(Msg, _From, State) ->
     {stop, {unexpected_call, Msg}, {unexpected_call, Msg}, State}.
@@ -65,14 +66,22 @@ terminate(_Reason, State) -> State.
 %%----------------------------------------------------------------------------
 
 interval_gc(State = #state{last_interval = LastInterval}) ->
+    {ok, IdealInterval} = application:get_env(rabbit, background_gc_target_interval),
     {ok, Interval} = rabbit_misc:interval_operation(
                        {?MODULE, gc, []},
-                       ?MAX_RATIO, ?MAX_INTERVAL, ?IDEAL_INTERVAL, LastInterval),
+                       ?MAX_RATIO, ?MAX_INTERVAL, IdealInterval, LastInterval),
     erlang:send_after(Interval, self(), run),
     State#state{last_interval = Interval}.
 
 gc() ->
-    [garbage_collect(P) || P <- processes(),
-                           {status, waiting} == process_info(P, status)],
-    garbage_collect(), %% since we will never be waiting...
+    Enabled = rabbit_misc:get_env(rabbit, background_gc_enabled, true),
+    case Enabled of
+        true ->
+            [garbage_collect(P) || P <- processes(),
+                                   {status, waiting} == process_info(P, status)],
+            %% since we will never be waiting...
+            garbage_collect();
+         false ->
+            ok
+    end,
     ok.

src/background_gc.erl

@@ -117,5 +117,7 @@
          ]},
 
          %% rabbitmq-server-973
-         {lazy_queue_explicit_gc_run_operation_threshold, 250}
+         {lazy_queue_explicit_gc_run_operation_threshold, 250},
+         {background_gc_enabled, true},
+         {background_gc_target_interval, 60000}
         ]}]}.

src/rabbit.app.src

@@ -22,7 +22,7 @@
          stop_and_halt/0, await_startup/0, status/0, is_running/0, alarms/0,
          is_running/1, environment/0, rotate_logs/0, force_event_refresh/1,
          start_fhc/0]).
--export([start/2, stop/1, prep_stop/1]).
+-export([start/2, stop/1]).
 -export([start_apps/1, stop_apps/1]).
 -export([log_locations/0, config_files/0, decrypt_config/2]). %% for testing and mgmt-agent
 
@@ -727,17 +727,15 @@ start(normal, []) ->
             Error
     end.
 
-prep_stop(_State) ->
+stop(_State) ->
     ok = rabbit_alarm:stop(),
     ok = case rabbit_mnesia:is_clustered() of
              true  -> ok;
              false -> rabbit_table:clear_ram_only_tables()
          end,
     ok.
 
-stop(_) -> ok.
-
--spec boot_error(atom(), term()) -> no_return().
+-spec boot_error(term(), not_available | [tuple()]) -> no_return().
 
 boot_error(_, {could_not_start, rabbit, {{timeout_waiting_for_tables, _}, _}}) ->
     AllNodes = rabbit_mnesia:cluster_nodes(all),