Merge branch 'master' into rabbitmq-server-1146-full

Conflicts:
	Makefile

Author: michaelklishin

Makefile

@@ -116,6 +116,8 @@ define PROJECT_ENV
 	    {background_gc_target_interval, 60000},
 	    %% rabbitmq-server-589
 	    {proxy_protocol, false},
+	    {disk_monitor_failure_retries, 10},
+	    {disk_monitor_failure_retry_interval, 120000},
 	    %% can be stop_rabbit or give_up see rabbitmq-server-1458
 	    {vhost_restart_strategy, stop_rabbit}
 	  ]
@@ -224,6 +226,8 @@ clean-escripts:
 	    opt='--stringparam man.indent.verbatims=0' ; \
 	xsltproc --novalid $(DOCS_DIR)/examples-to-end.xsl $< > $<.tmp && \
 	xmlto -vv -o $(DOCS_DIR) $$opt man $< 2>&1 | (grep -v '^Note: Writing' || :) && \
+	awk -F"'u " '/^\.HP / { print $$1; print $$2; next; } { print; }' "$@" > "$@.tmp" && \
+	mv "$@.tmp" "$@" && \
 	test -f $@ && \
 	rm $<.tmp
 

erlang.mk

@@ -16,7 +16,7 @@
 
 ERLANG_MK_FILENAME := $(realpath $(lastword $(MAKEFILE_LIST)))
 
-ERLANG_MK_VERSION = 2.0.0-pre.2-207-g9e9b7d2
+ERLANG_MK_VERSION = 2.0.0-pre.2-220-g7a200f5
 
 # Make 3.81 and 3.82 are deprecated.
 
@@ -4203,17 +4203,16 @@ endif
 # in practice only Makefile is needed so far.
 define dep_autopatch
 	if [ -f $(DEPS_DIR)/$(1)/erlang.mk ]; then \
+		rm -rf $(DEPS_DIR)/$1/ebin/; \
 		$(call erlang,$(call dep_autopatch_appsrc.erl,$(1))); \
 		$(call dep_autopatch_erlang_mk,$(1)); \
 	elif [ -f $(DEPS_DIR)/$(1)/Makefile ]; then \
 		if [ 0 != `grep -c "include ../\w*\.mk" $(DEPS_DIR)/$(1)/Makefile` ]; then \
 			$(call dep_autopatch2,$(1)); \
-		elif [ 0 != `grep -ci rebar $(DEPS_DIR)/$(1)/Makefile` ]; then \
+		elif [ 0 != `grep -ci "^[^#].*rebar" $(DEPS_DIR)/$(1)/Makefile` ]; then \
 			$(call dep_autopatch2,$(1)); \
-		elif [ -n "`find $(DEPS_DIR)/$(1)/ -type f -name \*.mk -not -name erlang.mk -exec grep -i rebar '{}' \;`" ]; then \
+		elif [ -n "`find $(DEPS_DIR)/$(1)/ -type f -name \*.mk -not -name erlang.mk -exec grep -i "^[^#].*rebar" '{}' \;`" ]; then \
 			$(call dep_autopatch2,$(1)); \
-		else \
-			$(call erlang,$(call dep_autopatch_app.erl,$(1))); \
 		fi \
 	else \
 		if [ ! -d $(DEPS_DIR)/$(1)/src/ ]; then \
@@ -4225,6 +4224,8 @@ define dep_autopatch
 endef
 
 define dep_autopatch2
+	mv -n $(DEPS_DIR)/$1/ebin/$1.app $(DEPS_DIR)/$1/src/$1.app.src; \
+	rm -f $(DEPS_DIR)/$1/ebin/$1.app; \
 	if [ -f $(DEPS_DIR)/$1/src/$1.app.src.script ]; then \
 		$(call erlang,$(call dep_autopatch_appsrc_script.erl,$(1))); \
 	fi; \
@@ -4536,22 +4537,6 @@ define dep_autopatch_rebar.erl
 	halt()
 endef
 
-define dep_autopatch_app.erl
-	UpdateModules = fun(App) ->
-		case filelib:is_regular(App) of
-			false -> ok;
-			true ->
-				{ok, [{application, '$(1)', L0}]} = file:consult(App),
-				Mods = filelib:fold_files("$(call core_native_path,$(DEPS_DIR)/$1/src)", "\\\\.erl$$", true,
-					fun (F, Acc) -> [list_to_atom(filename:rootname(filename:basename(F)))|Acc] end, []),
-				L = lists:keystore(modules, 1, L0, {modules, Mods}),
-				ok = file:write_file(App, io_lib:format("~p.~n", [{application, '$(1)', L}]))
-		end
-	end,
-	UpdateModules("$(call core_native_path,$(DEPS_DIR)/$1/ebin/$1.app)"),
-	halt()
-endef
-
 define dep_autopatch_appsrc_script.erl
 	AppSrc = "$(call core_native_path,$(DEPS_DIR)/$1/src/$1.app.src)",
 	AppSrcScript = AppSrc ++ ".script",
@@ -4828,6 +4813,8 @@ COMPILE_FIRST_PATHS = $(addprefix src/,$(addsuffix .erl,$(COMPILE_FIRST)))
 ERLC_EXCLUDE ?=
 ERLC_EXCLUDE_PATHS = $(addprefix src/,$(addsuffix .erl,$(ERLC_EXCLUDE)))
 
+ERLC_ASN1_OPTS ?=
+
 ERLC_MIB_OPTS ?=
 COMPILE_MIB_FIRST ?=
 COMPILE_MIB_FIRST_PATHS = $(addprefix mibs/,$(addsuffix .mib,$(COMPILE_MIB_FIRST)))
@@ -4877,7 +4864,7 @@ endif
 
 ifeq ($(wildcard src/$(PROJECT_MOD).erl),)
 define app_file
-{application, $(PROJECT), [
+{application, '$(PROJECT)', [
 	{description, "$(PROJECT_DESCRIPTION)"},
 	{vsn, "$(PROJECT_VERSION)"},$(if $(IS_DEP),
 	{id$(comma)$(space)"$(1)"}$(comma))
@@ -4889,7 +4876,7 @@ define app_file
 endef
 else
 define app_file
-{application, $(PROJECT), [
+{application, '$(PROJECT)', [
 	{description, "$(PROJECT_DESCRIPTION)"},
 	{vsn, "$(PROJECT_VERSION)"},$(if $(IS_DEP),
 	{id$(comma)$(space)"$(1)"}$(comma))
@@ -4920,7 +4907,7 @@ ERL_FILES += $(addprefix src/,$(patsubst %.asn1,%.erl,$(notdir $(ASN1_FILES))))
 
 define compile_asn1
 	$(verbose) mkdir -p include/
-	$(asn1_verbose) erlc -v -I include/ -o asn1/ +noobj $(1)
+	$(asn1_verbose) erlc -v -I include/ -o asn1/ +noobj $(ERLC_ASN1_OPTS) $(1)
 	$(verbose) mv asn1/*.erl src/
 	$(verbose) mv asn1/*.hrl include/
 	$(verbose) mv asn1/*.asn1db include/
@@ -5052,7 +5039,7 @@ $(ERL_FILES) $(CORE_FILES) $(ASN1_FILES) $(MIB_FILES) $(XRL_FILES) $(YRL_FILES):
 ebin/$(PROJECT).app:: $(ERLANG_MK_TMP)/last-makefile-change
 endif
 
--include $(PROJECT).d
+include $(wildcard $(PROJECT).d)
 
 ebin/$(PROJECT).app:: ebin/
 
@@ -5277,6 +5264,7 @@ MAN_VERSION ?= $(PROJECT_VERSION)
 define asciidoc2man.erl
 try
 	[begin
+		io:format(" ADOC   ~s~n", [F]),
 		ok = asciideck:to_manpage(asciideck:parse_file(F), #{
 			compress => gzip,
 			outdir => filename:dirname(F),
@@ -5285,7 +5273,8 @@ try
 		})
 	end || F <- [$(shell echo $(addprefix $(comma)\",$(addsuffix \",$1)) | sed 's/^.//')]],
 	halt(0)
-catch _:_ ->
+catch C:E ->
+	io:format("Exception ~p:~p~nStacktrace: ~p~n", [C, E, erlang:get_stacktrace()]),
 	halt(1)
 end.
 endef
@@ -6123,6 +6112,7 @@ CT_SUITES := $(sort $(subst _SUITE.erl,,$(notdir $(call core_find,$(TEST_DIR)/,*
 endif
 endif
 CT_SUITES ?=
+CT_LOGS_DIR ?= $(CURDIR)/logs
 
 # Core targets.
 
@@ -6145,13 +6135,13 @@ CT_RUN = ct_run \
 	-noinput \
 	-pa $(CURDIR)/ebin $(DEPS_DIR)/*/ebin $(APPS_DIR)/*/ebin $(TEST_DIR) \
 	-dir $(TEST_DIR) \
-	-logdir $(CURDIR)/logs
+	-logdir $(CT_LOGS_DIR)
 
 ifeq ($(CT_SUITES),)
 ct: $(if $(IS_APP),,apps-ct)
 else
 ct: test-build $(if $(IS_APP),,apps-ct)
-	$(verbose) mkdir -p $(CURDIR)/logs/
+	$(verbose) mkdir -p $(CT_LOGS_DIR)
 	$(gen_verbose) $(CT_RUN) -sname ct_$(PROJECT) -suite $(addsuffix _SUITE,$(CT_SUITES)) $(CT_OPTS)
 endif
 
@@ -6179,14 +6169,14 @@ endif
 
 define ct_suite_target
 ct-$(1): test-build
-	$(verbose) mkdir -p $(CURDIR)/logs/
+	$(verbose) mkdir -p $(CT_LOGS_DIR)
 	$(gen_verbose) $(CT_RUN) -sname ct_$(PROJECT) -suite $(addsuffix _SUITE,$(1)) $(CT_EXTRA) $(CT_OPTS)
 endef
 
 $(foreach test,$(CT_SUITES),$(eval $(call ct_suite_target,$(test))))
 
 distclean-ct:
-	$(gen_verbose) rm -rf $(CURDIR)/logs/
+	$(gen_verbose) rm -rf $(CT_LOGS_DIR)
 
 # Copyright (c) 2013-2016, Loïc Hoguin <[email protected]>
 # This file is part of erlang.mk and subject to the terms of the ISC License.
@@ -6232,8 +6222,10 @@ define filter_opts.erl
 endef
 
 $(DIALYZER_PLT): deps app
-	$(verbose) dialyzer --build_plt --apps erts kernel stdlib $(PLT_APPS) $(OTP_DEPS) $(LOCAL_DEPS) \
-		`test -f $(ERLANG_MK_TMP)/deps.log && cat $(ERLANG_MK_TMP)/deps.log`
+	$(eval DEPS_LOG := $(shell test -f $(ERLANG_MK_TMP)/deps.log && \
+		while read p; do test -d $$p/ebin && echo $$p/ebin; done <$(ERLANG_MK_TMP)/deps.log))
+	$(verbose) dialyzer --build_plt --apps erts kernel stdlib \
+		$(PLT_APPS) $(OTP_DEPS) $(LOCAL_DEPS) $(DEPS_LOG)
 
 plt: $(DIALYZER_PLT)
 
@@ -6319,7 +6311,7 @@ escript:: escript-zip
 	$(verbose) chmod +x $(ESCRIPT_FILE)
 
 distclean-escript:
-	$(gen_verbose) rm -f $(ESCRIPT_NAME)
+	$(gen_verbose) rm -f $(ESCRIPT_FILE)
 
 # Copyright (c) 2015-2016, Loïc Hoguin <[email protected]>
 # Copyright (c) 2014, Enrique Fernandez <[email protected]>
@@ -6495,6 +6487,20 @@ build-shell-deps: $(ALL_SHELL_DEPS_DIRS)
 shell: build-shell-deps
 	$(gen_verbose) $(SHELL_ERL) -pa $(SHELL_PATHS) $(SHELL_OPTS)
 
+# Copyright (c) 2017, Jean-Sébastien Pédron <[email protected]>
+# This file is contributed to erlang.mk and subject to the terms of the ISC License.
+
+.PHONY: show-ERL_LIBS show-ERLC_OPTS show-TEST_ERLC_OPTS
+
+show-ERL_LIBS:
+	@echo $(ERL_LIBS)
+
+show-ERLC_OPTS:
+	@$(foreach opt,$(ERLC_OPTS) -pa ebin -I include,echo "$(opt)";)
+
+show-TEST_ERLC_OPTS:
+	@$(foreach opt,$(TEST_ERLC_OPTS) -pa ebin -I include,echo "$(opt)";)
+
 # Copyright (c) 2015-2016, Loïc Hoguin <[email protected]>
 # This file is part of erlang.mk and subject to the terms of the ISC License.
 

priv/schema/rabbitmq.schema

@@ -999,7 +999,7 @@ fun(Conf) ->
             [{lager_console_backend, ConsoleLevel}];
         false -> []
     end,
-    FileHandler = case cuttlefish:conf_get("log.file", Conf, false) of
+    FileHandler = case cuttlefish:conf_get("log.file", Conf, undefined) of
         false -> [];
         File  ->
             FileLevel = cuttlefish:conf_get("log.file.level", Conf, info),
@@ -1010,6 +1010,11 @@ fun(Conf) ->
                                    {level, FileLevel},
                                    {date, RotationDate},
                                    {size, RotationSize},
+                                   {formatter_config,
+                                        [date, " ", time, " ", color,
+                                         "[", severity, "] ",
+                                         {pid, ""},
+                                         " ", message, "\n"]},
                                    {count, RotationCount}]}]
     end,
     SyslogHandler = case cuttlefish:conf_get("log.syslog", Conf, false) of

scripts/rabbitmq-env.bat

@@ -417,7 +417,12 @@ set paths=
 exit /b
 
 :filter_path
-set ERL_LIBS=%ERL_LIBS%;%~dps1%~n1%~x1
+REM Ensure ERL_LIBS begins with valid path
+IF [%ERL_LIBS%] EQU [] (
+    set ERL_LIBS=%~dps1%~n1%~x1
+) else (
+    set ERL_LIBS=%ERL_LIBS%;%~dps1%~n1%~x1
+)
 exit /b
 
 :filter_paths_done

scripts/rabbitmq-server

@@ -1,4 +1,4 @@
-#!/bin/sh -e
+#!/bin/sh
 ##  The contents of this file are subject to the Mozilla Public License
 ##  Version 1.1 (the "License"); you may not use this file except in
 ##  compliance with the License. You may obtain a copy of the License
@@ -12,9 +12,11 @@
 ##  The Original Code is RabbitMQ.
 ##
 ##  The Initial Developer of the Original Code is GoPivotal, Inc.
-##  Copyright (c) 2007-2015 Pivotal Software, Inc.  All rights reserved.
+##  Copyright (c) 2007-2017 Pivotal Software, Inc.  All rights reserved.
 ##
 
+set -e
+
 # Get default settings with user overrides for (RABBITMQ_)<var_name>
 # Non-empty defaults should be set in rabbitmq-env
 . `dirname $0`/rabbitmq-env
@@ -278,21 +280,38 @@ else
     # The Erlang VM should ignore SIGINT.
     RABBITMQ_SERVER_START_ARGS="${RABBITMQ_SERVER_START_ARGS} ${RABBITMQ_IGNORE_SIGINT_FLAG}"
 
-    # Signal handlers. They all stop RabbitMQ properly (using
-    # rabbitmqctl stop). Depending on the signal, this script will exit
-    # with a non-zero error code:
+    # Signal handlers. They all stop RabbitMQ properly, using
+    # rabbitmqctl stop. This script will exit with different exit codes:
     #   SIGHUP SIGTERM SIGTSTP
-    #     They are considered a normal process termination, so the script
-    #     exits with 0.
+    #	  Exits 0 since this is considered a normal process termination.
     #   SIGINT
-    #     They are considered an abnormal process termination, the script
-    #     exits with the job exit code.
+    #     Exits 128 + $signal_number where $signal_number is 2 for SIGINT (see
+    #     http://pubs.opengroup.org/onlinepubs/009695399/utilities/kill.html).
+    #     This is considered an abnormal process termination. Normally, we
+    #     don't need to specify this exit code because the shell propagates it.
+    #     Unfortunately, the signal handler doesn't work as expected in Dash,
+    #     thus we need to explicitely restate the exit code.
     trap "stop_rabbitmq_server; exit 0" HUP TERM TSTP
-    trap "stop_rabbitmq_server" INT
+    trap "stop_rabbitmq_server; exit 130" INT
 
     start_rabbitmq_server "$@" &
+    rabbitmq_server_pid=$!
 
     # Block until RabbitMQ exits or a signal is caught.
     # Waits for last command (which is start_rabbitmq_server)
-    wait $!
+    #
+    # The "|| true" is here to work around an issue with Dash. Normally
+    # in a Bourne shell, if `wait` is interrupted by a signal, the
+    # signal handlers defined above are executed and the script
+    # terminates with the exit code of `wait` (unless the signal handler
+    # overrides that).
+    # In the case of Dash, it looks like `set -e` (set at the beginning
+    # of this script) gets precedence over signal handling. Therefore,
+    # when `wait` is interrupted, its exit code is non-zero and because
+    # of `set -e`, the script terminates immediately without running the
+    # signal handler. To work around this issue, we use "|| true" to
+    # force that statement to succeed and the signal handler to properly
+    # execute. Because the statement below has an exit code of 0, the
+    # signal handler has to restate the expected exit code.
+    wait $rabbitmq_server_pid || true
 fi

src/rabbit.erl

@@ -22,7 +22,7 @@
          stop_and_halt/0, await_startup/0, status/0, is_running/0, alarms/0,
          is_running/1, environment/0, rotate_logs/0, force_event_refresh/1,
          start_fhc/0]).
--export([start/2, stop/1]).
+-export([start/2, stop/1, prep_stop/1]).
 -export([start_apps/1, stop_apps/1]).
 -export([log_locations/0, config_files/0, decrypt_config/2]). %% for testing and mgmt-agent
 
@@ -327,7 +327,11 @@ broker_start() ->
     ToBeLoaded = Plugins ++ ?APPS,
     start_apps(ToBeLoaded),
     maybe_sd_notify(),
-    ok = log_broker_started(rabbit_plugins:strictly_plugins(rabbit_plugins:active())).
+    ok = log_broker_started(rabbit_plugins:strictly_plugins(rabbit_plugins:active())),
+    %% See rabbitmq/rabbitmq-server#1202 for details.
+    rabbit_peer_discovery:maybe_inject_randomized_delay(),
+    rabbit_peer_discovery:maybe_register(),
+    ok.
 
 %% Try to send systemd ready notification if it makes sense in the
 %% current environment. standard_error is used intentionally in all
@@ -471,6 +475,8 @@ stop() ->
     end,
     rabbit_log:info("RabbitMQ is asked to stop...~n", []),
     Apps = ?APPS ++ rabbit_plugins:active(),
+    %% this will also perform unregistration with the peer discovery backend
+    %% as needed
     stop_apps(app_utils:app_dependency_order(Apps, true)),
     rabbit_log:info("Successfully stopped RabbitMQ and its dependencies~n", []).
 
@@ -759,6 +765,10 @@ start(normal, []) ->
             Error
     end.
 
+prep_stop(State) ->
+  rabbit_peer_discovery:maybe_unregister(),
+  State.
+
 stop(_State) ->
     ok = rabbit_alarm:stop(),
     ok = case rabbit_mnesia:is_clustered() of

src/rabbit_amqqueue_process.erl

@@ -99,7 +99,7 @@
 -spec info_keys() -> rabbit_types:info_keys().
 -spec init_with_backing_queue_state
         (rabbit_types:amqqueue(), atom(), tuple(), any(),
-         [rabbit_types:delivery()], pmon:pmon(), dict:dict()) ->
+         [rabbit_types:delivery()], pmon:pmon(), gb_trees:tree()) ->
             #q{}.
 
 %%----------------------------------------------------------------------------