Skip to content

Update to sysmon_handler 1.1.0 #1821

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Jan 9, 2019
Merged

Update to sysmon_handler 1.1.0 #1821

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e300a0c
Update to sysmon_handler 1.1.0
lukebakken Jan 9, 2019
a3f0b3f
Debug log message
michaelklishin Jan 9, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
121 changes: 89 additions & 32 deletions priv/schema/rabbit.schema
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1363,95 +1363,152 @@ end}.
%% that are overly busy. Processes with large heaps or that take a
%% long time to garbage collect will count toward this threshold.
{mapping, "sysmon_handler.thresholds.busy_processes", "sysmon_handler.process_limit", [
{default, 30},
{datatype, integer},
hidden
{datatype, integer},
hidden
]}.

{translation, "sysmon_handler.process_limit",
fun(Conf) ->
case cuttlefish:conf_get("sysmon_handler.thresholds.busy_processes", Conf, undefined) of
undefined ->
cuttlefish:unset();
Int when is_integer(Int) ->
Int;
_ ->
cuttlefish:invalid("should be a non-negative integer")
end
end
}.

%% @doc The threshold at which to warn about the number of ports that
%% are overly busy. Ports with full input buffers count toward this
%% threshold.
{mapping, "sysmon_handler.thresholds.busy_ports", "sysmon_handler.port_limit", [
{default, 2},
{datatype, integer},
hidden
]}.

{translation, "sysmon_handler.port_limit",
fun(Conf) ->
case cuttlefish:conf_get("sysmon_handler.thresholds.busy_ports", Conf, undefined) of
undefined ->
cuttlefish:unset();
Int when is_integer(Int) ->
Int;
_ ->
cuttlefish:invalid("should be a non-negative integer")
end
end
}.

%% @doc A process will become busy when it exceeds this amount of time
%% doing garbage collection.
%%
%% NOTE: Enabling this setting can cause performance problems on
%% multi-core systems.
%% @see sysmon_handler.thresholds.busy_processes
{mapping, "sysmon_handler.triggers.process.garbage_collection", "sysmon_handler.gc_ms_limit", [
{default, off},
{datatype, [{atom, off},
{duration, ms}]},
hidden
]}.

{translation, "sysmon_handler.gc_ms_limit",
fun(Conf) ->
case cuttlefish:conf_get("sysmon_handler.triggers.process.garbage_collection", Conf) of
off -> 0;
Int -> Int
end
end}.
fun(Conf) ->
case cuttlefish:conf_get("sysmon_handler.triggers.process.garbage_collection", Conf, undefined) of
undefined ->
cuttlefish:unset();
off ->
0;
Int when is_integer(Int) ->
Int;
_ ->
cuttlefish:invalid("should be a non-negative integer")
end
end
}.

%% @doc A process will become busy when it exceeds this amount of time
%% during a single process scheduling & execution cycle.
{mapping, "sysmon_handler.triggers.process.long_scheduled_execution", "sysmon_handler.schedule_ms_limit", [
{default, off},
{datatype, [{atom, off},
{duration, ms}]},
hidden
]}.

{translation, "sysmon_handler.schedule_ms_limit",
fun(Conf) ->
case cuttlefish:conf_get("sysmon_handler.triggers.process.long_scheduled_execution", Conf) of
off -> 0;
Int -> Int
end
end}.
fun(Conf) ->
case cuttlefish:conf_get("sysmon_handler.triggers.process.long_scheduled_execution", Conf, undefined) of
undefined ->
cuttlefish:unset();
off ->
0;
Int when is_integer(Int) ->
Int;
_ ->
cuttlefish:invalid("should be a non-negative integer")
end
end
}.

%% @doc A process will become busy when its heap exceeds this size.
%% @see sysmon_handler.thresholds.busy_processes
{mapping, "sysmon_handler.triggers.process.heap_size", "sysmon_handler.heap_word_limit", [
{default, "160444000"},
{datatype, [bytesize, {atom, off}]},
{datatype, [{atom, off},
bytesize]},
hidden
]}.

{translation, "sysmon_handler.heap_word_limit",
fun(Conf) ->
case cuttlefish:conf_get("sysmon_handler.triggers.process.heap_size", Conf) of
off -> 0;
Bytes ->
WordSize = erlang:system_info(wordsize),
Bytes div WordSize
fun(Conf) ->
case cuttlefish:conf_get("sysmon_handler.triggers.process.heap_size", Conf, undefined) of
undefined ->
cuttlefish:unset();
off ->
0;
Bytes when is_integer(Bytes) ->
WordSize = erlang:system_info(wordsize),
Bytes div WordSize;
_ ->
cuttlefish:invalid("should be a non-negative integer")
end
end
end}.
}.

%% @doc Whether ports with full input buffers will be counted as
%% busy. Ports can represent open files or network sockets.
%% @see sysmon_handler.thresholds.busy_ports
{mapping, "sysmon_handler.triggers.port", "sysmon_handler.busy_port", [
{default, on},
{datatype, flag},
hidden
]}.

{translation, "sysmon_handler.busy_port",
fun(Conf) ->
case cuttlefish:conf_get("sysmon_handler.triggers.port", Conf, undefined) of
undefined ->
cuttlefish:unset();
Val -> Val
end
end
}.

%% @doc Whether distribution ports with full input buffers will be
%% counted as busy. Distribution ports connect Erlang nodes within a
%% single cluster.
%% @see sysmon_handler.thresholds.busy_ports
{mapping, "sysmon_handler.triggers.distribution_port", "sysmon_handler.busy_dist_port", [
{default, on},
{datatype, flag},
hidden
]}.

{translation, "sysmon_handler.busy_dist_port",
fun(Conf) ->
case cuttlefish:conf_get("sysmon_handler.triggers.distribution_port", Conf, undefined) of
undefined ->
cuttlefish:unset();
Val -> Val
end
end
}.

% ===============================
% Validators
% ===============================
Expand Down
2 changes: 1 addition & 1 deletion rabbitmq-components.mk
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ dep_lager = hex 3.6.5
dep_ra = git https://github.com/rabbitmq/ra.git master
dep_ranch = hex 1.7.1
dep_recon = hex 2.3.6
dep_sysmon_handler = hex 1.0.0
dep_sysmon_handler = hex 1.1.0

RABBITMQ_COMPONENTS = amqp_client \
amqp10_common \
Expand Down
26 changes: 25 additions & 1 deletion src/rabbit.erl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -523,6 +523,7 @@ start_apps(Apps) ->

start_apps(Apps, RestartTypes) ->
app_utils:load_applications(Apps),
ensure_sysmon_handler_app_config(),
ConfigEntryDecoder = case application:get_env(rabbit, config_entry_decoder) of
undefined ->
[];
Expand Down Expand Up @@ -553,7 +554,6 @@ start_apps(Apps, RestartTypes) ->
PassPhrase
},
decrypt_config(Apps, Algo),

OrderedApps = app_utils:app_dependency_order(Apps, false),
case lists:member(rabbit, Apps) of
false -> rabbit_boot_steps:run_boot_steps(Apps); %% plugin activation
Expand All @@ -563,6 +563,30 @@ start_apps(Apps, RestartTypes) ->
handle_app_error(could_not_start),
RestartTypes).

%% rabbitmq/rabbitmq-server#952
%% This function is to be called after configuration has been optionally generated
%% and the sysmon_handler application loaded, but not started. It will ensure that
%% sane defaults are used for configuration settings that haven't been set by the
%% user
ensure_sysmon_handler_app_config() ->
Defaults = [
{process_limit, 100},
{port_limit, 100},
{gc_ms_limit, 0},
{schedule_ms_limit, 0},
{heap_word_limit, 10485760},
{busy_port, false},
{busy_dist_port, true}
],
lists:foreach(fun({K, V}) ->
case application:get_env(sysmon_handler, K) of
undefined ->
application:set_env(sysmon_handler, K, V);
_ ->
ok
end
end, Defaults).

%% This function retrieves the correct IoDevice for requesting
%% input. The problem with using the default IoDevice is that
%% the Erlang shell prevents us from getting the input.
Expand Down
5 changes: 5 additions & 0 deletions src/rabbit_sysmon_handler.erl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,11 @@ handle_event({monitor, PidOrPort, Type, Info}, State=#state{timer_ref=TimerRef})
{Fmt, Args} = format_pretty_proc_or_port_info(PidOrPort),
rabbit_log:warning("~p ~w ~w " ++ Fmt ++ " ~w", [?MODULE, Type, PidOrPort] ++ Args ++ [Info]),
{ok, State#state{timer_ref=NewTimerRef}};
handle_event({suppressed, Type, Info}, State=#state{timer_ref=TimerRef}) ->
%% Reset the inactivity timeout
NewTimerRef = reset_timer(TimerRef),
rabbit_log:debug("~p encountered a suppressed event of type ~w: ~w", [?MODULE, Type, Info]),
{ok, State#state{timer_ref=NewTimerRef}};
handle_event(Event, State=#state{timer_ref=TimerRef}) ->
NewTimerRef = reset_timer(TimerRef),
rabbit_log:warning("~p unhandled event: ~p", [?MODULE, Event]),
Expand Down
Loading