Skip to content

Use system process rss memory when reporting vm memory #1259

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Jun 19, 2017
+125 −15
Merged

Use system process rss memory when reporting vm memory #1259

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
a83efdf
Use system process rss memory when reporting vm memory
Jun 12, 2017
6a84ef4
Current process system memory functions for various OSes.
Jun 13, 2017
b6efa2d
Use OS memory total when reporting memory usage
Jun 14, 2017
bc0309d
Change vm_memory_use_process_rss config to vm_memory_calculation_stra…
Jun 14, 2017
a0151e4
Move function that reports total used memory
Jun 15, 2017
bbc69fe
Merge branch 'stable' into rabbitmq-server-1223
michaelklishin Jun 19, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ define PROJECT_ENV
{ssl_options, []},
{vm_memory_high_watermark, 0.4},
{vm_memory_high_watermark_paging_ratio, 0.5},
{vm_memory_calculation_strategy, rss},
{memory_monitor_interval, 2500},
{disk_free_limit, 50000000}, %% 50MB
{msg_store_index_module, rabbit_msg_store_ets_index},
Expand Down
126 changes: 117 additions & 9 deletions src/rabbit_vm.erl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@

-module(rabbit_vm).

-export([memory/0, binary/0, ets_tables_memory/1]).
-export([memory/0, total_memory/0, binary/0, ets_tables_memory/1]).

-define(MAGIC_PLUGINS, ["cowboy", "sockjs", "rfc4627_jsonrpc"]).

Expand All @@ -30,7 +30,6 @@

%%----------------------------------------------------------------------------

%% Like erlang:memory(), but with awareness of rabbit-y things
memory() ->
All = interesting_sups(),
{Sums, _Other} = sum_processes(
Expand All @@ -41,7 +40,7 @@ memory() ->
[aggregate(Names, Sums, memory, fun (X) -> X end)
|| Names <- distinguished_interesting_sups()],

Mnesia = mnesia_memory(),
MnesiaETS = mnesia_memory(),
MsgIndexETS = ets_memory([msg_store_persistent, msg_store_transient]),
MetricsETS = ets_memory([rabbit_metrics]),
MetricsProc = try
Expand All @@ -52,8 +51,10 @@ memory() ->
0
end,
MgmtDbETS = ets_memory([rabbit_mgmt_storage]),
OsTotal = total_memory(),

[{total, Total},

[{total, ErlangTotal},
{processes, Processes},
{ets, ETS},
{atom, Atom},
Expand All @@ -66,30 +67,137 @@ memory() ->
- ConnsReader - ConnsWriter - ConnsChannel - ConnsOther
- Qs - QsSlave - MsgIndexProc - Plugins - MgmtDbProc - MetricsProc,

[{total, Total},
[
%% Connections
{connection_readers, ConnsReader},
{connection_writers, ConnsWriter},
{connection_channels, ConnsChannel},
{connection_other, ConnsOther},

%% Queues
{queue_procs, Qs},
{queue_slave_procs, QsSlave},

%% Processes
{plugins, Plugins},
{other_proc, lists:max([0, OtherProc])}, %% [1]
{mnesia, Mnesia},

%% Metrics
{metrics, MetricsETS + MetricsProc},
{mgmt_db, MgmtDbETS + MgmtDbProc},
{msg_index, MsgIndexETS + MsgIndexProc},
{other_ets, ETS - Mnesia - MsgIndexETS - MgmtDbETS},

%% ETS
{mnesia, MnesiaETS},
{other_ets, ETS - MnesiaETS - MetricsETS - MgmtDbETS - MsgIndexETS},

%% Messages (mostly, some binaries are not messages)
{binary, Bin},
{msg_index, MsgIndexETS + MsgIndexProc},

%% System
{code, Code},
{atom, Atom},
{other_system, System - ETS - Atom - Bin - Code}].
{other_system, System - ETS - Bin - Code - Atom + (OsTotal - ErlangTotal)},

{total, OsTotal}
].
%% [1] - erlang:memory(processes) can be less than the sum of its
%% parts. Rather than display something nonsensical, just silence any
%% claims about negative memory. See
%% http://erlang.org/pipermail/erlang-questions/2012-September/069320.html

%% Memory reported by erlang:memory(total) is not supposed to
%% be equal to the total size of all pages mapped to the emulator,
%% according to http://erlang.org/doc/man/erlang.html#memory-0
%% erlang:memory(total) under-reports memory usage by around 20%
-spec total_memory() -> Bytes :: integer().
total_memory() ->
case get_memory_calculation_strategy() of
rss ->
case get_system_process_resident_memory() of
{ok, MemInBytes} ->
MemInBytes;
{error, Reason} ->
rabbit_log:debug("Unable to get system memory used. Reason: ~p."
" Falling back to erlang memory reporting",
[Reason]),
erlang:memory(total)
end;
erlang ->
erlang:memory(total)
end.

-spec get_memory_calculation_strategy() -> rss | erlang.
get_memory_calculation_strategy() ->
case application:get_env(rabbit, vm_memory_calculation_strategy, rss) of
erlang ->
erlang;
rss ->
rss;
UnsupportedValue ->
rabbit_log:warning(
"Unsupported value '~p' for vm_memory_calculation_strategy. "
"Supported values: (rss|erlang). "
"Defaulting to 'rss'",
[UnsupportedValue]
),
rss
end.

-spec get_system_process_resident_memory() -> {ok, Bytes :: integer()} | {error, term()}.
get_system_process_resident_memory() ->
try
get_system_process_resident_memory(os:type())
catch _:Error ->
{error, {"Failed to get process resident memory", Error}}
end.

get_system_process_resident_memory({unix,darwin}) ->
get_ps_memory();

get_system_process_resident_memory({unix, linux}) ->
get_ps_memory();

get_system_process_resident_memory({unix,freebsd}) ->
get_ps_memory();

get_system_process_resident_memory({unix,openbsd}) ->
get_ps_memory();

get_system_process_resident_memory({win32,_OSname}) ->
OsPid = os:getpid(),
Cmd = " tasklist /fi \"pid eq " ++ OsPid ++ "\" /fo LIST 2>&1 ",
CmdOutput = os:cmd(Cmd),
%% Memory usage is displayed in kilobytes
%% with comma-separated thousands
case re:run(CmdOutput, "Mem Usage:\\s+([0-9,]+)\\s+K", [{capture, all_but_first, list}]) of
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The output of tasklist is localized, thus this function can only work on an English Windows.

Here is a sample output from a French Windows:

C:\Users\Jean-Sébastien>tasklist /fi "pid eq 7116" /fo LIST

Nom de l’image:      explorer.exe
PID:                 7116
Nom de la session:   Console
Numéro de session:   1
Utilisation mémoire: 72 008 Ko

C:\Users\Jean-Sébastien>

Thus:

  • the label is translated ("Utilisation mémoire")
  • the number is localized (a space to separate thousands).

Perhaps there is a way to set the locale for a particular command, but I don't know how.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is it acceptable to fallback to Erlang memory calculation for RabbitMQ installations on French Windows?

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@dumbbell good find. I'll look into this today.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I managed to find this but no details about how that can be adapted for one-off commands.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Apparently chcp does not change local-specific formatting of numbers/dates/etc. So it's not an option here.

{match, [Match]} ->
NoCommas = [ N || N <- Match, N =/= $, ],
{ok, list_to_integer(NoCommas) * 1024};
_ ->
{error, {unexpected_output_from_command, Cmd, CmdOutput}}
end;

get_system_process_resident_memory({unix, sunos}) ->
get_ps_memory();

get_system_process_resident_memory({unix, aix}) ->
get_ps_memory();

get_system_process_resident_memory(_OsType) ->
{error, not_implemented_for_os}.

get_ps_memory() ->
OsPid = os:getpid(),
Cmd = "ps -p " ++ OsPid ++ " -o rss=",
CmdOutput = os:cmd(Cmd),
case re:run(CmdOutput, "[0-9]+", [{capture, first, list}]) of
{match, [Match]} ->
{ok, list_to_integer(Match) * 1024};
_ ->
{error, {unexpected_output_from_command, Cmd, CmdOutput}}
end.

binary() ->
All = interesting_sups(),
{Sums, Rest} =
Expand Down
13 changes: 7 additions & 6 deletions src/vm_memory_monitor.erl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,14 +117,14 @@ get_memory_limit() ->

get_memory_use(bytes) ->
MemoryLimit = get_memory_limit(),
{erlang:memory(total), case MemoryLimit > 0.0 of
true -> MemoryLimit;
false -> infinity
end};
{rabbit_vm:total_memory(), case MemoryLimit > 0.0 of
true -> MemoryLimit;
false -> infinity
end};
get_memory_use(ratio) ->
MemoryLimit = get_memory_limit(),
case MemoryLimit > 0.0 of
true -> erlang:memory(total) / MemoryLimit;
true -> rabbit_vm:total_memory() / MemoryLimit;
false -> infinity
end.

Expand Down Expand Up @@ -268,7 +268,7 @@ parse_mem_limit(_) ->
internal_update(State = #state { memory_limit = MemLimit,
alarmed = Alarmed,
alarm_funs = {AlarmSet, AlarmClear} }) ->
MemUsed = erlang:memory(total),
MemUsed = rabbit_vm:total_memory(),
NewAlarmed = MemUsed > MemLimit,
case {Alarmed, NewAlarmed} of
{false, true} -> emit_update_info(set, MemUsed, MemLimit),
Expand Down Expand Up @@ -365,6 +365,7 @@ get_total_memory({unix, aix}) ->
get_total_memory(_OsType) ->
unknown.


%% A line looks like "Foo bar: 123456."
parse_line_mach(Line) ->
[Name, RHS | _Rest] = string:tokens(Line, ":"),
Expand Down