Skip to content

Commit ce98122

Browse files
dumbbelldumbbell
committed
Merge branch 'stable'
2 parents 72415b1 + c253b42 commit ce98122

File tree

3 files changed

+117
-111
lines changed

3 files changed

+117
-111
lines changed

‎src/rabbit_vm.erl

Lines changed: 2 additions & 94 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616

1717
-module(rabbit_vm).
1818

19-
-export([memory/0, total_memory/0, binary/0, ets_tables_memory/1]).
19+
-export([memory/0, binary/0, ets_tables_memory/1]).
2020

2121
-define(MAGIC_PLUGINS, ["cowboy", "ranch", "sockjs"]).
2222

@@ -52,7 +52,7 @@ memory() ->
5252
0
5353
end,
5454
MgmtDbETS = ets_memory([rabbit_mgmt_storage]),
55-
OsTotal = total_memory(),
55+
OsTotal = vm_memory_monitor:get_process_memory(),
5656

5757
[{total, ErlangTotal},
5858
{processes, Processes},
@@ -106,98 +106,6 @@ memory() ->
106106
%% claims about negative memory. See
107107
%% http://erlang.org/pipermail/erlang-questions/2012-September/069320.html
108108

109-
%% Memory reported by erlang:memory(total) is not supposed to
110-
%% be equal to the total size of all pages mapped to the emulator,
111-
%% according to http://erlang.org/doc/man/erlang.html#memory-0
112-
%% erlang:memory(total) under-reports memory usage by around 20%
113-
-spec total_memory() -> Bytes :: integer().
114-
total_memory() ->
115-
case get_memory_calculation_strategy() of
116-
rss ->
117-
case get_system_process_resident_memory() of
118-
{ok, MemInBytes} ->
119-
MemInBytes;
120-
{error, Reason} ->
121-
rabbit_log:debug("Unable to get system memory used. Reason: ~p."
122-
" Falling back to erlang memory reporting",
123-
[Reason]),
124-
erlang:memory(total)
125-
end;
126-
erlang ->
127-
erlang:memory(total)
128-
end.
129-
130-
-spec get_memory_calculation_strategy() -> rss | erlang.
131-
get_memory_calculation_strategy() ->
132-
case application:get_env(rabbit, vm_memory_calculation_strategy, rss) of
133-
erlang ->
134-
erlang;
135-
rss ->
136-
rss;
137-
UnsupportedValue ->
138-
rabbit_log:warning(
139-
"Unsupported value '~p' for vm_memory_calculation_strategy. "
140-
"Supported values: (rss|erlang). "
141-
"Defaulting to 'rss'",
142-
[UnsupportedValue]
143-
),
144-
rss
145-
end.
146-
147-
-spec get_system_process_resident_memory() -> {ok, Bytes :: integer()} | {error, term()}.
148-
get_system_process_resident_memory() ->
149-
try
150-
get_system_process_resident_memory(os:type())
151-
catch _:Error ->
152-
{error, {"Failed to get process resident memory", Error}}
153-
end.
154-
155-
get_system_process_resident_memory({unix,darwin}) ->
156-
get_ps_memory();
157-
158-
get_system_process_resident_memory({unix, linux}) ->
159-
get_ps_memory();
160-
161-
get_system_process_resident_memory({unix,freebsd}) ->
162-
get_ps_memory();
163-
164-
get_system_process_resident_memory({unix,openbsd}) ->
165-
get_ps_memory();
166-
167-
get_system_process_resident_memory({win32,_OSname}) ->
168-
OsPid = os:getpid(),
169-
Cmd = " tasklist /fi \"pid eq " ++ OsPid ++ "\" /fo LIST 2>&1 ",
170-
CmdOutput = os:cmd(Cmd),
171-
%% Memory usage is displayed in kilobytes
172-
%% with comma-separated thousands
173-
case re:run(CmdOutput, "Mem Usage:\\s+([0-9,]+)\\s+K", [{capture, all_but_first, list}]) of
174-
{match, [Match]} ->
175-
NoCommas = [ N || N <- Match, N =/= $, ],
176-
{ok, list_to_integer(NoCommas) * 1024};
177-
_ ->
178-
{error, {unexpected_output_from_command, Cmd, CmdOutput}}
179-
end;
180-
181-
get_system_process_resident_memory({unix, sunos}) ->
182-
get_ps_memory();
183-
184-
get_system_process_resident_memory({unix, aix}) ->
185-
get_ps_memory();
186-
187-
get_system_process_resident_memory(_OsType) ->
188-
{error, not_implemented_for_os}.
189-
190-
get_ps_memory() ->
191-
OsPid = os:getpid(),
192-
Cmd = "ps -p " ++ OsPid ++ " -o rss=",
193-
CmdOutput = os:cmd(Cmd),
194-
case re:run(CmdOutput, "[0-9]+", [{capture, first, list}]) of
195-
{match, [Match]} ->
196-
{ok, list_to_integer(Match) * 1024};
197-
_ ->
198-
{error, {unexpected_output_from_command, Cmd, CmdOutput}}
199-
end.
200-
201109
binary() ->
202110
All = interesting_sups(),
203111
{Sums, Rest} =

‎src/vm_memory_monitor.erl

Lines changed: 101 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,8 @@
3535
-export([get_total_memory/0, get_vm_limit/0,
3636
get_check_interval/0, set_check_interval/1,
3737
get_vm_memory_high_watermark/0, set_vm_memory_high_watermark/1,
38-
get_memory_limit/0, get_memory_use/1]).
38+
get_memory_limit/0, get_memory_use/1,
39+
get_process_memory/0]).
3940

4041
%% for tests
4142
-export([parse_line_linux/1]).
@@ -117,17 +118,109 @@ get_memory_limit() ->
117118

118119
get_memory_use(bytes) ->
119120
MemoryLimit = get_memory_limit(),
120-
{rabbit_vm:total_memory(), case MemoryLimit > 0.0 of
121-
true -> MemoryLimit;
122-
false -> infinity
123-
end};
121+
{get_process_memory(), case MemoryLimit > 0.0 of
122+
true -> MemoryLimit;
123+
false -> infinity
124+
end};
124125
get_memory_use(ratio) ->
125126
MemoryLimit = get_memory_limit(),
126127
case MemoryLimit > 0.0 of
127-
true -> rabbit_vm:total_memory() / MemoryLimit;
128+
true -> get_process_memory() / MemoryLimit;
128129
false -> infinity
129130
end.
130131

132+
%% Memory reported by erlang:memory(total) is not supposed to
133+
%% be equal to the total size of all pages mapped to the emulator,
134+
%% according to http://erlang.org/doc/man/erlang.html#memory-0
135+
%% erlang:memory(total) under-reports memory usage by around 20%
136+
-spec get_process_memory() -> Bytes :: integer().
137+
get_process_memory() ->
138+
case get_memory_calculation_strategy() of
139+
rss ->
140+
case get_system_process_resident_memory() of
141+
{ok, MemInBytes} ->
142+
MemInBytes;
143+
{error, Reason} ->
144+
rabbit_log:debug("Unable to get system memory used. Reason: ~p."
145+
" Falling back to erlang memory reporting",
146+
[Reason]),
147+
erlang:memory(total)
148+
end;
149+
erlang ->
150+
erlang:memory(total)
151+
end.
152+
153+
-spec get_memory_calculation_strategy() -> rss | erlang.
154+
get_memory_calculation_strategy() ->
155+
case application:get_env(rabbit, vm_memory_calculation_strategy, rss) of
156+
erlang ->
157+
erlang;
158+
rss ->
159+
rss;
160+
UnsupportedValue ->
161+
rabbit_log:warning(
162+
"Unsupported value '~p' for vm_memory_calculation_strategy. "
163+
"Supported values: (rss|erlang). "
164+
"Defaulting to 'rss'",
165+
[UnsupportedValue]
166+
),
167+
rss
168+
end.
169+
170+
-spec get_system_process_resident_memory() -> {ok, Bytes :: integer()} | {error, term()}.
171+
get_system_process_resident_memory() ->
172+
try
173+
get_system_process_resident_memory(os:type())
174+
catch _:Error ->
175+
{error, {"Failed to get process resident memory", Error}}
176+
end.
177+
178+
get_system_process_resident_memory({unix,darwin}) ->
179+
get_ps_memory();
180+
181+
get_system_process_resident_memory({unix, linux}) ->
182+
get_ps_memory();
183+
184+
get_system_process_resident_memory({unix,freebsd}) ->
185+
get_ps_memory();
186+
187+
get_system_process_resident_memory({unix,openbsd}) ->
188+
get_ps_memory();
189+
190+
get_system_process_resident_memory({win32,_OSname}) ->
191+
OsPid = os:getpid(),
192+
Cmd = " tasklist /fi \"pid eq " ++ OsPid ++ "\" /fo LIST 2>&1 ",
193+
CmdOutput = os:cmd(Cmd),
194+
%% Memory usage is displayed in kilobytes
195+
%% with comma-separated thousands
196+
case re:run(CmdOutput, "Mem Usage:\\s+([0-9,]+)\\s+K", [{capture, all_but_first, list}]) of
197+
{match, [Match]} ->
198+
NoCommas = [ N || N <- Match, N =/= $, ],
199+
{ok, list_to_integer(NoCommas) * 1024};
200+
_ ->
201+
{error, {unexpected_output_from_command, Cmd, CmdOutput}}
202+
end;
203+
204+
get_system_process_resident_memory({unix, sunos}) ->
205+
get_ps_memory();
206+
207+
get_system_process_resident_memory({unix, aix}) ->
208+
get_ps_memory();
209+
210+
get_system_process_resident_memory(_OsType) ->
211+
{error, not_implemented_for_os}.
212+
213+
get_ps_memory() ->
214+
OsPid = os:getpid(),
215+
Cmd = "ps -p " ++ OsPid ++ " -o rss=",
216+
CmdOutput = os:cmd(Cmd),
217+
case re:run(CmdOutput, "[0-9]+", [{capture, first, list}]) of
218+
{match, [Match]} ->
219+
{ok, list_to_integer(Match) * 1024};
220+
_ ->
221+
{error, {unexpected_output_from_command, Cmd, CmdOutput}}
222+
end.
223+
131224
%%----------------------------------------------------------------------------
132225
%% gen_server callbacks
133226
%%----------------------------------------------------------------------------
@@ -149,7 +242,7 @@ init([MemFraction, AlarmFuns]) ->
149242
{ok, set_mem_limits(State, MemFraction)}.
150243

151244
handle_call(get_vm_memory_high_watermark, _From,
152-
#state{memory_config_limit = MemLimit} = State) ->
245+
#state{memory_config_limit = MemLimit} = State) ->
153246
{reply, MemLimit, State};
154247

155248
handle_call({set_vm_memory_high_watermark, MemLimit}, _From, State) ->
@@ -268,7 +361,7 @@ parse_mem_limit(_) ->
268361
internal_update(State = #state { memory_limit = MemLimit,
269362
alarmed = Alarmed,
270363
alarm_funs = {AlarmSet, AlarmClear} }) ->
271-
MemUsed = rabbit_vm:total_memory(),
364+
MemUsed = get_process_memory(),
272365
NewAlarmed = MemUsed > MemLimit,
273366
case {Alarmed, NewAlarmed} of
274367
{false, true} -> emit_update_info(set, MemUsed, MemLimit),
@@ -365,7 +458,6 @@ get_total_memory({unix, aix}) ->
365458
get_total_memory(_OsType) ->
366459
unknown.
367460

368-
369461
%% A line looks like "Foo bar: 123456."
370462
parse_line_mach(Line) ->
371463
[Name, RHS | _Rest] = string:tokens(Line, ":"),

‎test/term_to_binary_compat_prop_SUITE.erl

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@
2323
-include_lib("common_test/include/ct.hrl").
2424
-include_lib("proper/include/proper.hrl").
2525

26+
-define(ITERATIONS_TO_RUN_UNTIL_CONFIDENT, 10000).
27+
2628
all() ->
2729
[
2830
pre_3_6_11_works,
@@ -48,33 +50,37 @@ init_per_testcase(Testcase, Config) ->
4850
%% If this test fails - the erlang version is not supported in
4951
%% RabbitMQ-3.6.10 and earlier.
5052
pre_3_6_11_works(Config) ->
51-
Fun = fun () -> prop_pre_3_6_11_works(Config) end,
52-
rabbit_ct_proper_helpers:run_proper(Fun, [], 50000).
53+
Property = fun () -> prop_pre_3_6_11_works(Config) end,
54+
rabbit_ct_proper_helpers:run_proper(Property, [],
55+
?ITERATIONS_TO_RUN_UNTIL_CONFIDENT).
5356

5457
prop_pre_3_6_11_works(_Config) ->
5558
?FORALL(Term, any(),
5659
begin
5760
Current = term_to_binary(Term),
5861
Compat = term_to_binary_compat:term_to_binary_1(Term),
59-
Current =:= Compat
62+
binary_to_term(Current) =:= binary_to_term(Compat)
6063
end).
6164

6265
term_to_binary_latin_atom(Config) ->
63-
Fun = fun () -> prop_term_to_binary_latin_atom(Config) end,
64-
rabbit_ct_proper_helpers:run_proper(Fun, [], 10000).
66+
Property = fun () -> prop_term_to_binary_latin_atom(Config) end,
67+
rabbit_ct_proper_helpers:run_proper(Property, [],
68+
?ITERATIONS_TO_RUN_UNTIL_CONFIDENT).
6569

6670
prop_term_to_binary_latin_atom(_Config) ->
6771
?FORALL(LatinString, list(integer(0, 255)),
6872
begin
6973
Length = length(LatinString),
7074
Atom = list_to_atom(LatinString),
7175
Binary = list_to_binary(LatinString),
72-
<<131,100, Length:16, Binary/binary>> =:= term_to_binary_compat:term_to_binary_1(Atom)
76+
<<131,100, Length:16, Binary/binary>> =:=
77+
term_to_binary_compat:term_to_binary_1(Atom)
7378
end).
7479

7580
queue_name_to_binary(Config) ->
76-
Fun = fun () -> prop_queue_name_to_binary(Config) end,
77-
rabbit_ct_proper_helpers:run_proper(Fun, [], 10000).
81+
Property = fun () -> prop_queue_name_to_binary(Config) end,
82+
rabbit_ct_proper_helpers:run_proper(Property, [],
83+
?ITERATIONS_TO_RUN_UNTIL_CONFIDENT).
7884

7985

8086
prop_queue_name_to_binary(_Config) ->

0 commit comments

Comments
 (0)