Skip to content

Commit ac9c943

Browse files
michaelklishinmichaelklishin
authored
Merge pull request #10348 from rabbitmq/mk-information-units-kubernetes-style
Handle more information units (Mi, Gi, Ti), add rabbitmq.conf validation for IU values
2 parents a3c8880 + 79d52b9 commit ac9c943

File tree

6 files changed

+210
-20
lines changed

6 files changed

+210
-20
lines changed

‎deps/rabbit/priv/schema/rabbit.schema

Lines changed: 32 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1064,10 +1064,13 @@ end}.
10641064
%% GB: gigabytes (10^9 - 1,000,000,000 bytes)
10651065

10661066
{mapping, "vm_memory_high_watermark.relative", "rabbit.vm_memory_high_watermark", [
1067-
{datatype, float}]}.
1067+
{datatype, float}
1068+
]}.
10681069

10691070
{mapping, "vm_memory_high_watermark.absolute", "rabbit.vm_memory_high_watermark", [
1070-
{datatype, [integer, string]}]}.
1071+
{datatype, [integer, string]},
1072+
{validators, ["is_supported_information_unit"]}
1073+
]}.
10711074

10721075

10731076
{translation, "rabbit.vm_memory_high_watermark",
@@ -1126,7 +1129,6 @@ end}.
11261129
%% {disk_free_limit, 50000000},
11271130
%%
11281131
%% Or you can set it using memory units (same as in vm_memory_high_watermark)
1129-
%% with RabbitMQ 3.6.0+.
11301132
%% {disk_free_limit, "50MB"},
11311133
%% {disk_free_limit, "50000kB"},
11321134
%% {disk_free_limit, "2GB"},
@@ -1140,7 +1142,9 @@ end}.
11401142
{datatype, float}]}.
11411143

11421144
{mapping, "disk_free_limit.absolute", "rabbit.disk_free_limit", [
1143-
{datatype, [integer, string]}]}.
1145+
{datatype, [integer, string]},
1146+
{validators, ["is_supported_information_unit"]}
1147+
]}.
11441148

11451149

11461150
{translation, "rabbit.disk_free_limit",
@@ -2639,3 +2643,27 @@ end}.
26392643
(String) -> {Res, _ } = re:compile(String),
26402644
Res =:= ok
26412645
end}.
2646+
2647+
{validator, "is_supported_information_unit", "supported formats: 500MB, 500MiB, 10GB, 10GiB, 2TB, 2TiB, 10000000000",
2648+
fun(S0) ->
2649+
case is_integer(S0) of
2650+
true -> true;
2651+
false ->
2652+
%% this is a string
2653+
S = string:strip(S0, right),
2654+
%% The prefix is optional
2655+
{ok, HasIUSuffix} = re:compile("([0-9]+)([a-zA-Z]){1,3}$", [dollar_endonly, caseless]),
2656+
%% Here are the prefixes we accept. This must match
2657+
%% what rabbit_resource_monitor_misc and 'rabbitmq-diagnostics status' can format.
2658+
{ok, SuffixExtractor} = re:compile("(k|ki|kb|kib|m|mi|mb|mib|g|gi|gb|gib|t|ti|tb|tib|p|pi|pb|pib)$", [dollar_endonly, caseless]),
2659+
case re:run(S, HasIUSuffix) of
2660+
nomatch -> false;
2661+
{match, _} ->
2662+
case re:split(S, SuffixExtractor) of
2663+
[] -> false;
2664+
[_CompleteMatch] -> false;
2665+
[_CompleteMatch, Suffix | _] -> true
2666+
end
2667+
end
2668+
end
2669+
end}.

‎deps/rabbit/test/config_schema_SUITE_data/rabbit.snippets

Lines changed: 87 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -125,10 +125,53 @@ ssl_options.fail_if_no_peer_cert = true",
125125
"disk_free_limit.relative = 1.0",
126126
[{rabbit, [{disk_free_limit, {mem_relative, 1.0}}]}],[]},
127127

128-
{disk_free_limit_only_absolute,
128+
%%
129+
%% Absolute free disk space limit
130+
%%
131+
132+
{disk_free_limit_only_absolute_integer,
129133
"disk_free_limit.absolute = 50000",
130134
[{rabbit, [{disk_free_limit, 50000}]}],[]},
131135

136+
{disk_free_limit_only_absolute_units_gb,
137+
"disk_free_limit.absolute = 2GB",
138+
[{rabbit,[{disk_free_limit, "2GB"}]}],
139+
[]},
140+
{disk_free_limit_only_absolute_units_gib,
141+
"disk_free_limit.absolute = 2GiB",
142+
[{rabbit,[{disk_free_limit, "2GiB"}]}],
143+
[]},
144+
{disk_free_limit_only_absolute_units_g,
145+
"disk_free_limit.absolute = 2G",
146+
[{rabbit,[{disk_free_limit, "2G"}]}],
147+
[]},
148+
149+
{disk_free_limit_only_absolute_units_tb,
150+
"disk_free_limit.absolute = 2TB",
151+
[{rabbit,[{disk_free_limit, "2TB"}]}],
152+
[]},
153+
{disk_free_limit_only_absolute_units_tib,
154+
"disk_free_limit.absolute = 2TiB",
155+
[{rabbit,[{disk_free_limit, "2TiB"}]}],
156+
[]},
157+
{disk_free_limit_only_absolute_units_t,
158+
"disk_free_limit.absolute = 2T",
159+
[{rabbit,[{disk_free_limit, "2T"}]}],
160+
[]},
161+
162+
{disk_free_limit_only_absolute_units_pb,
163+
"disk_free_limit.absolute = 2PB",
164+
[{rabbit,[{disk_free_limit, "2PB"}]}],
165+
[]},
166+
{disk_free_limit_only_absolute_units_pib,
167+
"disk_free_limit.absolute = 2PiB",
168+
[{rabbit,[{disk_free_limit, "2PiB"}]}],
169+
[]},
170+
{disk_free_limit_only_absolute_units_p,
171+
"disk_free_limit.absolute = 2P",
172+
[{rabbit,[{disk_free_limit, "2P"}]}],
173+
[]},
174+
132175
{default_users,
133176
"
134177
default_users.a.vhost_pattern = banana
@@ -252,14 +295,55 @@ tcp_listen_options.exit_on_close = false",
252295
[{tcp_listen_options,
253296
[{backlog,128},{nodelay,true},{exit_on_close,false}]}]}],
254297
[]},
255-
{vm_memory_watermark_absolute,
298+
299+
%%
300+
%% Absolute high runtime memory watermark
301+
%%
302+
303+
{vm_memory_watermark_absolute_integer,
256304
"vm_memory_high_watermark.absolute = 1073741824",
257305
[{rabbit,[{vm_memory_high_watermark,{absolute,1073741824}}]}],
258306
[]},
259-
{vm_memory_watermark_absolute_units,
307+
308+
{vm_memory_watermark_absolute_units_mb,
260309
"vm_memory_high_watermark.absolute = 1024MB",
261310
[{rabbit,[{vm_memory_high_watermark,{absolute,"1024MB"}}]}],
262311
[]},
312+
{vm_memory_watermark_absolute_units_mib,
313+
"vm_memory_high_watermark.absolute = 1024MiB",
314+
[{rabbit,[{vm_memory_high_watermark,{absolute,"1024MiB"}}]}],
315+
[]},
316+
{vm_memory_watermark_absolute_units_m,
317+
"vm_memory_high_watermark.absolute = 1024M",
318+
[{rabbit,[{vm_memory_high_watermark,{absolute,"1024M"}}]}],
319+
[]},
320+
321+
{vm_memory_watermark_absolute_units_gb,
322+
"vm_memory_high_watermark.absolute = 4GB",
323+
[{rabbit,[{vm_memory_high_watermark,{absolute,"4GB"}}]}],
324+
[]},
325+
{vm_memory_watermark_absolute_units_gib,
326+
"vm_memory_high_watermark.absolute = 3GiB",
327+
[{rabbit,[{vm_memory_high_watermark,{absolute,"3GiB"}}]}],
328+
[]},
329+
{vm_memory_watermark_absolute_units_g,
330+
"vm_memory_high_watermark.absolute = 10G",
331+
[{rabbit,[{vm_memory_high_watermark,{absolute,"10G"}}]}],
332+
[]},
333+
334+
{vm_memory_watermark_absolute_units_tb,
335+
"vm_memory_high_watermark.absolute = 1TB",
336+
[{rabbit,[{vm_memory_high_watermark,{absolute,"1TB"}}]}],
337+
[]},
338+
{vm_memory_watermark_absolute_units_tib,
339+
"vm_memory_high_watermark.absolute = 1TiB",
340+
[{rabbit,[{vm_memory_high_watermark,{absolute,"1TiB"}}]}],
341+
[]},
342+
{vm_memory_watermark_absolute_units_t,
343+
"vm_memory_high_watermark.absolute = 1T",
344+
[{rabbit,[{vm_memory_high_watermark,{absolute,"1T"}}]}],
345+
[]},
346+
263347
{vm_memory_watermark_absolute_priority,
264348
"vm_memory_high_watermark.absolute = 1073741824
265349
vm_memory_high_watermark.relative = 0.4",

‎deps/rabbit_common/src/rabbit_resource_monitor_misc.erl

Lines changed: 16 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -17,20 +17,29 @@ parse_information_unit(Value) when is_integer(Value) -> {ok, Value};
1717
parse_information_unit(Value0) ->
1818
Value = rabbit_data_coercion:to_list(Value0),
1919
case re:run(Value,
20-
"^(?<VAL>[0-9]+)(?<UNIT>kB|KB|MB|GB|kb|mb|gb|Kb|Mb|Gb|kiB|KiB|MiB|GiB|kib|mib|gib|KIB|MIB|GIB|k|K|m|M|g|G)?$",
20+
"^(?<VAL>[0-9]+)(?<UNIT>kB|Ki|Mi|Gi|Ti|Pi|KB|MB|GB|TB|PB|kb|ki|mb|gb|tb|pb|Kb|Mb|Gb|Tb|Pb|kiB|KiB|MiB|GiB|TiB|PiB|kib|mib|gib|tib|pib|KIB|MIB|GIB|TIB|PIB|k|K|m|M|g|G|p|P)?$",
2121
[{capture, all_but_first, list}]) of
2222
{match, [[], _]} ->
2323
{ok, list_to_integer(Value)};
2424
{match, [Num]} ->
2525
{ok, list_to_integer(Num)};
2626
{match, [Num, Unit]} ->
27+
%% Note: there is no industry standard on what K, M, G, T, P means (is G a gigabyte or a gibibyte?), so
28+
%% starting with 3.13 we treat those the same way as Kubernetes does [1].
29+
%%
30+
%% 1. https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#meaning-of-memory
2731
Multiplier = case Unit of
28-
KiB when KiB =:= "k"; KiB =:= "kiB"; KiB =:= "K"; KiB =:= "KIB"; KiB =:= "kib" -> 1024;
29-
MiB when MiB =:= "m"; MiB =:= "MiB"; MiB =:= "M"; MiB =:= "MIB"; MiB =:= "mib" -> 1024*1024;
30-
GiB when GiB =:= "g"; GiB =:= "GiB"; GiB =:= "G"; GiB =:= "GIB"; GiB =:= "gib" -> 1024*1024*1024;
31-
KB when KB =:= "KB"; KB =:= "kB"; KB =:= "kb"; KB =:= "Kb" -> 1000;
32-
MB when MB =:= "MB"; MB =:= "mB"; MB =:= "mb"; MB =:= "Mb" -> 1000000;
33-
GB when GB =:= "GB"; GB =:= "gB"; GB =:= "gb"; GB =:= "Gb" -> 1000000000
32+
KiB when KiB =:= "kiB"; KiB =:= "KIB"; KiB =:= "kib"; KiB =:= "ki"; KiB =:= "Ki" -> 1024;
33+
MiB when MiB =:= "MiB"; MiB =:= "MIB"; MiB =:= "mib"; MiB =:= "mi"; MiB =:= "Mi" -> 1024 * 1024;
34+
GiB when GiB =:= "GiB"; GiB =:= "GIB"; GiB =:= "gib"; GiB =:= "gi"; GiB =:= "Gi" -> 1024 * 1024 * 1024;
35+
TiB when TiB =:= "TiB"; TiB =:= "TIB"; TiB =:= "tib"; TiB =:= "ti"; TiB =:= "Ti" -> 1024 * 1024 * 1024 * 1024;
36+
PiB when PiB =:= "PiB"; PiB =:= "PIB"; PiB =:= "pib"; PiB =:= "pi"; PiB =:= "Pi" -> 1024 * 1024 * 1024 * 1024 * 2014;
37+
38+
KB when KB =:= "k"; KB =:= "K"; KB =:= "KB"; KB =:= "kB"; KB =:= "kb"; KB =:= "Kb" -> 1000;
39+
MB when MB =:= "m"; MB =:= "M"; MB =:= "MB"; MB =:= "mB"; MB =:= "mb"; MB =:= "Mb" -> 1000_000;
40+
GB when GB =:= "g"; GB =:= "G"; GB =:= "GB"; GB =:= "gB"; GB =:= "gb"; GB =:= "Gb" -> 1000_000_000;
41+
TB when TB =:= "t"; TB =:= "T"; TB =:= "TB"; TB =:= "tB"; TB =:= "tb"; TB =:= "Tb" -> 1000_000_000_000;
42+
PB when PB =:= "p"; PB =:= "P"; PB =:= "PB"; PB =:= "pB"; PB =:= "pb"; PB =:= "Pb" -> 1000_000_000_000_000
3443
end,
3544
{ok, list_to_integer(Num) * Multiplier};
3645
nomatch ->

‎deps/rabbitmq_cli/lib/rabbitmq/cli/ctl/commands/status_command.ex

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,8 @@ defmodule RabbitMQ.CLI.Ctl.Commands.StatusCommand do
3939
:ok
4040

4141
false ->
42-
{:validation_failure, "unit '#{unit}' is not supported. Please use one of: bytes, mb, gb"}
42+
{:validation_failure,
43+
"unit '#{unit}' is not supported. Please use one of: bytes, mb, mib, gb, gib, tb, tib"}
4344
end
4445
end
4546

@@ -212,7 +213,10 @@ defmodule RabbitMQ.CLI.Ctl.Commands.StatusCommand do
212213

213214
def usage_additional() do
214215
[
215-
["--unit <bytes | mb | gb>", "byte multiple (bytes, megabytes, gigabytes) to use"],
216+
[
217+
"--unit <bytes | mb | mib | gb | gib>",
218+
"byte multiple (bytes, megabytes, gigabytes) to use"
219+
],
216220
["--formatter <json | erlang>", "alternative formatter (JSON, Erlang terms)"]
217221
]
218222
end

‎deps/rabbitmq_cli/lib/rabbitmq/cli/information_unit.ex

Lines changed: 66 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,18 +11,42 @@ defmodule RabbitMQ.CLI.InformationUnit do
1111
@megabyte_bytes @kilobyte_bytes * 1000
1212
@gigabyte_bytes @megabyte_bytes * 1000
1313
@terabyte_bytes @gigabyte_bytes * 1000
14+
@petabyte_bytes @terabyte_bytes * 1000
15+
16+
@kibibyte_bytes 1024
17+
@mebibyte_bytes @kibibyte_bytes * 1024
18+
@gibibyte_bytes @mebibyte_bytes * 1024
19+
@tebibyte_bytes @gibibyte_bytes * 1024
20+
@pebibyte_bytes @tebibyte_bytes * 1024
1421

1522
def known_units() do
1623
MapSet.new([
1724
"bytes",
25+
"k",
1826
"kb",
27+
"ki",
28+
"kib",
1929
"kilobytes",
30+
"m",
2031
"mb",
32+
"mi",
33+
"mib",
2134
"megabytes",
35+
"g",
2236
"gb",
37+
"gi",
38+
"gib",
2339
"gigabytes",
40+
"t",
2441
"tb",
25-
"terabytes"
42+
"ti",
43+
"tib",
44+
"terabytes",
45+
"p",
46+
"pb",
47+
"pi",
48+
"pib",
49+
"petabytes"
2650
])
2751
end
2852

@@ -50,23 +74,64 @@ defmodule RabbitMQ.CLI.InformationUnit do
5074
Float.round(bytes / @kilobyte_bytes, 4)
5175
end
5276

77+
defp do_convert(bytes, "k"), do: do_convert(bytes, "kb")
78+
79+
defp do_convert(bytes, "ki") do
80+
Float.round(bytes / @kibibyte_bytes, 4)
81+
end
82+
83+
defp do_convert(bytes, "kib"), do: do_convert(bytes, "ki")
5384
defp do_convert(bytes, "kilobytes"), do: do_convert(bytes, "kb")
5485

5586
defp do_convert(bytes, "mb") do
5687
Float.round(bytes / @megabyte_bytes, 4)
5788
end
5889

90+
defp do_convert(bytes, "m"), do: do_convert(bytes, "mb")
91+
92+
defp do_convert(bytes, "mi") do
93+
Float.round(bytes / @mebibyte_bytes, 4)
94+
end
95+
96+
defp do_convert(bytes, "mib"), do: do_convert(bytes, "mi")
5997
defp do_convert(bytes, "megabytes"), do: do_convert(bytes, "mb")
6098

6199
defp do_convert(bytes, "gb") do
62100
Float.round(bytes / @gigabyte_bytes, 4)
63101
end
64102

103+
defp do_convert(bytes, "g"), do: do_convert(bytes, "gb")
104+
105+
defp do_convert(bytes, "gi") do
106+
Float.round(bytes / @gigabyte_bytes, 4)
107+
end
108+
109+
defp do_convert(bytes, "gib"), do: do_convert(bytes, "gi")
65110
defp do_convert(bytes, "gigabytes"), do: do_convert(bytes, "gb")
66111

67112
defp do_convert(bytes, "tb") do
68113
Float.round(bytes / @terabyte_bytes, 4)
69114
end
70115

116+
defp do_convert(bytes, "t"), do: do_convert(bytes, "tb")
117+
118+
defp do_convert(bytes, "ti") do
119+
Float.round(bytes / @tebibyte_bytes, 4)
120+
end
121+
122+
defp do_convert(bytes, "tib"), do: do_convert(bytes, "ti")
71123
defp do_convert(bytes, "terabytes"), do: do_convert(bytes, "tb")
124+
125+
defp do_convert(bytes, "pb") do
126+
Float.round(bytes / @petabyte_bytes, 4)
127+
end
128+
129+
defp do_convert(bytes, "p"), do: do_convert(bytes, "pb")
130+
131+
defp do_convert(bytes, "pi") do
132+
Float.round(bytes / @pebibyte_bytes, 4)
133+
end
134+
135+
defp do_convert(bytes, "pib"), do: do_convert(bytes, "pi")
136+
defp do_convert(bytes, "petabytes"), do: do_convert(bytes, "pb")
72137
end

‎deps/rabbitmq_ct_helpers/src/rabbit_ct_config_schema.erl

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ snippet_id(A) when is_atom(A) ->
4040
snippet_id(L) when is_list(L) ->
4141
L.
4242

43-
test_snippet(Config, Snippet, Expected, _Plugins) ->
43+
test_snippet(Config, Snippet = {SnipID, _, _}, Expected, _Plugins) ->
4444
{ConfFile, AdvancedFile} = write_snippet(Config, Snippet),
4545
%% We ignore the rabbit -> log portion of the config on v3.9+, where the lager
4646
%% dependency has been dropped
@@ -55,8 +55,8 @@ test_snippet(Config, Snippet, Expected, _Plugins) ->
5555
case Exp of
5656
Gen -> ok;
5757
_ ->
58-
ct:pal("Expected: ~tp~ngenerated: ~tp", [Expected, Generated]),
59-
ct:pal("Expected (sorted): ~tp~ngenerated (sorted): ~tp", [Exp, Gen]),
58+
ct:pal("Snippet ~tp. Expected: ~tp~ngenerated: ~tp", [SnipID, Expected, Generated]),
59+
ct:pal("Snippet ~tp. Expected (sorted): ~tp~ngenerated (sorted): ~tp", [SnipID, Exp, Gen]),
6060
error({config_mismatch, Snippet, Exp, Gen})
6161
end.
6262

0 commit comments

Comments
 (0)