Skip to content

Commit 48ce1b5

Browse files
michaelklishinmichaelklishin
committed
Improve supported information units (Mi, Gi, Ti)
This revisits the information system conversion, that is, support for suffixes like GiB, GB. When configuration values like disk_free_limit.absolute, vm_memory_high_watermark.absolute are set, the value can contain an information unit (IU) suffix. We now support several new suffixes and the meaning a few more changes. First, the changes: * k, K now mean kilobytes and not kibibytes * m, M now mean megabytes and not mebibytes * g, G now means gigabytes and not gibibytes This is to match the system used by Kubernetes. There is no consensus in the industry about how "k", "m", "g", and similar single letter suffixes should be treated. Previously it was a power of 2, now a power of 10 to align with a very popular OSS project that explicitly documents what suffixes it supports. Now, the additions: Finally, the node will now validate these suffixes at boot time, so an unsupported value will cause the node to stop with a rabbitmq.conf validation error. The message logged will look like this: ```` 2024-01-15 22:11:17.829272-05:00 [error] <0.164.0> disk_free_limit.absolute invalid, supported formats: 500MB, 500MiB, 10GB, 10GiB, 2TB, 2TiB, 10000000000 2024-01-15 22:11:17.829376-05:00 [error] <0.164.0> Error preparing configuration in phase validation: 2024-01-15 22:11:17.829387-05:00 [error] <0.164.0> - disk_free_limit.absolute invalid, supported formats: 500MB, 500MiB, 10GB, 10GiB, 2TB, 2TiB, 10000000000 ```` Closes #10310
1 parent 838cb93 commit 48ce1b5

File tree

6 files changed

+195
-24
lines changed

6 files changed

+195
-24
lines changed

‎deps/rabbit/priv/schema/rabbit.schema

Lines changed: 32 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1064,10 +1064,13 @@ end}.
10641064
%% GB: gigabytes (10^9 - 1,000,000,000 bytes)
10651065

10661066
{mapping, "vm_memory_high_watermark.relative", "rabbit.vm_memory_high_watermark", [
1067-
{datatype, float}]}.
1067+
{datatype, float}
1068+
]}.
10681069

10691070
{mapping, "vm_memory_high_watermark.absolute", "rabbit.vm_memory_high_watermark", [
1070-
{datatype, [integer, string]}]}.
1071+
{datatype, [integer, string]},
1072+
{validators, ["is_supported_information_unit"]}
1073+
]}.
10711074

10721075

10731076
{translation, "rabbit.vm_memory_high_watermark",
@@ -1126,7 +1129,6 @@ end}.
11261129
%% {disk_free_limit, 50000000},
11271130
%%
11281131
%% Or you can set it using memory units (same as in vm_memory_high_watermark)
1129-
%% with RabbitMQ 3.6.0+.
11301132
%% {disk_free_limit, "50MB"},
11311133
%% {disk_free_limit, "50000kB"},
11321134
%% {disk_free_limit, "2GB"},
@@ -1140,7 +1142,9 @@ end}.
11401142
{datatype, float}]}.
11411143

11421144
{mapping, "disk_free_limit.absolute", "rabbit.disk_free_limit", [
1143-
{datatype, [integer, string]}]}.
1145+
{datatype, [integer, string]},
1146+
{validators, ["is_supported_information_unit"]}
1147+
]}.
11441148

11451149

11461150
{translation, "rabbit.disk_free_limit",
@@ -2639,3 +2643,27 @@ end}.
26392643
(String) -> {Res, _ } = re:compile(String),
26402644
Res =:= ok
26412645
end}.
2646+
2647+
{validator, "is_supported_information_unit", "supported formats: 500MB, 500MiB, 10GB, 10GiB, 2TB, 2TiB, 10000000000",
2648+
fun(S0) ->
2649+
case is_integer(S0) of
2650+
true -> true;
2651+
false ->
2652+
%% this is a string
2653+
S = string:strip(S0, right),
2654+
%% The prefix is optional
2655+
{ok, HasIUSuffix} = re:compile("([0-9]+)([a-zA-Z]){1,3}$", [dollar_endonly, caseless]),
2656+
%% Here are the prefixes we accept. This must match
2657+
%% what rabbit_resource_monitor_misc and 'rabbitmq-diagnostics status' can format.
2658+
{ok, SuffixExtractor} = re:compile("(k|ki|kb|kib|m|mi|mb|mib|g|gi|gb|gib|t|ti|tb|tib|p|pi|pb|pib)$", [dollar_endonly, caseless]),
2659+
case re:run(S, HasIUSuffix) of
2660+
nomatch -> false;
2661+
{match, _} ->
2662+
case re:split(S, SuffixExtractor) of
2663+
[] -> false;
2664+
[_CompleteMatch] -> false;
2665+
[_CompleteMatch, Suffix | _] -> true
2666+
end
2667+
end
2668+
end
2669+
end}.

‎deps/rabbit/test/config_schema_SUITE_data/rabbit.snippets

Lines changed: 87 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -125,10 +125,53 @@ ssl_options.fail_if_no_peer_cert = true",
125125
"disk_free_limit.relative = 1.0",
126126
[{rabbit, [{disk_free_limit, {mem_relative, 1.0}}]}],[]},
127127

128-
{disk_free_limit_only_absolute,
128+
%%
129+
%% Absolute free disk space limit
130+
%%
131+
132+
{disk_free_limit_only_absolute_integer,
129133
"disk_free_limit.absolute = 50000",
130134
[{rabbit, [{disk_free_limit, 50000}]}],[]},
131135

136+
{disk_free_limit_only_absolute_units_gb,
137+
"disk_free_limit.absolute = 2GB",
138+
[{rabbit,[{disk_free_limit, "2GB"}]}],
139+
[]},
140+
{disk_free_limit_only_absolute_units_gib,
141+
"disk_free_limit.absolute = 2GiB",
142+
[{rabbit,[{disk_free_limit, "2GiB"}]}],
143+
[]},
144+
{disk_free_limit_only_absolute_units_g,
145+
"disk_free_limit.absolute = 2G",
146+
[{rabbit,[{disk_free_limit, "2G"}]}],
147+
[]},
148+
149+
{disk_free_limit_only_absolute_units_tb,
150+
"disk_free_limit.absolute = 2TB",
151+
[{rabbit,[{disk_free_limit, "2TB"}]}],
152+
[]},
153+
{disk_free_limit_only_absolute_units_tib,
154+
"disk_free_limit.absolute = 2TiB",
155+
[{rabbit,[{disk_free_limit, "2TiB"}]}],
156+
[]},
157+
{disk_free_limit_only_absolute_units_t,
158+
"disk_free_limit.absolute = 2T",
159+
[{rabbit,[{disk_free_limit, "2T"}]}],
160+
[]},
161+
162+
{disk_free_limit_only_absolute_units_pb,
163+
"disk_free_limit.absolute = 2PB",
164+
[{rabbit,[{disk_free_limit, "2PB"}]}],
165+
[]},
166+
{disk_free_limit_only_absolute_units_pib,
167+
"disk_free_limit.absolute = 2PiB",
168+
[{rabbit,[{disk_free_limit, "2PiB"}]}],
169+
[]},
170+
{disk_free_limit_only_absolute_units_p,
171+
"disk_free_limit.absolute = 2P",
172+
[{rabbit,[{disk_free_limit, "2P"}]}],
173+
[]},
174+
132175
{default_users,
133176
"
134177
default_users.a.vhost_pattern = banana
@@ -252,14 +295,55 @@ tcp_listen_options.exit_on_close = false",
252295
[{tcp_listen_options,
253296
[{backlog,128},{nodelay,true},{exit_on_close,false}]}]}],
254297
[]},
255-
{vm_memory_watermark_absolute,
298+
299+
%%
300+
%% Absolute high runtime memory watermark
301+
%%
302+
303+
{vm_memory_watermark_absolute_integer,
256304
"vm_memory_high_watermark.absolute = 1073741824",
257305
[{rabbit,[{vm_memory_high_watermark,{absolute,1073741824}}]}],
258306
[]},
259-
{vm_memory_watermark_absolute_units,
307+
308+
{vm_memory_watermark_absolute_units_mb,
260309
"vm_memory_high_watermark.absolute = 1024MB",
261310
[{rabbit,[{vm_memory_high_watermark,{absolute,"1024MB"}}]}],
262311
[]},
312+
{vm_memory_watermark_absolute_units_mib,
313+
"vm_memory_high_watermark.absolute = 1024MiB",
314+
[{rabbit,[{vm_memory_high_watermark,{absolute,"1024MiB"}}]}],
315+
[]},
316+
{vm_memory_watermark_absolute_units_m,
317+
"vm_memory_high_watermark.absolute = 1024M",
318+
[{rabbit,[{vm_memory_high_watermark,{absolute,"1024M"}}]}],
319+
[]},
320+
321+
{vm_memory_watermark_absolute_units_gb,
322+
"vm_memory_high_watermark.absolute = 4GB",
323+
[{rabbit,[{vm_memory_high_watermark,{absolute,"4GB"}}]}],
324+
[]},
325+
{vm_memory_watermark_absolute_units_gib,
326+
"vm_memory_high_watermark.absolute = 3GiB",
327+
[{rabbit,[{vm_memory_high_watermark,{absolute,"3GiB"}}]}],
328+
[]},
329+
{vm_memory_watermark_absolute_units_g,
330+
"vm_memory_high_watermark.absolute = 10G",
331+
[{rabbit,[{vm_memory_high_watermark,{absolute,"10G"}}]}],
332+
[]},
333+
334+
{vm_memory_watermark_absolute_units_tb,
335+
"vm_memory_high_watermark.absolute = 1TB",
336+
[{rabbit,[{vm_memory_high_watermark,{absolute,"1TB"}}]}],
337+
[]},
338+
{vm_memory_watermark_absolute_units_tib,
339+
"vm_memory_high_watermark.absolute = 1TiB",
340+
[{rabbit,[{vm_memory_high_watermark,{absolute,"1TiB"}}]}],
341+
[]},
342+
{vm_memory_watermark_absolute_units_t,
343+
"vm_memory_high_watermark.absolute = 1T",
344+
[{rabbit,[{vm_memory_high_watermark,{absolute,"1T"}}]}],
345+
[]},
346+
263347
{vm_memory_watermark_absolute_priority,
264348
"vm_memory_high_watermark.absolute = 1073741824
265349
vm_memory_high_watermark.relative = 0.4",

‎deps/rabbit_common/src/rabbit_resource_monitor_misc.erl

Lines changed: 16 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -17,20 +17,29 @@ parse_information_unit(Value) when is_integer(Value) -> {ok, Value};
1717
parse_information_unit(Value0) ->
1818
Value = rabbit_data_coercion:to_list(Value0),
1919
case re:run(Value,
20-
"^(?<VAL>[0-9]+)(?<UNIT>kB|KB|MB|GB|kb|mb|gb|Kb|Mb|Gb|kiB|KiB|MiB|GiB|kib|mib|gib|KIB|MIB|GIB|k|K|m|M|g|G)?$",
20+
"^(?<VAL>[0-9]+)(?<UNIT>kB|Ki|Mi|Gi|Ti|Pi|KB|MB|GB|TB|PB|kb|ki|mb|gb|tb|pb|Kb|Mb|Gb|Tb|Pb|kiB|KiB|MiB|GiB|TiB|PiB|kib|mib|gib|tib|pib|KIB|MIB|GIB|TIB|PIB|k|K|m|M|g|G|p|P)?$",
2121
[{capture, all_but_first, list}]) of
2222
{match, [[], _]} ->
2323
{ok, list_to_integer(Value)};
2424
{match, [Num]} ->
2525
{ok, list_to_integer(Num)};
2626
{match, [Num, Unit]} ->
27+
%% Note: there is no industry standard on what K, M, G, T, P means (is G a gigabyte or a gibibyte?), so
28+
%% starting with 3.13 we treat those the same way as Kubernetes does [1].
29+
%%
30+
%% 1. https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#meaning-of-memory
2731
Multiplier = case Unit of
28-
KiB when KiB =:= "k"; KiB =:= "kiB"; KiB =:= "K"; KiB =:= "KIB"; KiB =:= "kib" -> 1024;
29-
MiB when MiB =:= "m"; MiB =:= "MiB"; MiB =:= "M"; MiB =:= "MIB"; MiB =:= "mib" -> 1024*1024;
30-
GiB when GiB =:= "g"; GiB =:= "GiB"; GiB =:= "G"; GiB =:= "GIB"; GiB =:= "gib" -> 1024*1024*1024;
31-
KB when KB =:= "KB"; KB =:= "kB"; KB =:= "kb"; KB =:= "Kb" -> 1000;
32-
MB when MB =:= "MB"; MB =:= "mB"; MB =:= "mb"; MB =:= "Mb" -> 1000000;
33-
GB when GB =:= "GB"; GB =:= "gB"; GB =:= "gb"; GB =:= "Gb" -> 1000000000
32+
KiB when KiB =:= "kiB"; KiB =:= "KIB"; KiB =:= "kib"; KiB =:= "ki"; KiB =:= "Ki" -> 1024;
33+
MiB when MiB =:= "MiB"; MiB =:= "MIB"; MiB =:= "mib"; MiB =:= "mi"; MiB =:= "Mi" -> 1024 * 1024;
34+
GiB when GiB =:= "GiB"; GiB =:= "GIB"; GiB =:= "gib"; GiB =:= "gi"; GiB =:= "Gi" -> 1024 * 1024 * 1024;
35+
TiB when TiB =:= "TiB"; TiB =:= "TIB"; TiB =:= "tib"; TiB =:= "ti"; TiB =:= "Ti" -> 1024 * 1024 * 1024 * 1024;
36+
PiB when PiB =:= "PiB"; PiB =:= "PIB"; PiB =:= "pib"; PiB =:= "pi"; PiB =:= "Pi" -> 1024 * 1024 * 1024 * 1024 * 2014;
37+
38+
KB when KB =:= "k"; KB =:= "K"; KB =:= "KB"; KB =:= "kB"; KB =:= "kb"; KB =:= "Kb" -> 1000;
39+
MB when MB =:= "m"; MB =:= "M"; MB =:= "MB"; MB =:= "mB"; MB =:= "mb"; MB =:= "Mb" -> 1000_000;
40+
GB when GB =:= "g"; GB =:= "G"; GB =:= "GB"; GB =:= "gB"; GB =:= "gb"; GB =:= "Gb" -> 1000_000_000;
41+
TB when TB =:= "t"; TB =:= "T"; TB =:= "TB"; TB =:= "tB"; TB =:= "tb"; TB =:= "Tb" -> 1000_000_000_000;
42+
PB when PB =:= "p"; PB =:= "P"; PB =:= "PB"; PB =:= "pB"; PB =:= "pb"; PB =:= "Pb" -> 1000_000_000_000_000
3443
end,
3544
{ok, list_to_integer(Num) * Multiplier};
3645
nomatch ->

‎deps/rabbitmq_cli/lib/rabbitmq/cli/ctl/commands/status_command.ex

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ defmodule RabbitMQ.CLI.Ctl.Commands.StatusCommand do
3939
:ok
4040

4141
false ->
42-
{:validation_failure, "unit '#{unit}' is not supported. Please use one of: bytes, mb, gb"}
42+
{:validation_failure, "unit '#{unit}' is not supported. Please use one of: bytes, mb, mib, gb, gib, tb, tib"}
4343
end
4444
end
4545

@@ -212,7 +212,7 @@ defmodule RabbitMQ.CLI.Ctl.Commands.StatusCommand do
212212

213213
def usage_additional() do
214214
[
215-
["--unit <bytes | mb | gb>", "byte multiple (bytes, megabytes, gigabytes) to use"],
215+
["--unit <bytes | mb | mib | gb | gib>", "byte multiple (bytes, megabytes, gigabytes) to use"],
216216
["--formatter <json | erlang>", "alternative formatter (JSON, Erlang terms)"]
217217
]
218218
end

‎deps/rabbitmq_cli/lib/rabbitmq/cli/information_unit.ex

Lines changed: 55 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,18 +11,42 @@ defmodule RabbitMQ.CLI.InformationUnit do
1111
@megabyte_bytes @kilobyte_bytes * 1000
1212
@gigabyte_bytes @megabyte_bytes * 1000
1313
@terabyte_bytes @gigabyte_bytes * 1000
14+
@petabyte_bytes @terabyte_bytes * 1000
15+
16+
@kibibyte_bytes 1024
17+
@mebibyte_bytes @kibibyte_bytes * 1024
18+
@gibibyte_bytes @mebibyte_bytes * 1024
19+
@tebibyte_bytes @gibibyte_bytes * 1024
20+
@pebibyte_bytes @tebibyte_bytes * 1024
1421

1522
def known_units() do
1623
MapSet.new([
1724
"bytes",
25+
"k",
1826
"kb",
27+
"ki",
28+
"kib",
1929
"kilobytes",
30+
"m",
2031
"mb",
32+
"mi",
33+
"mib",
2134
"megabytes",
35+
"g",
2236
"gb",
37+
"gi",
38+
"gib",
2339
"gigabytes",
40+
"t",
2441
"tb",
25-
"terabytes"
42+
"ti",
43+
"tib",
44+
"terabytes",
45+
"p",
46+
"pb",
47+
"pi",
48+
"pib",
49+
"petabytes"
2650
])
2751
end
2852

@@ -49,24 +73,50 @@ defmodule RabbitMQ.CLI.InformationUnit do
4973
defp do_convert(bytes, "kb") do
5074
Float.round(bytes / @kilobyte_bytes, 4)
5175
end
52-
76+
defp do_convert(bytes, "k"), do: do_convert(bytes, "kb")
77+
defp do_convert(bytes, "ki") do
78+
Float.round(bytes / @kibibyte_bytes, 4)
79+
end
80+
defp do_convert(bytes, "kib"), do: do_convert(bytes, "ki")
5381
defp do_convert(bytes, "kilobytes"), do: do_convert(bytes, "kb")
5482

5583
defp do_convert(bytes, "mb") do
5684
Float.round(bytes / @megabyte_bytes, 4)
5785
end
58-
86+
defp do_convert(bytes, "m"), do: do_convert(bytes, "mb")
87+
defp do_convert(bytes, "mi") do
88+
Float.round(bytes / @mebibyte_bytes, 4)
89+
end
90+
defp do_convert(bytes, "mib"), do: do_convert(bytes, "mi")
5991
defp do_convert(bytes, "megabytes"), do: do_convert(bytes, "mb")
6092

6193
defp do_convert(bytes, "gb") do
6294
Float.round(bytes / @gigabyte_bytes, 4)
6395
end
64-
96+
defp do_convert(bytes, "g"), do: do_convert(bytes, "gb")
97+
defp do_convert(bytes, "gi") do
98+
Float.round(bytes / @gigabyte_bytes, 4)
99+
end
100+
defp do_convert(bytes, "gib"), do: do_convert(bytes, "gi")
65101
defp do_convert(bytes, "gigabytes"), do: do_convert(bytes, "gb")
66102

67103
defp do_convert(bytes, "tb") do
68104
Float.round(bytes / @terabyte_bytes, 4)
69105
end
70-
106+
defp do_convert(bytes, "t"), do: do_convert(bytes, "tb")
107+
defp do_convert(bytes, "ti") do
108+
Float.round(bytes / @tebibyte_bytes, 4)
109+
end
110+
defp do_convert(bytes, "tib"), do: do_convert(bytes, "ti")
71111
defp do_convert(bytes, "terabytes"), do: do_convert(bytes, "tb")
112+
113+
defp do_convert(bytes, "pb") do
114+
Float.round(bytes / @petabyte_bytes, 4)
115+
end
116+
defp do_convert(bytes, "p"), do: do_convert(bytes, "pb")
117+
defp do_convert(bytes, "pi") do
118+
Float.round(bytes / @pebibyte_bytes, 4)
119+
end
120+
defp do_convert(bytes, "pib"), do: do_convert(bytes, "pi")
121+
defp do_convert(bytes, "petabytes"), do: do_convert(bytes, "pb")
72122
end

‎deps/rabbitmq_ct_helpers/src/rabbit_ct_config_schema.erl

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ snippet_id(A) when is_atom(A) ->
4040
snippet_id(L) when is_list(L) ->
4141
L.
4242

43-
test_snippet(Config, Snippet, Expected, _Plugins) ->
43+
test_snippet(Config, Snippet = {SnipID, _, _}, Expected, _Plugins) ->
4444
{ConfFile, AdvancedFile} = write_snippet(Config, Snippet),
4545
%% We ignore the rabbit -> log portion of the config on v3.9+, where the lager
4646
%% dependency has been dropped
@@ -55,8 +55,8 @@ test_snippet(Config, Snippet, Expected, _Plugins) ->
5555
case Exp of
5656
Gen -> ok;
5757
_ ->
58-
ct:pal("Expected: ~tp~ngenerated: ~tp", [Expected, Generated]),
59-
ct:pal("Expected (sorted): ~tp~ngenerated (sorted): ~tp", [Exp, Gen]),
58+
ct:pal("Snippet ~tp. Expected: ~tp~ngenerated: ~tp", [SnipID, Expected, Generated]),
59+
ct:pal("Snippet ~tp. Expected (sorted): ~tp~ngenerated (sorted): ~tp", [SnipID, Exp, Gen]),
6060
error({config_mismatch, Snippet, Exp, Gen})
6161
end.
6262

0 commit comments

Comments
 (0)