Skip to content

Handle more information units (Mi, Gi, Ti), add rabbitmq.conf validation for IU values #10348

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Jan 16, 2024
Merged

Handle more information units (Mi, Gi, Ti), add rabbitmq.conf validation for IU values #10348

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
48ce1b5
Improve supported information units (Mi, Gi, Ti)
michaelklishin Jan 16, 2024
79d52b9
CLI: mix format
michaelklishin Jan 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 32 additions & 4 deletions deps/rabbit/priv/schema/rabbit.schema
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1064,10 +1064,13 @@ end}.
%% GB: gigabytes (10^9 - 1,000,000,000 bytes)

{mapping, "vm_memory_high_watermark.relative", "rabbit.vm_memory_high_watermark", [
{datatype, float}]}.
{datatype, float}
]}.

{mapping, "vm_memory_high_watermark.absolute", "rabbit.vm_memory_high_watermark", [
{datatype, [integer, string]}]}.
{datatype, [integer, string]},
{validators, ["is_supported_information_unit"]}
]}.


{translation, "rabbit.vm_memory_high_watermark",
Expand Down Expand Up @@ -1126,7 +1129,6 @@ end}.
%% {disk_free_limit, 50000000},
%%
%% Or you can set it using memory units (same as in vm_memory_high_watermark)
%% with RabbitMQ 3.6.0+.
%% {disk_free_limit, "50MB"},
%% {disk_free_limit, "50000kB"},
%% {disk_free_limit, "2GB"},
Expand All @@ -1140,7 +1142,9 @@ end}.
{datatype, float}]}.

{mapping, "disk_free_limit.absolute", "rabbit.disk_free_limit", [
{datatype, [integer, string]}]}.
{datatype, [integer, string]},
{validators, ["is_supported_information_unit"]}
]}.


{translation, "rabbit.disk_free_limit",
Expand Down Expand Up @@ -2639,3 +2643,27 @@ end}.
(String) -> {Res, _ } = re:compile(String),
Res =:= ok
end}.

{validator, "is_supported_information_unit", "supported formats: 500MB, 500MiB, 10GB, 10GiB, 2TB, 2TiB, 10000000000",
fun(S0) ->
case is_integer(S0) of
true -> true;
false ->
%% this is a string
S = string:strip(S0, right),
%% The prefix is optional
{ok, HasIUSuffix} = re:compile("([0-9]+)([a-zA-Z]){1,3}$", [dollar_endonly, caseless]),
%% Here are the prefixes we accept. This must match
%% what rabbit_resource_monitor_misc and 'rabbitmq-diagnostics status' can format.
{ok, SuffixExtractor} = re:compile("(k|ki|kb|kib|m|mi|mb|mib|g|gi|gb|gib|t|ti|tb|tib|p|pi|pb|pib)$", [dollar_endonly, caseless]),
case re:run(S, HasIUSuffix) of
nomatch -> false;
{match, _} ->
case re:split(S, SuffixExtractor) of
[] -> false;
[_CompleteMatch] -> false;
[_CompleteMatch, Suffix | _] -> true
end
end
end
end}.
90 changes: 87 additions & 3 deletions deps/rabbit/test/config_schema_SUITE_data/rabbit.snippets
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,10 +125,53 @@ ssl_options.fail_if_no_peer_cert = true",
"disk_free_limit.relative = 1.0",
[{rabbit, [{disk_free_limit, {mem_relative, 1.0}}]}],[]},

{disk_free_limit_only_absolute,
%%
%% Absolute free disk space limit
%%

{disk_free_limit_only_absolute_integer,
"disk_free_limit.absolute = 50000",
[{rabbit, [{disk_free_limit, 50000}]}],[]},

{disk_free_limit_only_absolute_units_gb,
"disk_free_limit.absolute = 2GB",
[{rabbit,[{disk_free_limit, "2GB"}]}],
[]},
{disk_free_limit_only_absolute_units_gib,
"disk_free_limit.absolute = 2GiB",
[{rabbit,[{disk_free_limit, "2GiB"}]}],
[]},
{disk_free_limit_only_absolute_units_g,
"disk_free_limit.absolute = 2G",
[{rabbit,[{disk_free_limit, "2G"}]}],
[]},

{disk_free_limit_only_absolute_units_tb,
"disk_free_limit.absolute = 2TB",
[{rabbit,[{disk_free_limit, "2TB"}]}],
[]},
{disk_free_limit_only_absolute_units_tib,
"disk_free_limit.absolute = 2TiB",
[{rabbit,[{disk_free_limit, "2TiB"}]}],
[]},
{disk_free_limit_only_absolute_units_t,
"disk_free_limit.absolute = 2T",
[{rabbit,[{disk_free_limit, "2T"}]}],
[]},

{disk_free_limit_only_absolute_units_pb,
"disk_free_limit.absolute = 2PB",
[{rabbit,[{disk_free_limit, "2PB"}]}],
[]},
{disk_free_limit_only_absolute_units_pib,
"disk_free_limit.absolute = 2PiB",
[{rabbit,[{disk_free_limit, "2PiB"}]}],
[]},
{disk_free_limit_only_absolute_units_p,
"disk_free_limit.absolute = 2P",
[{rabbit,[{disk_free_limit, "2P"}]}],
[]},

{default_users,
"
default_users.a.vhost_pattern = banana
Expand Down Expand Up @@ -252,14 +295,55 @@ tcp_listen_options.exit_on_close = false",
[{tcp_listen_options,
[{backlog,128},{nodelay,true},{exit_on_close,false}]}]}],
[]},
{vm_memory_watermark_absolute,

%%
%% Absolute high runtime memory watermark
%%

{vm_memory_watermark_absolute_integer,
"vm_memory_high_watermark.absolute = 1073741824",
[{rabbit,[{vm_memory_high_watermark,{absolute,1073741824}}]}],
[]},
{vm_memory_watermark_absolute_units,

{vm_memory_watermark_absolute_units_mb,
"vm_memory_high_watermark.absolute = 1024MB",
[{rabbit,[{vm_memory_high_watermark,{absolute,"1024MB"}}]}],
[]},
{vm_memory_watermark_absolute_units_mib,
"vm_memory_high_watermark.absolute = 1024MiB",
[{rabbit,[{vm_memory_high_watermark,{absolute,"1024MiB"}}]}],
[]},
{vm_memory_watermark_absolute_units_m,
"vm_memory_high_watermark.absolute = 1024M",
[{rabbit,[{vm_memory_high_watermark,{absolute,"1024M"}}]}],
[]},

{vm_memory_watermark_absolute_units_gb,
"vm_memory_high_watermark.absolute = 4GB",
[{rabbit,[{vm_memory_high_watermark,{absolute,"4GB"}}]}],
[]},
{vm_memory_watermark_absolute_units_gib,
"vm_memory_high_watermark.absolute = 3GiB",
[{rabbit,[{vm_memory_high_watermark,{absolute,"3GiB"}}]}],
[]},
{vm_memory_watermark_absolute_units_g,
"vm_memory_high_watermark.absolute = 10G",
[{rabbit,[{vm_memory_high_watermark,{absolute,"10G"}}]}],
[]},

{vm_memory_watermark_absolute_units_tb,
"vm_memory_high_watermark.absolute = 1TB",
[{rabbit,[{vm_memory_high_watermark,{absolute,"1TB"}}]}],
[]},
{vm_memory_watermark_absolute_units_tib,
"vm_memory_high_watermark.absolute = 1TiB",
[{rabbit,[{vm_memory_high_watermark,{absolute,"1TiB"}}]}],
[]},
{vm_memory_watermark_absolute_units_t,
"vm_memory_high_watermark.absolute = 1T",
[{rabbit,[{vm_memory_high_watermark,{absolute,"1T"}}]}],
[]},

{vm_memory_watermark_absolute_priority,
"vm_memory_high_watermark.absolute = 1073741824
vm_memory_high_watermark.relative = 0.4",
Expand Down
23 changes: 16 additions & 7 deletions deps/rabbit_common/src/rabbit_resource_monitor_misc.erl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,20 +17,29 @@ parse_information_unit(Value) when is_integer(Value) -> {ok, Value};
parse_information_unit(Value0) ->
Value = rabbit_data_coercion:to_list(Value0),
case re:run(Value,
"^(?<VAL>[0-9]+)(?<UNIT>kB|KB|MB|GB|kb|mb|gb|Kb|Mb|Gb|kiB|KiB|MiB|GiB|kib|mib|gib|KIB|MIB|GIB|k|K|m|M|g|G)?$",
"^(?<VAL>[0-9]+)(?<UNIT>kB|Ki|Mi|Gi|Ti|Pi|KB|MB|GB|TB|PB|kb|ki|mb|gb|tb|pb|Kb|Mb|Gb|Tb|Pb|kiB|KiB|MiB|GiB|TiB|PiB|kib|mib|gib|tib|pib|KIB|MIB|GIB|TIB|PIB|k|K|m|M|g|G|p|P)?$",
[{capture, all_but_first, list}]) of
{match, [[], _]} ->
{ok, list_to_integer(Value)};
{match, [Num]} ->
{ok, list_to_integer(Num)};
{match, [Num, Unit]} ->
%% Note: there is no industry standard on what K, M, G, T, P means (is G a gigabyte or a gibibyte?), so
%% starting with 3.13 we treat those the same way as Kubernetes does [1].
%%
%% 1. https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#meaning-of-memory
Multiplier = case Unit of
KiB when KiB =:= "k"; KiB =:= "kiB"; KiB =:= "K"; KiB =:= "KIB"; KiB =:= "kib" -> 1024;
MiB when MiB =:= "m"; MiB =:= "MiB"; MiB =:= "M"; MiB =:= "MIB"; MiB =:= "mib" -> 1024*1024;
GiB when GiB =:= "g"; GiB =:= "GiB"; GiB =:= "G"; GiB =:= "GIB"; GiB =:= "gib" -> 1024*1024*1024;
KB when KB =:= "KB"; KB =:= "kB"; KB =:= "kb"; KB =:= "Kb" -> 1000;
MB when MB =:= "MB"; MB =:= "mB"; MB =:= "mb"; MB =:= "Mb" -> 1000000;
GB when GB =:= "GB"; GB =:= "gB"; GB =:= "gb"; GB =:= "Gb" -> 1000000000
KiB when KiB =:= "kiB"; KiB =:= "KIB"; KiB =:= "kib"; KiB =:= "ki"; KiB =:= "Ki" -> 1024;
MiB when MiB =:= "MiB"; MiB =:= "MIB"; MiB =:= "mib"; MiB =:= "mi"; MiB =:= "Mi" -> 1024 * 1024;
GiB when GiB =:= "GiB"; GiB =:= "GIB"; GiB =:= "gib"; GiB =:= "gi"; GiB =:= "Gi" -> 1024 * 1024 * 1024;
TiB when TiB =:= "TiB"; TiB =:= "TIB"; TiB =:= "tib"; TiB =:= "ti"; TiB =:= "Ti" -> 1024 * 1024 * 1024 * 1024;
PiB when PiB =:= "PiB"; PiB =:= "PIB"; PiB =:= "pib"; PiB =:= "pi"; PiB =:= "Pi" -> 1024 * 1024 * 1024 * 1024 * 2014;

KB when KB =:= "k"; KB =:= "K"; KB =:= "KB"; KB =:= "kB"; KB =:= "kb"; KB =:= "Kb" -> 1000;
MB when MB =:= "m"; MB =:= "M"; MB =:= "MB"; MB =:= "mB"; MB =:= "mb"; MB =:= "Mb" -> 1000_000;
GB when GB =:= "g"; GB =:= "G"; GB =:= "GB"; GB =:= "gB"; GB =:= "gb"; GB =:= "Gb" -> 1000_000_000;
TB when TB =:= "t"; TB =:= "T"; TB =:= "TB"; TB =:= "tB"; TB =:= "tb"; TB =:= "Tb" -> 1000_000_000_000;
PB when PB =:= "p"; PB =:= "P"; PB =:= "PB"; PB =:= "pB"; PB =:= "pb"; PB =:= "Pb" -> 1000_000_000_000_000
end,
{ok, list_to_integer(Num) * Multiplier};
nomatch ->
Expand Down
8 changes: 6 additions & 2 deletions deps/rabbitmq_cli/lib/rabbitmq/cli/ctl/commands/status_command.ex
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,8 @@ defmodule RabbitMQ.CLI.Ctl.Commands.StatusCommand do
:ok

false ->
{:validation_failure, "unit '#{unit}' is not supported. Please use one of: bytes, mb, gb"}
{:validation_failure,
"unit '#{unit}' is not supported. Please use one of: bytes, mb, mib, gb, gib, tb, tib"}
end
end

Expand Down Expand Up @@ -212,7 +213,10 @@ defmodule RabbitMQ.CLI.Ctl.Commands.StatusCommand do

def usage_additional() do
[
["--unit <bytes | mb | gb>", "byte multiple (bytes, megabytes, gigabytes) to use"],
[
"--unit <bytes | mb | mib | gb | gib>",
"byte multiple (bytes, megabytes, gigabytes) to use"
],
["--formatter <json | erlang>", "alternative formatter (JSON, Erlang terms)"]
]
end
Expand Down
67 changes: 66 additions & 1 deletion deps/rabbitmq_cli/lib/rabbitmq/cli/information_unit.ex
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,18 +11,42 @@ defmodule RabbitMQ.CLI.InformationUnit do
@megabyte_bytes @kilobyte_bytes * 1000
@gigabyte_bytes @megabyte_bytes * 1000
@terabyte_bytes @gigabyte_bytes * 1000
@petabyte_bytes @terabyte_bytes * 1000

@kibibyte_bytes 1024
@mebibyte_bytes @kibibyte_bytes * 1024
@gibibyte_bytes @mebibyte_bytes * 1024
@tebibyte_bytes @gibibyte_bytes * 1024
@pebibyte_bytes @tebibyte_bytes * 1024

def known_units() do
MapSet.new([
"bytes",
"k",
"kb",
"ki",
"kib",
"kilobytes",
"m",
"mb",
"mi",
"mib",
"megabytes",
"g",
"gb",
"gi",
"gib",
"gigabytes",
"t",
"tb",
"terabytes"
"ti",
"tib",
"terabytes",
"p",
"pb",
"pi",
"pib",
"petabytes"
])
end

Expand Down Expand Up @@ -50,23 +74,64 @@ defmodule RabbitMQ.CLI.InformationUnit do
Float.round(bytes / @kilobyte_bytes, 4)
end

defp do_convert(bytes, "k"), do: do_convert(bytes, "kb")

defp do_convert(bytes, "ki") do
Float.round(bytes / @kibibyte_bytes, 4)
end

defp do_convert(bytes, "kib"), do: do_convert(bytes, "ki")
defp do_convert(bytes, "kilobytes"), do: do_convert(bytes, "kb")

defp do_convert(bytes, "mb") do
Float.round(bytes / @megabyte_bytes, 4)
end

defp do_convert(bytes, "m"), do: do_convert(bytes, "mb")

defp do_convert(bytes, "mi") do
Float.round(bytes / @mebibyte_bytes, 4)
end

defp do_convert(bytes, "mib"), do: do_convert(bytes, "mi")
defp do_convert(bytes, "megabytes"), do: do_convert(bytes, "mb")

defp do_convert(bytes, "gb") do
Float.round(bytes / @gigabyte_bytes, 4)
end

defp do_convert(bytes, "g"), do: do_convert(bytes, "gb")

defp do_convert(bytes, "gi") do
Float.round(bytes / @gigabyte_bytes, 4)
end

defp do_convert(bytes, "gib"), do: do_convert(bytes, "gi")
defp do_convert(bytes, "gigabytes"), do: do_convert(bytes, "gb")

defp do_convert(bytes, "tb") do
Float.round(bytes / @terabyte_bytes, 4)
end

defp do_convert(bytes, "t"), do: do_convert(bytes, "tb")

defp do_convert(bytes, "ti") do
Float.round(bytes / @tebibyte_bytes, 4)
end

defp do_convert(bytes, "tib"), do: do_convert(bytes, "ti")
defp do_convert(bytes, "terabytes"), do: do_convert(bytes, "tb")

defp do_convert(bytes, "pb") do
Float.round(bytes / @petabyte_bytes, 4)
end

defp do_convert(bytes, "p"), do: do_convert(bytes, "pb")

defp do_convert(bytes, "pi") do
Float.round(bytes / @pebibyte_bytes, 4)
end

defp do_convert(bytes, "pib"), do: do_convert(bytes, "pi")
defp do_convert(bytes, "petabytes"), do: do_convert(bytes, "pb")
end
6 changes: 3 additions & 3 deletions deps/rabbitmq_ct_helpers/src/rabbit_ct_config_schema.erl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ snippet_id(A) when is_atom(A) ->
snippet_id(L) when is_list(L) ->
L.

test_snippet(Config, Snippet, Expected, _Plugins) ->
test_snippet(Config, Snippet = {SnipID, _, _}, Expected, _Plugins) ->
{ConfFile, AdvancedFile} = write_snippet(Config, Snippet),
%% We ignore the rabbit -> log portion of the config on v3.9+, where the lager
%% dependency has been dropped
Expand All @@ -55,8 +55,8 @@ test_snippet(Config, Snippet, Expected, _Plugins) ->
case Exp of
Gen -> ok;
_ ->
ct:pal("Expected: ~tp~ngenerated: ~tp", [Expected, Generated]),
ct:pal("Expected (sorted): ~tp~ngenerated (sorted): ~tp", [Exp, Gen]),
ct:pal("Snippet ~tp. Expected: ~tp~ngenerated: ~tp", [SnipID, Expected, Generated]),
ct:pal("Snippet ~tp. Expected (sorted): ~tp~ngenerated (sorted): ~tp", [SnipID, Exp, Gen]),
error({config_mismatch, Snippet, Exp, Gen})
end.

Expand Down