QQ: default to compressed mem tables and set a wal max entries default

Compressed ETS tables may introduce a small throughput penalty (low single
digit %) but can reduce peak Ra memory use by 30-50%.

Also set a default wal_max_entries value to avoid mem tables growing
too large when using very small message sizes (as more than 1M tiny
messages can easily fit into one WAL file).

Ra 2.10.1 has a type spec fix needed.

Author: kjnilsson

MODULE.bazel

@@ -349,8 +349,8 @@ erlang_package.hex_package(
     name = "ra",
     build_file = "@rabbitmq-server//bazel:BUILD.ra",
     pkg = "ra",
-    sha256 = "0c9c02e8db48a9ed8f9b56e3b105dc25c72fde01c3dc07e2dc4f0618545595cd",
-    version = "2.10.0",
+    sha256 = "ab0e53c69b6964b237d94a3ac8f2ae5ba5cdc2dc9ef73ef3731d85f0c01d35d0",
+    version = "2.10.1",
 )
 
 erlang_package.git_package(

deps/rabbit/src/rabbit_ra_systems.erl

@@ -23,6 +23,7 @@
 
 -define(COORD_WAL_MAX_SIZE_B, 64_000_000).
 -define(QUORUM_AER_MAX_RPC_SIZE, 16).
+-define(QUORUM_DEFAULT_WAL_MAX_ENTRIES, 500_000).
 
 -spec setup() -> ok | no_return().
 
@@ -111,13 +112,22 @@ get_config(quorum_queues = RaSystem) ->
     DefaultConfig = get_default_config(),
     Checksums = application:get_env(rabbit, quorum_compute_checksums, true),
     WalChecksums = application:get_env(rabbit, quorum_wal_compute_checksums, Checksums),
-    SegmentChecksums = application:get_env(rabbit, quorum_segment_compute_checksums, Checksums),
+    SegmentChecksums = application:get_env(rabbit, quorum_segment_compute_checksums,
+                                           Checksums),
+    WalMaxEntries = case DefaultConfig of
+                        #{wal_max_entries := MaxEntries}
+                          when is_integer(MaxEntries) ->
+                            MaxEntries;
+                        _ ->
+                            ?QUORUM_DEFAULT_WAL_MAX_ENTRIES
+                    end,
     AERBatchSize = application:get_env(rabbit, quorum_max_append_entries_rpc_batch_size,
                                        ?QUORUM_AER_MAX_RPC_SIZE),
-    CompressMemTables = application:get_env(rabbit, quorum_compress_mem_tables, false),
+    CompressMemTables = application:get_env(rabbit, quorum_compress_mem_tables, true),
     DefaultConfig#{name => RaSystem,
                    default_max_append_entries_rpc_batch_size => AERBatchSize,
                    wal_compute_checksums => WalChecksums,
+                   wal_max_entries => WalMaxEntries,
                    segment_compute_checksums => SegmentChecksums,
                    compress_mem_tables => CompressMemTables};
 get_config(coordination = RaSystem) ->

rabbitmq-components.mk

@@ -123,7 +123,7 @@ dep_khepri = hex 0.13.0
 dep_khepri_mnesia_migration = hex 0.4.0
 dep_looking_glass = git https://github.com/rabbitmq/looking_glass.git main
 dep_prometheus = hex 4.11.0
-dep_ra = hex 2.10.0
+dep_ra = hex 2.10.1
 dep_ranch = hex 2.1.0
 dep_recon = hex 2.5.3
 dep_redbug = hex 2.0.7