Try evaluating byte size independently of time window

also increase max size

Author: kjnilsson

deps/rabbit/src/rabbit_fifo.erl

@@ -2948,28 +2948,33 @@ do_checkpoints(Ts, #checkpoint{index = ChIdx,
     MsgsTot = messages_total(MacState),
     %% more than 64MB (by default) of message data has been written to the log
     %% best take a checkpoint
-    EnoughDataWritten = BytesIn - LastBytesIn > ?CHECK_MAX_BYTES,
 
     {CheckMinInterval, CheckMinIndexes, CheckMaxIndexes} =
         persistent_term:get(quorum_queue_checkpoint_config,
                             {?CHECK_MIN_INTERVAL_MS, ?CHECK_MIN_INDEXES,
                              ?CHECK_MAX_INDEXES}),
+
+    %% scale the bytes limit as the backlog increases
+    MaxBytesFactor = max(1, MsgsTot / CheckMaxIndexes),
+    EnoughDataWritten = BytesIn - LastBytesIn > (?CHECK_MAX_BYTES * MaxBytesFactor),
     EnoughTimeHasPassed = TimeSince > CheckMinInterval,
 
-    case EnoughTimeHasPassed andalso
-         (
-          %% condition 1: enough indexes have been committed since the last
-          %% checkpoint
-          (IndexesSince > MinIndexes) orelse
-          %% condition 2: the queue is empty and _some_ commands
-          %% have been applied since the last checkpoint
-          (MsgsTot == 0 andalso IndexesSince > 32) orelse
-          %% condition 3: enough message data has been written to warrant a new
-          %% checkpoint
-          EnoughDataWritten orelse
-          %% force was requested, e.g. after a purge
-          Force
-         ) of
+    case (EnoughTimeHasPassed andalso
+          (
+           %% condition 1: enough indexes have been committed since the last
+           %% checkpoint
+           (IndexesSince > MinIndexes) orelse
+           %% condition 2: the queue is empty and _some_ commands
+           %% have been applied since the last checkpoint
+           (MsgsTot == 0 andalso IndexesSince > 32)
+          )
+         ) orelse
+         %% condition 3: enough message data has been written to warrant a new
+         %% checkpoint, this ignores the time windowing
+         EnoughDataWritten orelse
+         %% force was requested, e.g. after a purge
+         Force
+    of
         true ->
             %% take fewer checkpoints the more messages there are on queue
             NextIndexes = min(max(MsgsTot, CheckMinIndexes), CheckMaxIndexes),

deps/rabbit/src/rabbit_fifo.hrl

@@ -104,7 +104,7 @@
 -define(CHECK_MAX_INDEXES, 666_667).
 %% once these many bytes have been written since the last checkpoint
 %% we request a checkpoint irrespectively
--define(CHECK_MAX_BYTES, 64_000_000).
+-define(CHECK_MAX_BYTES, 128_000_000).
 
 -define(USE_AVG_HALF_LIFE, 10000.0).
 %% an average QQ without any message uses about 100KB so setting this limit