Make length of block size histogram configurable.

Author: Nikratio

filters/stats/nbdkit-stats-filter.pod

@@ -4,7 +4,8 @@ nbdkit-stats-filter - display statistics about operations
 
 =head1 SYNOPSIS
 
- nbdkit --filter=stats PLUGIN statsfile=FILE [statsappend=true]
+ nbdkit --filter=stats PLUGIN statsfile=FILE
+                       [statsappend=true] [statsthreshold=PERCENTILE]
 
 =head1 DESCRIPTION
 
@@ -15,53 +16,47 @@ are written to a file once when nbdkit exits.
 =head1 EXAMPLE OUTPUT
 
  # nbdkit --filter=exitfirst --filter=stats memory 25G statsfile=example.txt
- # nbd-client localhost /dev/nbd1 && mkfs.ext4 /dev/nbd1 && nbd-client -d /dev/nbd1
+ # nbd-client localhost /dev/nbd1 && mkfs.ext4 /dev/nbd1 &&
+   nbd-client -d /dev/nbd1
  [....]
  # cat example.txt
  total: 1207 ops, 24.756059 s, 25.13 GiB, 1.02 GiB/s
  read: 101 ops, 0.000322 s, 1.64 MiB, 4.99 GiB/s op, 68.02 KiB/s total
  write: 1088 ops, 0.067040 s, 132.38 MiB, 1.93 GiB/s op, 5.35 MiB/s total
  trim: 14 ops, 0.002605 s, 25.00 GiB, 9596.93 GiB/s op, 1.01 GiB/s total
  flush: 4 ops, 0.000001 s, 0 bytes, 0 bytes/s op, 0 bytes/s total
-
- READ Request sizes (top 28):
+ 
+ READ Request sizes:
      blocksize     request count
-          4096                67 (66.34%)
-         16384                 7 (6.93%)
-          8192                 3 (2.97%)
-         24576                 3 (2.97%)
-         61440                 3 (2.97%)
-         20480                 2 (1.98%)
-         28672                 2 (1.98%)
-         36864                 2 (1.98%)
-         40960                 2 (1.98%)
-         57344                 2 (1.98%)
-         65536                 2 (1.98%)
-         69632                 2 (1.98%)
-         32768                 1 (0.99%)
-        122880                 1 (0.99%)
-        126976                 1 (0.99%)
-        131072                 1 (0.99%)
-
- WRITE Request sizes (top 28):
+          4096                93 (68.38%)
+         61440                 5 (3.68%)
+         24576                 5 (3.68%)
+          8192                 5 (3.68%)
+         16384                 5 (3.68%)
+        126976                 3 (2.21%)
+        258048                 3 (2.21%)
+         57344                 2 (1.47%)
+        262144                 2 (1.47%)
+         20480                 2 (1.47%)
+         86016                 2 (1.47%)
+         40960                 2 (1.47%)
+      (others)                 7 (5.15%)
+ 
+ WRITE Request sizes:
      blocksize     request count
-        131072              1056 (97.06%)
-          4096                18 (1.65%)
-         20480                10 (0.92%)
-         24576                 2 (0.18%)
-          8192                 1 (0.09%)
-         65536                 1 (0.09%)
-
- TRIM Request sizes (top 28):
+        524288               264 (89.19%)
+          4096                18 (6.08%)
+      (others)                14 (4.73%)
+ 
+ TRIM Request sizes:
      blocksize     request count
     2147483648                12 (85.71%)
-      16777216                 1 (7.14%)
     1056964608                 1 (7.14%)
-
- ZERO Request sizes (top 28):
+      (others)                 1 (7.14%)
+ 
+ ZERO Request sizes:
      blocksize     request count
 
-
 =head1 PARAMETERS
 
 =over 4
@@ -76,6 +71,12 @@ This parameter is required.
 
 If set then we append to the file instead of replacing it.
 
+=item B<statsthreshold=>PERCENTILE
+
+Percentile of requests for which block size histogram should be printed.
+Set to zero to disable collection of block size statistics. Set to 100
+to get a list of every block size encountered. Default: 95.
+
 =back
 
 =head1 FILES
@@ -104,6 +105,8 @@ L<nbdkit-log-filter(1)>.
 
 Richard W.M. Jones
 
+Nikolaus Rath
+
 =head1 COPYRIGHT
 
-Copyright (C) 2019 Red Hat Inc.
+Copyright (C) 2019-2022 Red Hat Inc.

filters/stats/stats.cpp

@@ -59,6 +59,7 @@ static char *filename;
 static bool append;
 static FILE *fp;
 static struct timeval start_t;
+static double print_threshold = 0.95;
 
 typedef struct {
   const char *name;
@@ -158,7 +159,7 @@ static void
 inc_blksize_ctr (blksize_hist_t &hist, size_t blksize)
 {
   static bool out_of_memory = false;
-  if (out_of_memory)
+  if (out_of_memory || print_threshold == 0)
     return;
 
   ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&lock);
@@ -175,7 +176,7 @@ inc_blksize_ctr (blksize_hist_t &hist, size_t blksize)
 }
 
 static void
-print_histogram (const blksize_hist_t& hist, int count)
+print_histogram (const blksize_hist_t &hist)
 {
   double total = 0;
   for (auto el : hist) {
@@ -184,38 +185,48 @@ print_histogram (const blksize_hist_t& hist, int count)
 
   // Sort
   auto pairs = std::vector<std::pair<size_t, size_t>> (hist.begin(), hist.end());
-  std::sort(pairs.begin(), pairs.end(),
-    [](decltype(pairs[0]) a, decltype(pairs[0]) b) {
-      return a.second > b.second;
-    });
-
-  int i = 0;
+  std::sort (pairs.begin(), pairs.end(),
+             [](decltype(pairs[0]) a, decltype(pairs[0]) b) {
+               return a.second > b.second;
+             });
+
+  // Print values until we have covered *print_threshold* percent of
+  // requests.
+  size_t to_print = static_cast<ssize_t>(print_threshold * total);
+  size_t printed = 0;
   for (auto el : pairs) {
-    if (++i >= count)
+    if (printed >= to_print) {
+      size_t requests = total - printed;
+      fprintf (fp, "     (others)         %9zu (%.2f%%)\n",
+               requests, static_cast<double>(requests) / total * 100);
       break;
+    }
+    size_t blocksize = el.first;
+    size_t requests = el.second;
     fprintf (fp, "%13zu         %9zu (%.2f%%)\n",
-      el.first, el.second, static_cast<double>(el.second) / total * 100);
+             blocksize, requests, static_cast<double>(requests) / total * 100);
+    printed += requests;
   }
 }
 
 static void
 print_blocksize_stats (void)
 {
-  fprintf (fp, "\nREAD Request sizes (top 28):\n");
+  fprintf (fp, "\nREAD Request sizes:\n");
   fprintf (fp, "    blocksize     request count\n");
-  print_histogram (blksize_pread_st, 28);
-  
-  fprintf (fp, "\nWRITE Request sizes (top 28):\n");
+  print_histogram (blksize_pread_st);
+
+  fprintf (fp, "\nWRITE Request sizes:\n");
   fprintf (fp, "    blocksize     request count\n");
-  print_histogram (blksize_pwrite_st, 28);
+  print_histogram (blksize_pwrite_st);
 
-  fprintf (fp, "\nTRIM Request sizes (top 28):\n");
+  fprintf (fp, "\nTRIM Request sizes:\n");
   fprintf (fp, "    blocksize     request count\n");
-  print_histogram (blksize_trim_st, 28);
-  
-  fprintf (fp, "\nZERO Request sizes (top 28):\n");
+  print_histogram (blksize_trim_st);
+
+  fprintf (fp, "\nZERO Request sizes:\n");
   fprintf (fp, "    blocksize     request count\n");
-  print_histogram (blksize_zero_st, 28);
+  print_histogram (blksize_zero_st);
 }
 
 static inline void
@@ -229,7 +240,8 @@ print_stats (int64_t usecs)
   print_stat (&extents_st, usecs);
   print_stat (&cache_st,   usecs);
   print_stat (&flush_st,   usecs);
-  print_blocksize_stats();
+  if (print_threshold != 0)
+    print_blocksize_stats ();
   fflush (fp);
 }
 
@@ -271,6 +283,18 @@ stats_config (nbdkit_next_config *next, nbdkit_backend *nxdata,
     append = r;
     return 0;
   }
+  else if (strcmp (key, "statsthreshold") == 0) {
+    int ival;
+    r = nbdkit_parse_int ("printing threshold", value, &ival);
+    if (r == -1)
+      return -1;
+    if (ival > 100 or ival < 0) {
+      nbdkit_error ("statsthreshold must be between 0 and 100 (percent)");
+      return -1;
+    }
+    print_threshold = static_cast<double>(ival) / 100;
+    return 0;
+  }
 
   return next (nxdata, key, value);
 }