Only report unstable tests and don't fail the build (mysql#84)

* Only report unstable tests and don't fail the build

* Show contents of log files after test failures

* Add a --flakes option

* Start a list of known flaky tests: flaky-tests.txt

* Add more flaky tests from mysql#82

* One more flaky test

* Fix and check the path to flaky-tests.txt

* Ensure --flakes gets a non-empty arg

Oddly, `--flakes` is ignored, as is `--flakes=""`, because getopt doesn't
call the handler in either of those cases.

However: `--flakes ""` will now trigger the `die` added here.  The idea
here is to catch the error corrected in fe801e631d5a.

* Fix path for flaky tests file

This fixes the path for the flaky tests, fixes the logic for reading the
hash, decreases parallelism further and adds a bunch of tests to the
flakes set.

* Add flaky tests seen in mysql#80

---------

Co-authored-by: Patrick Reynolds <[email protected]>
Signed-off-by: Dirkjan Bussink <[email protected]>

Author: dbussink

flaky-tests.txt

@@ -0,0 +1,20 @@
+innodb.autoinc_persist_debug
+innodb.disable_log_encryption
+main.async_client
+main.information_schema_stats_expiry_debug
+main.join
+main.join_cache_bka
+main.join_cache_bka_nobnl
+main.join_outer
+main.join_outer_bka_nobnl
+main.join_cache_bnl
+main.query_expression
+main.sp-threads
+perfschema.idx_compare_mems_by_thread_by_event_name
+perfschema.idx_compare_threads
+rpl.rpl_relay_log_space_limit_deadlock
+rpl_gtid.rpl_apply_binlog_with_same_server_id
+rpl_gtid.rpl_sync_relay_log_info
+sysschema.pr_diagnostics
+sysschema.v_io_global_by_file_by_bytes
+x.notice_timeout

mysql-test/lib/mtr_report.pm

@@ -572,14 +572,18 @@ sub mtr_report_stats ($$;$) {
       $tot_failed++;
 
       if ($::opt_report_unstable_tests and defined $tinfo->{retries}) {
-        my $num_passed = $tinfo->{retries} - $tinfo->{failures} - 1;
-        # Tests which exhibit both passing and failing behaviour with
-        # the same code are unstable tests. The level of instability is
-        # not restricted i.e. a failed test which is successful on at
-        # least one retry is marked unstable.
-        if ($num_passed > 0) {
-          $tot_unstable++;
-          $tinfo->{'result'} = 'MTR_RES_UNSTABLE';
+        if (!%::opt_flakes || $::opt_flakes{$tinfo->{name}}) {
+          my $num_passed = $tinfo->{retries} - $tinfo->{failures} - 1;
+          # Tests which exhibit both passing and failing behaviour with
+          # the same code are unstable tests. The level of instability is
+          # not restricted i.e. a failed test which is successful on at
+          # least one retry is marked unstable.
+          if ($num_passed > 0) {
+            $tot_unstable++;
+            $tinfo->{'result'} = 'MTR_RES_UNSTABLE';
+          }
+        } else {
+          print "Unstable test '$tinfo->{name}' not included in --flakes\n";
         }
       }
     } elsif ($tinfo->{'result'} eq 'MTR_RES_SKIPPED') {

mysql-test/mysql-test-run.pl

@@ -218,6 +218,7 @@
 our $opt_no_skip;
 our $opt_record;
 our $opt_report_unstable_tests;
+our %opt_flakes;
 our $opt_skip_combinations;
 our $opt_suites;
 our $opt_suite_opt;
@@ -1030,6 +1031,14 @@ ($$$)
               }
 
               mtr_report(" - the logfile can be found in '$logfilepath'\n");
+              if (open(my $file, '<', $logfilepath)) {
+                print "---[ contents of '$logfilepath' ]---\n";
+                while (my $line = <$file>) {
+                  print $line;
+                }
+                close($file);
+                print "------------------------------------------------------------------------\n";
+              }
 
               # Move any core files from e.g. mysqltest
               foreach my $coref (glob("core*"), glob("*.dmp")) {
@@ -1759,6 +1768,11 @@ sub command_line_setup {
     'disk-usage!'           => \&report_option,
     'enable-disabled'       => \&collect_option,
     'fast!'                  => \$opt_fast,
+    'flakes=s'              => sub {
+      my ($opt, $value) = @_;
+      die "--flakes requires an argument" unless $value;
+      $opt_flakes{$_} = 1 for (split(' ', $value));
+    },
     'force-restart'         => \$opt_force_restart,
     'help|h'                => \$opt_usage,
     'keep-ndbfs'            => \$opt_keep_ndbfs,
@@ -7960,6 +7974,7 @@ ($)
   debug-server          Use debug version of server, but without turning on
                         tracing.
   debugger=NAME         Start mysqld in the selected debugger.
+  flakes                List of tests allowed to pass by --report-unstable-tests
   gdb                   Start the mysqld(s) in gdb.
   lldb                  Start the mysqld(s) in lldb.
   manual-boot-gdb       Let user manually start mysqld in gdb, during initialize