[Fortran/gfortran] Correctly handle "shouldfail" tests

Correctly handle the DejaGNU shouldfail annotation which asserts that the test
is expected to fail at runtime. Disable tests that now fail. These may be
actual bugs in the runtime or a case of flang behaving differently from
gfortran and will need to be investigated.

Author: tarunprabhu

Fortran/gfortran/CMakeLists.txt

@@ -530,8 +530,9 @@ endfunction()
 # Setup an "execute" test. In the case of multi-file tests, MAIN will be the
 # main file. For multi-file tests, OTHERS will be the remaining files needed by
 # the test. FFLAGS are additional compiler flags needed by the test. LDFLAGS
-# are the other linker flags needed by the test.
-function(gfortran_add_execute_test main others fflags ldflags)
+# are the other linker flags needed by the test. If EXPECT_ERROR evaluates to
+# true, the test is expected to fail.
+function(gfortran_add_execute_test expect_error main others fflags ldflags)
   # PREFIX_EXECUTE will have been defined in the subdirectory from which this
   # function is called.
   gfortran_unique_target_name("${PREFIX_EXECUTE}" "${main}" target)
@@ -540,6 +541,20 @@ function(gfortran_add_execute_test main others fflags ldflags)
 
   llvm_test_executable_no_test(${target} ${main} ${others})
   llvm_test_run(WORKDIR "%S/${working_dir_name}")
+  # FIXME: All we need is to negate the exit code. LLVM's not utility can do
+  # this, but if it is not already installed on the system, it will have to be
+  # obtained from LLVM's build directory. It is not clear that we can rely on
+  # this when running the test suite.
+  if (WIN32)
+    # Probably need to check %errorlevel% here.
+    message(FATAL_ERROR "Verification not implemented on Windows.")
+  else ()
+    if (expect_error)
+      llvm_test_verify(${TESTCMD} $? -ne 0)
+    else ()
+      llvm_test_verify(${TESTCMD} $? -eq 0)
+    endif ()
+  endif ()
   llvm_add_test_for_target(${target})
 
   target_include_directories(${target}
@@ -576,6 +591,9 @@ function(gfortran_add_execute_tests_from tests)
   cmake_parse_arguments(GFORTRAN "" "" "FFLAGS;LDFLAGS" ${ARGN})
 
   foreach(file ${tests})
+    # Whether this test is expected to pass or fail.
+    set(expect_error OFF)
+
     # The file containing the "run" directive will be the main file.
     set(main "")
 
@@ -611,13 +629,15 @@ function(gfortran_add_execute_tests_from tests)
         separate_arguments(file_fflags UNIX_COMMAND ${CMAKE_MATCH_2})
         list(REMOVE_ITEM file_fflags ${FLANG_ERRORING_FFLAGS})
         list(APPEND fflags ${file_fflags})
+      elseif (line MATCHES "[{][ ]*dg-shouldfail[ ]+(.*)[}]")
+        set(expect_error ON)
       endif()
     endforeach()
 
     # Since any dependent files could also be processed by this function, there
     # is no guarantee that main will have been set.
     if (main)
-      gfortran_add_execute_test(${main} "${others}" "${fflags}" "${ldflags}")
+      gfortran_add_execute_test(${expect_error} ${main} "${others}" "${fflags}" "${ldflags}")
     endif()
   endforeach()
 endfunction()
@@ -632,7 +652,7 @@ function(gfortran_add_execute_tests tests)
   list(APPEND ldflags ${GFORTRAN_LDFLAGS})
 
   foreach(file ${tests})
-    gfortran_add_execute_test(${file} "" "${fflags}" "${ldflags}")
+    gfortran_add_execute_test(OFF ${file} "" "${fflags}" "${ldflags}")
   endforeach()
 endfunction()
 

Fortran/gfortran/regression/DisabledFiles.cmake

@@ -2082,6 +2082,77 @@ file(GLOB FAILING_FILES CONFIGURE_DEPENDS
   write_check.f90
   zero_sized_1.f90
 
+  # ---------------------------------------------------------------------------
+  #
+  # These tests are expected to fail at runtime, but pass instead.
+  #
+  allocate_error_1.f90
+  bounds_check_7.f90
+  bounds_check_10.f90
+  all_bounds_1.f90
+  bounds_check_12.f90
+  bounds_check_array_ctor_1.f90
+  bounds_check_array_ctor_2.f90
+  bounds_check_array_ctor_4.f90
+  bounds_check_array_ctor_6.f90
+  bounds_check_array_ctor_7.f90
+  bounds_check_array_ctor_8.f90
+  bounds_check_fail_3.f90
+  bounds_check_fail_4.f90
+  bounds_check_strlen_1.f90
+  bounds_check_strlen_2.f90
+  bounds_check_strlen_3.f90
+  bounds_check_strlen_4.f90
+  bounds_check_strlen_5.f90
+  bounds_check_strlen_7.f90
+  char_bounds_check_fail_1.f90
+  char_pointer_assign_4.f90
+  char_pointer_assign_5.f90
+  cshift_bounds_2.f90
+  deallocate_error_1.f90
+  deallocate_error_2.f90
+  do_check_2.f90
+  do_check_3.f90
+  do_check_4.f90
+  do_check_11.f90
+  do_check_12.f90
+  endfile_4.f90
+  fmt_g0_2.f08
+  inline_sum_bounds_check_1.f90
+  inline_sum_bounds_check_2.f90
+  io_real_boz2.f90
+  io_real_boz_4.f90
+  io_real_boz_5.f90
+  matmul_bounds_2.f90
+  matmul_bounds_3.f90
+  matmul_bounds_4.f90
+  matmul_bounds_5.f90
+  matmul_bounds_8.f90
+  matmul_bounds_10.f90
+  maxloc_bounds_1.f90
+  maxloc_bounds_2.f90
+  maxloc_bounds_3.f90
+  maxloc_bounds_4.f90
+  maxloc_bounds_5.f90
+  maxloc_bounds_7.f90
+  maxloc_bounds_8.f90
+  merge_char_3.f90
+  no_unit_error_1.f90
+  pack_bounds_1.f90
+  pointer_check_10.f90
+  pointer_remapping_6.f08
+  pr96436_6.f90
+  pr96436_7.f90
+  pr96436_8.f90
+  pr96436_9.f90
+  pr96436_10.f90
+  recursive_check_9.f90
+  recursive_check_11.f90
+  recursive_check_13.f90
+  spread_bounds_1.f90
+  transpose_2.f90
+  unpack_bounds_1.f90
+
   # ---------------------------------------------------------------------------
   #
   # This test fails with optimizations enabled, but succeeds when compiled
@@ -2689,28 +2760,6 @@ file(GLOB FAILING_FILES CONFIGURE_DEPENDS
   openacc-define-3.f90
   openmp-define-3.f90
 
-  # Tests looking for runtime errors (e.g., bound checks). Correctly
-  # caught by flang runtime when it is used for array assignments.
-  allocate_error_6.f90
-  allocate_with_source_22.f03
-  allocate_with_source_23.f03
-  all_bounds_1.f90
-  associate_32.f03
-  bounds_check_12.f90
-  bounds_check_array_ctor_4.f90
-  bounds_check_fail_3.f90
-  inline_matmul_1.f90
-  maxloc_bounds_1.f90
-  maxloc_bounds_2.f90
-  maxloc_bounds_3.f90
-  maxloc_bounds_4.f90
-  maxloc_bounds_5.f90
-  maxloc_bounds_7.f90
-  maxloc_bounds_8.f90
-  pack_bounds_1.f90
-  ptr_func_assign_1.f08
-  spread_bounds_1.f90
-
   # Bad test, assigning an 11 elements array to a 12 elements array.
   transfer_array_intrinsic_4.f90