[Fortran/gfortran] Correctly handle "shouldfail" tests

Correctly handle the DejaGNU shouldfail annotation which asserts that the test
is expected to fail at runtime. Disable tests that now fail. These may be
actual bugs in the runtime or a case of flang behaving differently from
gfortran and will need to be investigated.

Author: tarunprabhu

Fortran/gfortran/CMakeLists.txt

@@ -68,6 +68,7 @@ set(FLANG_ERRORING_FFLAGS
   -fcheck-array-temporaries
   -fcheck=bounds
   -fcheck=do
+  -fcheck=mem
   -fcheck=recursion
   -fcoarray=lib
   -fcray-pointer
@@ -530,16 +531,23 @@ endfunction()
 # Setup an "execute" test. In the case of multi-file tests, MAIN will be the
 # main file. For multi-file tests, OTHERS will be the remaining files needed by
 # the test. FFLAGS are additional compiler flags needed by the test. LDFLAGS
-# are the other linker flags needed by the test.
-function(gfortran_add_execute_test main others fflags ldflags)
+# are the other linker flags needed by the test. If EXPECT_ERROR evaluates to
+# true, the test is expected to fail.
+function(gfortran_add_execute_test expect_error main others fflags ldflags)
   # PREFIX_EXECUTE will have been defined in the subdirectory from which this
   # function is called.
   gfortran_unique_target_name("${PREFIX_EXECUTE}" "${main}" target)
   gfortran_make_working_dir("${target}" working_dir)
   get_filename_component(working_dir_name "${working_dir}" NAME)
 
   llvm_test_executable_no_test(${target} ${main} ${others})
-  llvm_test_run(WORKDIR "%S/${working_dir_name}")
+  if (expect_error)
+    llvm_test_run(
+      EXECUTABLE "%b/not --crash %S/${target}"
+      WORKDIR "%S/${working_dir_name}")
+  else ()
+    llvm_test_run(WORKDIR "%S/${working_dir_name}")
+  endif ()
   llvm_add_test_for_target(${target})
 
   target_include_directories(${target}
@@ -576,6 +584,9 @@ function(gfortran_add_execute_tests_from tests)
   cmake_parse_arguments(GFORTRAN "" "" "FFLAGS;LDFLAGS" ${ARGN})
 
   foreach(file ${tests})
+    # Whether this test is expected to pass or fail.
+    set(expect_error OFF)
+
     # The file containing the "run" directive will be the main file.
     set(main "")
 
@@ -611,13 +622,15 @@ function(gfortran_add_execute_tests_from tests)
         separate_arguments(file_fflags UNIX_COMMAND ${CMAKE_MATCH_2})
         list(REMOVE_ITEM file_fflags ${FLANG_ERRORING_FFLAGS})
         list(APPEND fflags ${file_fflags})
+      elseif (line MATCHES "[{][ ]*dg-shouldfail[ ]+(.*)[}]")
+        set(expect_error ON)
       endif()
     endforeach()
 
     # Since any dependent files could also be processed by this function, there
     # is no guarantee that main will have been set.
     if (main)
-      gfortran_add_execute_test(${main} "${others}" "${fflags}" "${ldflags}")
+      gfortran_add_execute_test(${expect_error} ${main} "${others}" "${fflags}" "${ldflags}")
     endif()
   endforeach()
 endfunction()
@@ -632,7 +645,7 @@ function(gfortran_add_execute_tests tests)
   list(APPEND ldflags ${GFORTRAN_LDFLAGS})
 
   foreach(file ${tests})
-    gfortran_add_execute_test(${file} "" "${fflags}" "${ldflags}")
+    gfortran_add_execute_test(OFF ${file} "" "${fflags}" "${ldflags}")
   endforeach()
 endfunction()
 

Fortran/gfortran/regression/DisabledFiles.cmake

@@ -316,6 +316,8 @@ file(GLOB UNIMPLEMENTED_FILES CONFIGURE_DEPENDS
   allocate_with_source_14.f03
   allocate_with_source_17.f03
   allocate_with_source_21.f03
+  allocate_with_source_22.f03
+  allocate_with_source_23.f03
   allocate_with_source_25.f90
   allocate_with_source_8.f08
   associate_18.f08
@@ -534,6 +536,7 @@ file(GLOB UNIMPLEMENTED_FILES CONFIGURE_DEPENDS
   proc_ptr_comp_50.f90
   proc_ptr_comp_pass_3.f90
   proc_ptr_comp_pass_6.f90
+  ptr_func_assign_1.f08
   ptr_func_assign_2.f08
   ptr_func_assign_3.f08
   realloc_on_assign_20.f90
@@ -1858,17 +1861,41 @@ file(GLOB FAILING_FILES CONFIGURE_DEPENDS
   # require further analysis
   Wall.f90 # no warning for missing & on continuation line in char constant
   Wno-all.f90 # no warning for missing & on continuation line in char constant
+  allocate_error_1.f90
+  # flang throws a runtime error: "mismatching element counts in array
+  # assignment", but gfortran does not for associate_32.f03
+  associate_32.f03
   auto_save_2.f90 # requires -fno-automatic -finit-local-zero
   bessel_7.f90 # unclear
+  bounds_check_10.f90
+  bounds_check_7.f90
+  bounds_check_array_ctor_1.f90
+  bounds_check_array_ctor_2.f90
+  bounds_check_array_ctor_6.f90
+  bounds_check_array_ctor_7.f90
+  bounds_check_array_ctor_8.f90
+  bounds_check_fail_4.f90
+  bounds_check_strlen_1.f90
+  bounds_check_strlen_2.f90
+  bounds_check_strlen_3.f90
+  bounds_check_strlen_4.f90
+  bounds_check_strlen_5.f90
+  bounds_check_strlen_7.f90
+  char_bounds_check_fail_1.f90
+  char_pointer_assign_4.f90
+  char_pointer_assign_5.f90
   check_bits_1.f90 # requires -fcheck=bits to catch ISHFTC runtime error
   check_bits_2.f90 # requires -fcheck=bits to catch ISHFTC runtime error
   cr_lf.f90
+  cshift_bounds_2.f90
   cshift_bounds_3.f90
   cshift_bounds_4.f90
   cshift_large_1.f90
   data_char_1.f90
   data_char_3.f90
   deallocate_alloc_opt_3.f90
+  deallocate_error_1.f90
+  deallocate_error_2.f90
   deallocate_stat_2.f90
   deallocate_stat.f90
   dependency_45.f90
@@ -1877,9 +1904,15 @@ file(GLOB FAILING_FILES CONFIGURE_DEPENDS
   dim_sum_3.f90
   direct_io_7.f90
   do_check_1.f90
+  do_check_11.f90
+  do_check_12.f90
+  do_check_2.f90
+  do_check_3.f90
+  do_check_4.f90
   dollar_edit_descriptor_4.f
   elemental_dependency_1.f90
   endfile_3.f90
+  endfile_4.f90
   eoshift_bounds_1.f90
   eoshift_large_1.f90
   error_format.f90
@@ -1891,6 +1924,7 @@ file(GLOB FAILING_FILES CONFIGURE_DEPENDS
   fmt_f_default_field_width_1.f90
   fmt_f_default_field_width_2.f90
   fmt_g0_1.f08
+  fmt_g0_2.f08
   fmt_g_default_field_width_1.f90
   fmt_g_default_field_width_2.f90
   fmt_g.f
@@ -1910,7 +1944,12 @@ file(GLOB FAILING_FILES CONFIGURE_DEPENDS
   init_flag_6.f90
   init_flag_7.f90
   init_flag_9.f90
+  # flang throws a runtime error: "mismatching element counts in array
+  # assignment", but gfortran does not for inline_matmul_1.f90
+  inline_matmul_1.f90
   inline_matmul_15.f90
+  inline_sum_bounds_check_1.f90
+  inline_sum_bounds_check_2.f90
   inquire_13.f90
   inquire_15.f90
   inquire_5.f90
@@ -1924,7 +1963,10 @@ file(GLOB FAILING_FILES CONFIGURE_DEPENDS
   internal_write_1.f90
   intrinsic_std_4.f90
   io_err_1.f90
+  io_real_boz2.f90
   io_real_boz_3.f90
+  io_real_boz_4.f90
+  io_real_boz_5.f90
   io_real_boz.f90
   iostat_4.f90
   iostat_5.f90
@@ -1936,10 +1978,16 @@ file(GLOB FAILING_FILES CONFIGURE_DEPENDS
   literal_character_constant_1_z.F
   masklr_2.F90
   matmul_5.f90
+  matmul_bounds_10.f90
   matmul_bounds_11.f90
   matmul_bounds_13.f90
   matmul_bounds_15.f
   matmul_bounds_16.f
+  matmul_bounds_2.f90
+  matmul_bounds_3.f90
+  matmul_bounds_4.f90
+  matmul_bounds_5.f90
+  matmul_bounds_8.f90
   matmul_bounds_7.f90
   matmul_bounds_9.f90
   maxloc_2.f90
@@ -1949,6 +1997,7 @@ file(GLOB FAILING_FILES CONFIGURE_DEPENDS
   maxlocval_2.f90
   maxlocval_4.f90
   merge_bits_2.F90
+  merge_char_3.f90
   minloc_1.f90
   minlocval_1.f90
   minlocval_4.f90
@@ -1993,6 +2042,7 @@ file(GLOB FAILING_FILES CONFIGURE_DEPENDS
   negative_unit.f
   negative_unit_int8.f
   no_range_check_1.f90
+  no_unit_error_1.f90
   norm2_1.f90
   nosigned_zero_2.f90
   open_access_append_2.f90
@@ -2005,6 +2055,8 @@ file(GLOB FAILING_FILES CONFIGURE_DEPENDS
   pad_source_4.f
   pad_source_5.f
   pointer_check_6.f90
+  pointer_check_10.f90
+  pointer_remapping_6.f08
   pr12884.f
   pr17285.f90
   pr17286.f90
@@ -2019,6 +2071,11 @@ file(GLOB FAILING_FILES CONFIGURE_DEPENDS
   pr96436_3.f90
   pr96436_4.f90
   pr96436_5.f90
+  pr96436_6.f90
+  pr96436_7.f90
+  pr96436_8.f90
+  pr96436_9.f90
+  pr96436_10.f90
   promotion_3.f90
   promotion_4.f90
   promotion.f90
@@ -2050,7 +2107,10 @@ file(GLOB FAILING_FILES CONFIGURE_DEPENDS
   real8-4.f90
   real_const_3.f90
   realloc_on_assign_11.f90
+  recursive_check_11.f90
+  recursive_check_13.f90
   recursive_check_7.f90
+  recursive_check_9.f90
   repeat_1.f90
   reshape_order_1.f90
   reshape_order_2.f90
@@ -2065,10 +2125,12 @@ file(GLOB FAILING_FILES CONFIGURE_DEPENDS
   streamio_17.f90
   streamio_4.f90
   system_clock_3.f08
+  transpose_2.f90
   unf_io_convert_4.f90
   unf_read_corrupted_1.f90
   unf_short_record_1.f90
   unformatted_subrecord_1.f90
+  unpack_bounds_1.f90
   unpack_bounds_2.f90
   unpack_bounds_3.f90
   utf8_1.f03
@@ -2689,28 +2751,6 @@ file(GLOB FAILING_FILES CONFIGURE_DEPENDS
   openacc-define-3.f90
   openmp-define-3.f90
 
-  # Tests looking for runtime errors (e.g., bound checks). Correctly
-  # caught by flang runtime when it is used for array assignments.
-  allocate_error_6.f90
-  allocate_with_source_22.f03
-  allocate_with_source_23.f03
-  all_bounds_1.f90
-  associate_32.f03
-  bounds_check_12.f90
-  bounds_check_array_ctor_4.f90
-  bounds_check_fail_3.f90
-  inline_matmul_1.f90
-  maxloc_bounds_1.f90
-  maxloc_bounds_2.f90
-  maxloc_bounds_3.f90
-  maxloc_bounds_4.f90
-  maxloc_bounds_5.f90
-  maxloc_bounds_7.f90
-  maxloc_bounds_8.f90
-  pack_bounds_1.f90
-  ptr_func_assign_1.f08
-  spread_bounds_1.f90
-
   # Bad test, assigning an 11 elements array to a 12 elements array.
   transfer_array_intrinsic_4.f90