Fix parallel build issues

CMakeLists.txt

@@ -644,6 +644,11 @@ define_property(TARGET
   FULL_DOCS "Property to mark executable targets run as tests that they require 2^n images."
   )
 
+#-------------------------------------------------------------------------------------
+# Add global openmpi property, because using a variable an setting in parent scope did
+# not work as expected, i.e., not at all, on Linux Fedora 39.
+#-------------------------------------------------------------------------------------
+define_property(GLOBAL PROPERTY openmpi BRIEF_DOCS "True when mpi is openMPI.")
 
 #-------------------------------
 # Recurse into the src directory
@@ -715,6 +720,7 @@ function(add_caf_test name num_caf_img test_target)
     endif()
   endif()
   # Add a host file for OMPI
+  get_property(openmpi GLOBAL PROPERTY openmpi)
   if ( openmpi )
     set(test_parameters --hostfile ${CMAKE_BINARY_DIR}/hostfile)
   endif()

src/CMakeLists.txt

@@ -1,5 +1,5 @@
-add_subdirectory(application-binary-interface )
-add_subdirectory(runtime-libraries )
+add_subdirectory(application-binary-interface)
+add_subdirectory(runtime-libraries)
 add_subdirectory(tests)
 
 set(N_CPU ${N_CPU} PARENT_SCOPE)

src/runtime-libraries/mpi/CMakeLists.txt

@@ -147,13 +147,17 @@ execute_process(COMMAND ${MPIEXEC_EXECUTABLE} --version
   OUTPUT_VARIABLE mpi_version_out)
 if (mpi_version_out MATCHES "[Oo]pen[ -][Mm][Pp][Ii]")
   message( STATUS "OpenMPI detected")
-  set ( openmpi true PARENT_SCOPE)
+  set_property(GLOBAL PROPERTY openmpi true)
   # Write out a host file because OMPI's mpiexec is dumb
   file(WRITE ${CMAKE_BINARY_DIR}/hostfile "${HOST_NAME} slots=${N_CPU}\n")
   message( STATUS "hostfile written to: ${CMAKE_BINARY_DIR}/hostfile")
   if(NOT DEFINED ENV{TRAVIS})
     message( STATUS "Open-MPI back end detected, passing --allow-run-as-root to allow tests to pass when run with sudo or as root." )
   endif()
+elseif (mpi_version_out MATCHES "HYDRA")
+  message(STATUS "MPICH detected")
+  target_compile_definitions(caf_mpi PRIVATE MPI_CLEAR_COMM_BEFORE_FREE)
+  target_compile_definitions(caf_mpi_static PRIVATE MPI_CLEAR_COMM_BEFORE_FREE)
 endif ()
 
 if("${CMAKE_Fortran_COMPILER_ID}" STREQUAL "GNU")

src/runtime-libraries/mpi/mpi_caf.c

@@ -1104,6 +1104,24 @@ finalize_internal(int status_code)
     ierr = MPI_Finalize(); chk_err(ierr);
   }
 #else
+#ifdef MPI_CLEAR_COMM_BEFORE_FREE
+  {
+    int probe_flag;
+    MPI_Status status;
+    do {
+      ierr = MPI_Iprobe(MPI_ANY_SOURCE, MPI_ANY_TAG, CAF_COMM_WORLD, &probe_flag,
+                        &status); /* error is not of interest. */
+      if (probe_flag) {
+        int cnt;
+        MPI_Get_count(&status, MPI_BYTE, &cnt);
+        void * buf = alloca(cnt);
+        ierr = MPI_Recv(buf, cnt, MPI_BYTE, status.MPI_SOURCE, status.MPI_TAG,
+                        CAF_COMM_WORLD, &status); chk_err(ierr);
+      }
+    } while (probe_flag);
+  }
+#endif
+  dprint("freeing caf's communicator.\n");
   ierr = MPI_Comm_free(&CAF_COMM_WORLD); chk_err(ierr);
 
   CAF_Win_unlock_all(*stat_tok);
@@ -1112,6 +1130,7 @@ finalize_internal(int status_code)
   /* Only call Finalize if CAF runtime Initialized MPI. */
   if (caf_owns_mpi)
   {
+    dprint("Finalizing MPI.\n");
     ierr = MPI_Finalize(); chk_err(ierr);
   }
 #endif

src/script-templates/caf.in

@@ -161,14 +161,13 @@ __caf_err_report() {
 trap '__caf_err_report "${FUNCNAME:-.}" ${LINENO}' ERR
 
 # See if we are compiling or compiling and/or linking
-__only_compiling () {
-    for arg in "${@}"; do
-      if [[ "${arg}" == "-c" ]]; then
-        return 0
-      fi
-    done
-    return 1
-}
+with_link="true"
+for arg in "${@}"; do
+  if [[ "${arg}" == "-c" || "${arg}" == "-cpp" ]]; then
+    with_link="false"
+    break
+  fi
+done
 
 #--------------------------------------------------------------------------
 # End configured variables, now process them and build compile/link command
@@ -223,16 +222,14 @@ if [[ -n "${mpi_compile_flags[*]:-}" ]]; then
       caf_pre_flags+=("${compileflag}")
   done
 fi
-if [[ -n "${mpi_link_flags[*]:-}" ]]; then
-  if ! __only_compiling "${@}"; then
-    for linkflag in "${mpi_link_flags[@]:-}"; do
-      caf_pre_flags+=("${linkflag}")
-    done
-  fi
+if [[ -n "${mpi_link_flags[*]:-}" && "${with_link}" == "true" ]]; then
+  for linkflag in "${mpi_link_flags[@]:-}"; do
+    caf_pre_flags+=("${linkflag}")
+  done
 fi
 
 # Now do libraries, IN CORRECT ORDER, to append to command
-if [[ -n "${caf_libs[*]:-}" ]]; then
+if [[ "${with_link}" == "true" && -n "${caf_libs[*]:-}" ]]; then
   for lib in "${caf_libs[@]:-}"; do
     caf_added_libs+=("$(substitute_lib "${prefix%/}/${lib}")")
   done
@@ -242,7 +239,7 @@ if [[ -n "${threads_lib}" ]]; then
   caf_added_libs+=("${threads_lib}")
 fi
 
-if [[ -n "${mpi_libs[*]:-}" ]]; then
+if [[ "${with_link}" == "true" && -n "${mpi_libs[*]:-}" ]]; then
   for lib in "${mpi_libs[@]:-}"; do
     caf_added_libs+=("$(substitute_lib "${lib}")")
   done
@@ -330,10 +327,8 @@ elif [[ ${1} == '-s' || ${1} == '--show' || ${1} == '-show' ]]; then
   if [[ "${args}" ]]; then
     compiler_args+=("${args}")
   fi
-  if [[ "${caf_added_libs[*]:-}" ]]; then
-    if ! __only_compiling "${@}"; then
-      compiler_args+=("${caf_added_libs[@]}")
-    fi
+  if [[ "${caf_added_libs[*]:-}" && "${with_link}" == "true" ]]; then
+    compiler_args+=("${caf_added_libs[@]}")
   fi
   echo "${cafc}" "${compiler_args[@]}"
   exit 0
@@ -365,10 +360,8 @@ fi
 if [[ "${*:-}" ]]; then
   compiler_args+=("${@}")
 fi
-if [[ "${caf_added_libs[*]:-}" ]]; then
-  if ! __only_compiling "${@}" ; then
-    compiler_args+=("${caf_added_libs[@]}")
-  fi
+if [[ "${caf_added_libs[*]:-}" && "${with_link}" == "true" ]]; then
+  compiler_args+=("${caf_added_libs[@]}")
 fi
 #set -o xtrace # Show what we're doing
 set +o errtrace

src/tests/integration/pde_solvers/coarrayBurgers/CMakeLists.txt

@@ -19,13 +19,25 @@ else()
 endif()
 
 set(CMAKE_Fortran_COMPILER "${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_BINDIR}/caf")
-add_executable(coarray_burgers_pde
-  main.F90
-  global_field.F90
-  local_field.F90
+add_library(cbp_base STATIC
+  ${config_directory}/compiler_capabilities.txt
   ${library_directory}/ForTrilinos_assertion_utility.F90
   ${library_directory}/object_interface.F90
   ${library_directory}/co_object_interface.F90
 )
-add_dependencies(coarray_burgers_pde caf_mpi_static)
-target_include_directories(coarray_burgers_pde PRIVATE ${config_directory})
+add_library(cbp_local STATIC
+  ${config_directory}/compiler_capabilities.txt
+  local_field.F90
+)
+add_executable(coarray_burgers_pde
+  ${config_directory}/compiler_capabilities.txt
+  global_field.F90
+  main.F90
+)
+target_link_libraries(cbp_local PUBLIC cbp_base caf_mpi_static)
+target_link_libraries(coarray_burgers_pde PRIVATE cbp_local)
+set_target_properties(cbp_base cbp_local coarray_burgers_pde
+  PROPERTIES
+  Fortran_MODULE_DIRECTORY "${config_directory}"
+  INCLUDE_DIRECTORIES "${CMAKE_BINARY_DIR};${config_directory}"
+)

src/tests/integration/pde_solvers/coarrayHeatSimplified/CMakeLists.txt

@@ -1,11 +1,13 @@
 set(CMAKE_Fortran_COMPILER "${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_BINDIR}/caf")
-add_library(local_field OBJECT local_field.f90)
-add_library(global_field OBJECT global_field.f90)
-add_dependencies(local_field caf_mpi_static)
-add_dependencies(global_field local_field caf_mpi_static)
+add_library(local_field STATIC local_field.f90)
+add_library(global_field STATIC global_field.f90)
 add_executable(co_heat
   main.f90
-  $<TARGET_OBJECTS:local_field>
-  $<TARGET_OBJECTS:global_field>
 )
-add_dependencies(co_heat caf_mpi_static)
+target_link_libraries(global_field PUBLIC local_field caf_mpi_static)
+target_link_libraries(co_heat PRIVATE global_field)
+set_target_properties(local_field global_field co_heat
+  PROPERTIES
+  INCLUDE_DIRECTORIES "${CMAKE_BINARY_DIR}"
+  Fortran_MODULE_DIRECTORY "${CMAKE_BINARY_DIR}"
+)

src/tests/integration/pde_solvers/coarrayHeatSimplified/global_field.f90

@@ -24,8 +24,8 @@
 ! ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 ! (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 !
-module global_field_module
-  use local_field_module, only : local_field
+module chs_global_field_module
+  use chs_local_field_module, only : local_field
   implicit none
   private
   public :: global_field

src/tests/integration/pde_solvers/coarrayHeatSimplified/local_field.f90

@@ -24,7 +24,7 @@
 ! ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 ! (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 !
-module local_field_module
+module chs_local_field_module
   implicit none
   private
   public :: local_field

src/tests/integration/pde_solvers/coarrayHeatSimplified/main.f90

@@ -27,7 +27,7 @@
 
 program main
   use IEEE_arithmetic, only : IEEE_is_NaN
-  use global_field_module, only : global_field
+  use chs_global_field_module, only : global_field
   implicit none
   type(global_field) :: T,laplacian_T,T_half
   real, parameter :: alpha=1.,dt=0.0001,final_time=1.,tolerance=1.E-3

src/tests/utilities/CMakeLists.txt

@@ -31,27 +31,21 @@ if ("${CMAKE_Fortran_COMPILER_ID}" MATCHES "GNU" )
   set(CMAKE_REQUIRED_FLAGS ${OLD_REQUIRED_FLAGS})
 endif()
 
-add_library( oc_test_interfaces OBJECT
+add_library( oc_test_interfaces STATIC
   opencoarrays_object_interface.f90
   oc_assertions_interface.F90
   )
 add_library( opencoarrays_test_utilities STATIC
-  oc_assertions_implementation.F90
   $<TARGET_OBJECTS:oc_test_interfaces>
+  oc_assertions_implementation.F90
   )
-set_target_properties(opencoarrays_test_utilities
-  PROPERTIES
-  Fortran_MODULE_DIRECTORY "${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_INCLUDEDIR}/${mod_dir_tail}")
-set_target_properties(oc_test_interfaces
+target_link_libraries(opencoarrays_test_utilities PUBLIC oc_test_interfaces)
+set_target_properties(oc_test_interfaces opencoarrays_test_utilities
   PROPERTIES
-  Fortran_MODULE_DIRECTORY "${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_INCLUDEDIR}/${mod_dir_tail}")
-
-target_compile_options(oc_test_interfaces
-  PUBLIC
-  "-fcoarray=lib")
-target_compile_options(opencoarrays_test_utilities
-  PUBLIC
-  "-fcoarray=lib")
+  Fortran_MODULE_DIRECTORY "${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_INCLUDEDIR}/${mod_dir_tail}"
+  INCLUDE_DIRECTORIES "${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_INCLUDEDIR}/${mod_dir_tail}"
+  COMPILE_OPTIONS "-fcoarray=lib"
+  )
 
 if(HAVE_ERROR_STOP_IN_PURE)
   target_compile_definitions(oc_test_interfaces

src/tests/utilities/opencoarrays_object_interface.f90

@@ -1,6 +1,6 @@
 module opencoarrays_object_interface
-  !! Object pattern abstract type to provide a universal interface to a userd-defined derived type ouptput
-  !! capability specified in a generic binding
+  !! Object pattern abstract type to provide a universal interface to a user-defined
+  !! derived type output capability specified in a generic binding
   implicit none
 
   type, abstract :: object