[Flang] Detect endianness in the preprocessor

[jhuber6]

Summary:
Currently we use TestBigEndian in CMake to determine endianness. This
doesn't work on all platforms and is deprecated since CMake 3.20.
Instead of using CMake, we can just use the GNU/Clang preprocessor
definitions.

The only difficulty is MSVC, mostly because they don't support the same
macros. But, as far as I'm aware, MSVC / Windows targets are always
little endian, and if not we can just override it for that specific
target in the future.

[Meinersbur]

There is also CMAKE_CXX_BYTE_ORDER but it also relies on a compiled program being executed, so not usable for cross-compiling either.

[llvmbot]

@llvm/pr-subscribers-flang-semantics

Author: Joseph Huber (jhuber6)

Changes

Summary:
Currently we use TestBigEndian in CMake to determine endianness. This
doesn't work on all platforms and is deprecated since CMake 3.20.
Instead of using CMake, we can just use the GNU/Clang preprocessor
definitions.

The only difficulty is MSVC, mostly because they don't support the same
macros. But, as far as I'm aware, MSVC / Windows targets are always
little endian, and if not we can just override it for that specific
target in the future.

---

Full diff: https://github.com/llvm/llvm-project/pull/132767.diff

3 Files Affected:

• (modified) flang/cmake/modules/FlangCommon.cmake (+13-27)
• (modified) flang/include/flang/Common/api-attrs.h (+17)
• (modified) flang/include/flang/Evaluate/common.h (+1)

diff --git a/flang/cmake/modules/FlangCommon.cmake b/flang/cmake/modules/FlangCommon.cmake
index 4726c640c97b7..9dbd1959fe7b1 100644
--- a/flang/cmake/modules/FlangCommon.cmake
+++ b/flang/cmake/modules/FlangCommon.cmake
@@ -24,30 +24,16 @@ if (FLANG_RUNTIME_F128_MATH_LIB)
   add_compile_definitions(FLANG_RUNTIME_F128_MATH_LIB="${FLANG_RUNTIME_F128_MATH_LIB}")
 endif()
 
-# The NVPTX target can't emit a binary due to the PTXAS dependency, just
-# hard-code this.
-if ("${LLVM_RUNTIMES_TARGET}" MATCHES "^nvptx")
-  add_compile_definitions(FLANG_LITTLE_ENDIAN=1)
-else ()
-  # Check if 128-bit float computations can be done via long double
-  # Note that '-nostdinc++' might be implied when this code kicks in
-  # (see 'runtimes/CMakeLists.txt'), so we cannot use 'cfloat' C++ header
-  # file in the test below.
-  # Compile it as C.
-  check_c_source_compiles(
-    "#include <float.h>
-     #if LDBL_MANT_DIG != 113
-     #error LDBL_MANT_DIG != 113
-     #endif
-     int main() { return 0; }
-    "
-    HAVE_LDBL_MANT_DIG_113)
-
-  include(TestBigEndian)
-  test_big_endian(IS_BIGENDIAN)
-  if (IS_BIGENDIAN)
-    add_compile_definitions(FLANG_BIG_ENDIAN=1)
-  else ()
-    add_compile_definitions(FLANG_LITTLE_ENDIAN=1)
-  endif ()
-endif ()
+# Check if 128-bit float computations can be done via long double
+# Note that '-nostdinc++' might be implied when this code kicks in
+# (see 'runtimes/CMakeLists.txt'), so we cannot use 'cfloat' C++ header
+# file in the test below.
+# Compile it as C.
+check_c_source_compiles(
+  "#include <float.h>
+   #if LDBL_MANT_DIG != 113
+   #error LDBL_MANT_DIG != 113
+   #endif
+   int main() { return 0; }
+  "
+  HAVE_LDBL_MANT_DIG_113)
diff --git a/flang/include/flang/Common/api-attrs.h b/flang/include/flang/Common/api-attrs.h
index 1ee91ca8e0d9d..fd524ee34ccff 100644
--- a/flang/include/flang/Common/api-attrs.h
+++ b/flang/include/flang/Common/api-attrs.h
@@ -189,4 +189,21 @@
 #define RT_OPTNONE_ATTR
 #endif
 
+/* Detect system endianness if it was not explicitly set. */
+#if !defined(FLANG_LITTLE_ENDIAN) && !defined(FLANG_BIG_ENDIAN)
+
+/* We always assume Windows is little endian, otherwise use the GCC compatible
+ * flags. */
+#if defined(_MSC_VER) || defined(_WIN32)
+#define FLANG_LITTLE_ENDIAN 1
+#elif defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
+#define FLANG_LITTLE_ENDIAN 1
+#elif defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
+#define FLANG_BIG_ENDIAN 1
+#else
+#error "Unknown or unsupported endianness."
+#endif
+
+#endif /* !defined(FLANG_LITTLE_ENDIAN) && !defined(FLANG_BIG_ENDIAN) */
+
 #endif /* !FORTRAN_RUNTIME_API_ATTRS_H_ */
diff --git a/flang/include/flang/Evaluate/common.h b/flang/include/flang/Evaluate/common.h
index 993540aebca57..cc9d14f298434 100644
--- a/flang/include/flang/Evaluate/common.h
+++ b/flang/include/flang/Evaluate/common.h
@@ -9,6 +9,7 @@
 #ifndef FORTRAN_EVALUATE_COMMON_H_
 #define FORTRAN_EVALUATE_COMMON_H_
 
+#include "api-attrs.h"
 #include "flang/Common/enum-set.h"
 #include "flang/Common/idioms.h"
 #include "flang/Common/indirection.h"

[Meinersbur]

LGTM

[kkwli]

Thanks