[flang][runtime] Add limit check to MOD/MODULO (#80026)

When testing the arguments to see whether they are integers, check first
that they are within the maximum range of a 64-bit integer; otherwise, a
value of larger magnitude will set an invalid operand exception flag.

Author: klausler

flang/runtime/numeric.cpp

@@ -145,14 +145,19 @@ inline RT_API_ATTRS T RealMod(
   } else if (std::isinf(p)) {
     return a;
   }
-  if (auto aInt{static_cast<std::int64_t>(a)}; a == aInt) {
-    if (auto pInt{static_cast<std::int64_t>(p)}; p == pInt) {
-      // Fast exact case for integer operands
-      auto mod{aInt - (aInt / pInt) * pInt};
-      if (IS_MODULO && (aInt > 0) != (pInt > 0)) {
-        mod += pInt;
+  T aAbs{std::abs(a)};
+  T pAbs{std::abs(p)};
+  if (aAbs <= static_cast<T>(std::numeric_limits<std::int64_t>::max()) &&
+      pAbs <= static_cast<T>(std::numeric_limits<std::int64_t>::max())) {
+    if (auto aInt{static_cast<std::int64_t>(a)}; a == aInt) {
+      if (auto pInt{static_cast<std::int64_t>(p)}; p == pInt) {
+        // Fast exact case for integer operands
+        auto mod{aInt - (aInt / pInt) * pInt};
+        if (IS_MODULO && (aInt > 0) != (pInt > 0)) {
+          mod += pInt;
+        }
+        return static_cast<T>(mod);
       }
-      return static_cast<T>(mod);
     }
   }
   if constexpr (std::is_same_v<T, float> || std::is_same_v<T, double> ||
@@ -183,9 +188,8 @@ inline RT_API_ATTRS T RealMod(
     // what's left is the desired remainder.  This is basically
     // the same algorithm as arbitrary precision binary long division,
     // discarding the quotient.
-    T tmp{std::abs(a)};
-    T pAbs{std::abs(p)};
-    for (T adj{SetExponent(pAbs, Exponent<int>(tmp))}; tmp >= pAbs; adj /= 2) {
+    T tmp{aAbs};
+    for (T adj{SetExponent(pAbs, Exponent<int>(aAbs))}; tmp >= pAbs; adj /= 2) {
       if (tmp >= adj) {
         tmp -= adj;
         if (tmp == 0) {