[libc++] Fix incorrect overflow checking in std::lcm

[ldionne]

We should have been using __builtin_mul_overflow from the start instead of adding a manual (and error-prone) check for overflow.

Fixes #96196

[llvmbot]

@llvm/pr-subscribers-libcxx

Author: Louis Dionne (ldionne)

Changes

We should have been using __builtin_mul_overflow from the start instead of adding a manual (and error-prone) check for overflow.

Fixes #96196

---

Full diff: https://github.com/llvm/llvm-project/pull/96310.diff

2 Files Affected:

• (modified) libcxx/include/__numeric/gcd_lcm.h (+5-2)
• (modified) libcxx/test/std/numerics/numeric.ops/numeric.ops.lcm/lcm.pass.cpp (+25-1)

diff --git a/libcxx/include/__numeric/gcd_lcm.h b/libcxx/include/__numeric/gcd_lcm.h
index 87ebbae0157f5..3916db2fae0e6 100644
--- a/libcxx/include/__numeric/gcd_lcm.h
+++ b/libcxx/include/__numeric/gcd_lcm.h
@@ -117,8 +117,11 @@ _LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI common_type_t<_Tp, _Up> lcm(_Tp __m, _Up
   using _Rp  = common_type_t<_Tp, _Up>;
   _Rp __val1 = __ct_abs<_Rp, _Tp>()(__m) / std::gcd(__m, __n);
   _Rp __val2 = __ct_abs<_Rp, _Up>()(__n);
-  _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN((numeric_limits<_Rp>::max() / __val1 > __val2), "Overflow in lcm");
-  return __val1 * __val2;
+  _Rp __res;
+  bool __overflow = __builtin_mul_overflow(__val1, __val2, &__res);
+  (void)__overflow;
+  _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(!__overflow, "Overflow in lcm");
+  return __res;
 }
 
 #endif // _LIBCPP_STD_VER
diff --git a/libcxx/test/std/numerics/numeric.ops/numeric.ops.lcm/lcm.pass.cpp b/libcxx/test/std/numerics/numeric.ops/numeric.ops.lcm/lcm.pass.cpp
index 16bfbbef41c45..992ecf64a8567 100644
--- a/libcxx/test/std/numerics/numeric.ops/numeric.ops.lcm/lcm.pass.cpp
+++ b/libcxx/test/std/numerics/numeric.ops/numeric.ops.lcm/lcm.pass.cpp
@@ -89,6 +89,13 @@ constexpr bool do_test(int = 0)
     return accumulate;
 }
 
+template <class T>
+constexpr bool test_limits() {
+    assert(std::lcm(std::numeric_limits<T>::max() - 1, std::numeric_limits<T>::max() - 1) == std::numeric_limits<T>::max() - 1);
+    assert(std::lcm(std::numeric_limits<T>::max(), std::numeric_limits<T>::max()) == std::numeric_limits<T>::max());
+    return true;
+}
+
 int main(int argc, char**)
 {
     int non_cce = argc; // a value that can't possibly be constexpr
@@ -141,5 +148,22 @@ int main(int argc, char**)
     assert(res1 == 1324997410816LL);
     }
 
-  return 0;
+    // https://github.com/llvm/llvm-project/issues/96196
+    {
+        assert(test_limits<unsigned int>());
+        assert(test_limits<std::uint32_t>());
+        assert(test_limits<std::uint64_t>());
+        assert(test_limits<int>());
+        assert(test_limits<std::int32_t>());
+        assert(test_limits<std::int64_t>());
+
+        static_assert(test_limits<unsigned int>(), "");
+        static_assert(test_limits<std::uint32_t>(), "");
+        static_assert(test_limits<std::uint64_t>(), "");
+        static_assert(test_limits<int>(), "");
+        static_assert(test_limits<std::int32_t>(), "");
+        static_assert(test_limits<std::int64_t>(), "");
+    }
+
+    return 0;
 }

[dimitry-unified-streaming]

LGTM

[philnik777]

Suggested change

	bool __overflow = __builtin_mul_overflow(__val1, __val2, &__res);
	(void)__overflow;
	_LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(!__overflow, "Overflow in lcm");
	[[maybe_unused]] bool __overflow = __builtin_mul_overflow(__val1, __val2, &__res);
	_LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(!__overflow, "Overflow in lcm");

[DimitryAndric]

What is the general preference in libc++ to suppress unused warnings? I haven't made any inventory so I'm not sure what is used more often, casting to void or using maybe_unused.

[philnik777]

Casting to void is used more often because a lot of code has to work in C++03 and not all compilers we support have [[]] attributes in C++03 yet.