Add extensive and exhaustive tests

Add a generator that will test all inputs for input spaces `u32::MAX` or
smaller (e.g. single-argument `f32` routines).  For anything larger,
still run approximately `u32::MAX` tests, but distribute inputs evenly
across the function domain.

Since we often only want to run one of these tests at a time, this
implementation parallelizes within each test using `rayon`. A custom
test runner is used so a progress bar is possible.

Specific tests must be enabled by setting the `LIBM_EXTENSIVE_TESTS`
environment variable, e.g.

    LIBM_EXTENSIVE_TESTS=all_f16,cos,cosf cargo run ...

Author: tgross35

crates/libm-test/Cargo.toml

@@ -26,12 +26,14 @@ short-benchmarks = []
 [dependencies]
 anyhow = "1.0.90"
 az = { version = "1.2.1", optional = true }
+indicatif = { version = "0.17.9", default-features = false }
 libm = { path = "../..", features = ["unstable-test-support"] }
 libm-macros = { path = "../libm-macros" }
 musl-math-sys = { path = "../musl-math-sys", optional = true }
 paste = "1.0.15"
 rand = "0.8.5"
 rand_chacha = "0.3.1"
+rayon = "1.10.0"
 rug = { version = "1.26.1", optional = true, default-features = false, features = ["float", "std"] }
 
 [target.'cfg(target_family = "wasm")'.dependencies]
@@ -43,11 +45,18 @@ rand = { version = "0.8.5", optional = true }
 
 [dev-dependencies]
 criterion = { version = "0.5.1", default-features = false, features = ["cargo_bench_support"] }
+libtest-mimic = "0.8.1"
 
 [[bench]]
 name = "random"
 harness = false
 
+[[test]]
+# No harness so that we can skip tests at runtime based on env. Prefixed with
+# `z` so these tests get run last.
+name = "z_extensive"
+harness = false
+
 [lints.rust]
 # Values from the chared config.rs used by `libm` but not the test crate
 unexpected_cfgs = { level = "warn", check-cfg = [

crates/libm-test/src/gen.rs

@@ -2,6 +2,7 @@
 
 pub mod domain_logspace;
 pub mod edge_cases;
+pub mod extensive;
 pub mod random;
 
 /// A wrapper to turn any iterator into an `ExactSizeIterator`. Asserts the final result to ensure

crates/libm-test/src/gen/extensive.rs

@@ -0,0 +1,133 @@
+use std::fmt;
+use std::ops::RangeInclusive;
+
+use libm::support::MinInt;
+
+use crate::domain::HasDomain;
+use crate::gen::KnownSize;
+use crate::op::OpITy;
+use crate::run_cfg::{int_range, iteration_count};
+use crate::{CheckCtx, GeneratorKind, MathOp, logspace};
+
+/// Generate a sequence of inputs that either cover the domain in completeness (for smaller float
+/// types and single argument functions) or provide evenly spaced inputs across the domain with
+/// approximately `u32::MAX` total iterations.
+pub trait ExtensiveInput<Op> {
+    fn get_cases(ctx: &CheckCtx) -> impl ExactSizeIterator<Item = Self> + Send;
+}
+
+/// Construct an iterator from `logspace` and also calculate the total number of steps expected
+/// for that iterator.
+fn logspace_steps<Op>(
+    start: Op::FTy,
+    end: Op::FTy,
+    ctx: &CheckCtx,
+    argnum: usize,
+) -> (impl Iterator<Item = Op::FTy> + Clone, u64)
+where
+    Op: MathOp,
+    OpITy<Op>: TryFrom<u64, Error: fmt::Debug>,
+    RangeInclusive<OpITy<Op>>: Iterator,
+{
+    let max_steps = iteration_count(ctx, GeneratorKind::Extensive, argnum);
+    let max_steps = OpITy::<Op>::try_from(max_steps).unwrap_or(OpITy::<Op>::MAX);
+    let iter = logspace(start, end, max_steps);
+
+    // `logspace` can't implement `ExactSizeIterator` because of the range, but its size hint
+    // should be accurate (assuming <= usize::MAX iterations).
+    let size_hint = iter.size_hint();
+    assert_eq!(size_hint.0, size_hint.1.unwrap());
+
+    (iter, size_hint.0.try_into().unwrap())
+}
+
+macro_rules! impl_extensive_input {
+    ($fty:ty) => {
+        impl<Op> ExtensiveInput<Op> for ($fty,)
+        where
+            Op: MathOp<RustArgs = Self, FTy = $fty>,
+            Op: HasDomain<Op::FTy>,
+        {
+            fn get_cases(ctx: &CheckCtx) -> impl ExactSizeIterator<Item = Self> {
+                let start = Op::DOMAIN.range_start();
+                let end = Op::DOMAIN.range_end();
+                let (iter0, steps0) = logspace_steps::<Op>(start, end, ctx, 0);
+                let iter0 = iter0.map(|v| (v,));
+                KnownSize::new(iter0, steps0)
+            }
+        }
+
+        impl<Op> ExtensiveInput<Op> for ($fty, $fty)
+        where
+            Op: MathOp<RustArgs = Self, FTy = $fty>,
+        {
+            fn get_cases(ctx: &CheckCtx) -> impl ExactSizeIterator<Item = Self> {
+                let start = <$fty>::NEG_INFINITY;
+                let end = <$fty>::INFINITY;
+                let (iter0, steps0) = logspace_steps::<Op>(start, end, ctx, 0);
+                let (iter1, steps1) = logspace_steps::<Op>(start, end, ctx, 1);
+                let iter =
+                    iter0.flat_map(move |first| iter1.clone().map(move |second| (first, second)));
+                let count = steps0.checked_mul(steps1).unwrap();
+                KnownSize::new(iter, count)
+            }
+        }
+
+        impl<Op> ExtensiveInput<Op> for ($fty, $fty, $fty)
+        where
+            Op: MathOp<RustArgs = Self, FTy = $fty>,
+        {
+            fn get_cases(ctx: &CheckCtx) -> impl ExactSizeIterator<Item = Self> {
+                let start = <$fty>::NEG_INFINITY;
+                let end = <$fty>::INFINITY;
+
+                let (iter0, steps0) = logspace_steps::<Op>(start, end, ctx, 0);
+                let (iter1, steps1) = logspace_steps::<Op>(start, end, ctx, 1);
+                let (iter2, steps2) = logspace_steps::<Op>(start, end, ctx, 2);
+
+                let iter = iter0
+                    .flat_map(move |first| iter1.clone().map(move |second| (first, second)))
+                    .flat_map(move |(first, second)| {
+                        iter2.clone().map(move |third| (first, second, third))
+                    });
+                let count = steps0.checked_mul(steps1).unwrap().checked_mul(steps2).unwrap();
+
+                KnownSize::new(iter, count)
+            }
+        }
+
+        impl<Op> ExtensiveInput<Op> for (i32, $fty)
+        where
+            Op: MathOp<RustArgs = Self, FTy = $fty>,
+        {
+            fn get_cases(ctx: &CheckCtx) -> impl ExactSizeIterator<Item = Self> {
+                let start = <$fty>::NEG_INFINITY;
+                let end = <$fty>::INFINITY;
+
+                let iter0 = int_range(ctx, GeneratorKind::Extensive, 0);
+                let steps0 = iteration_count(ctx, GeneratorKind::Extensive, 0);
+                let (iter1, steps1) = logspace_steps::<Op>(start, end, ctx, 1);
+
+                let iter =
+                    iter0.flat_map(move |first| iter1.clone().map(move |second| (first, second)));
+                let count = steps0.checked_mul(steps1).unwrap();
+
+                KnownSize::new(iter, count)
+            }
+        }
+    };
+}
+
+impl_extensive_input!(f32);
+impl_extensive_input!(f64);
+
+/// Create a test case iterator for extensive inputs.
+pub fn get_test_cases<Op>(
+    ctx: &CheckCtx,
+) -> impl ExactSizeIterator<Item = Op::RustArgs> + Send + use<'_, Op>
+where
+    Op: MathOp,
+    Op::RustArgs: ExtensiveInput<Op>,
+{
+    Op::RustArgs::get_cases(ctx)
+}

crates/libm-test/src/gen/random.rs

@@ -73,7 +73,7 @@ macro_rules! impl_random_input {
             fn get_cases(ctx: &CheckCtx) -> impl ExactSizeIterator<Item = Self> {
                 let count0 = iteration_count(ctx, GeneratorKind::Random, 0);
                 let count1 = iteration_count(ctx, GeneratorKind::Random, 1);
-                let range0 = int_range(ctx, 0);
+                let range0 = int_range(ctx, GeneratorKind::Random, 0);
                 let iter = random_ints(count0, range0)
                     .flat_map(move |f1: i32| random_floats(count1).map(move |f2: $fty| (f1, f2)));
                 KnownSize::new(iter, count0 * count1)
@@ -84,7 +84,7 @@ macro_rules! impl_random_input {
             fn get_cases(ctx: &CheckCtx) -> impl ExactSizeIterator<Item = Self> {
                 let count0 = iteration_count(ctx, GeneratorKind::Random, 0);
                 let count1 = iteration_count(ctx, GeneratorKind::Random, 1);
-                let range1 = int_range(ctx, 1);
+                let range1 = int_range(ctx, GeneratorKind::Random, 1);
                 let iter = random_floats(count0).flat_map(move |f1: $fty| {
                     random_ints(count1, range1.clone()).map(move |f2: i32| (f1, f2))
                 });

crates/libm-test/src/lib.rs

@@ -23,7 +23,7 @@ pub use libm::support::{Float, Int, IntTy, MinInt};
 pub use num::{FloatExt, logspace};
 pub use op::{BaseName, FloatTy, Identifier, MathOp, OpCFn, OpFTy, OpRustFn, OpRustRet, Ty};
 pub use precision::{MaybeOverride, SpecialCase, default_ulp};
-pub use run_cfg::{CheckBasis, CheckCtx, EXTENSIVE_ENV, GeneratorKind};
+pub use run_cfg::{CheckBasis, CheckCtx, EXTENSIVE_ENV, GeneratorKind, skip_extensive_test};
 pub use test_traits::{CheckOutput, Hex, TupleCall};
 
 /// Result type for tests is usually from `anyhow`. Most times there is no success value to

crates/libm-test/src/num.rs

@@ -215,7 +215,7 @@ fn as_ulp_steps<F: Float>(x: F) -> Option<F::SignedInt> {
 /// to logarithmic spacing of their values.
 ///
 /// Note that this tends to skip negative zero, so that needs to be checked explicitly.
-pub fn logspace<F: FloatExt>(start: F, end: F, steps: F::Int) -> impl Iterator<Item = F>
+pub fn logspace<F: FloatExt>(start: F, end: F, steps: F::Int) -> impl Iterator<Item = F> + Clone
 where
     RangeInclusive<F::Int>: Iterator,
 {

crates/libm-test/src/run_cfg.rs

@@ -9,6 +9,8 @@ use crate::{BaseName, FloatTy, Identifier, test_log};
 /// The environment variable indicating which extensive tests should be run.
 pub const EXTENSIVE_ENV: &str = "LIBM_EXTENSIVE_TESTS";
 
+const EXTENSIVE_MAX_ITERATIONS: u64 = u32::MAX as u64;
+
 /// Context passed to [`CheckOutput`].
 #[derive(Clone, Debug, PartialEq, Eq)]
 pub struct CheckCtx {
@@ -53,6 +55,7 @@ pub enum CheckBasis {
 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
 pub enum GeneratorKind {
     Domain,
+    Extensive,
     Random,
 }
 
@@ -170,8 +173,14 @@ pub fn iteration_count(ctx: &CheckCtx, gen_kind: GeneratorKind, argnum: usize) -
     let mut total_iterations = match gen_kind {
         GeneratorKind::Domain => domain_iter_count,
         GeneratorKind::Random => random_iter_count,
+        GeneratorKind::Extensive => EXTENSIVE_MAX_ITERATIONS,
     };
 
+    // FMA has a huge domain but is reasonably fast to run, so increase iterations.
+    if ctx.base_name == BaseName::Fma {
+        total_iterations *= 4;
+    }
+
     if cfg!(optimizations_enabled) {
         // Always run at least 10,000 tests.
         total_iterations = total_iterations.max(10_000);
@@ -202,7 +211,7 @@ pub fn iteration_count(ctx: &CheckCtx, gen_kind: GeneratorKind, argnum: usize) -
 }
 
 /// Some tests require that an integer be kept within reasonable limits; generate that here.
-pub fn int_range(ctx: &CheckCtx, argnum: usize) -> RangeInclusive<i32> {
+pub fn int_range(ctx: &CheckCtx, gen_kind: GeneratorKind, argnum: usize) -> RangeInclusive<i32> {
     let t_env = TestEnv::from_env(ctx);
 
     if ctx.base_name != BaseName::Jn {
@@ -213,10 +222,17 @@ pub fn int_range(ctx: &CheckCtx, argnum: usize) -> RangeInclusive<i32> {
 
     // The integer argument to `jn` is an iteration count. Limit this to ensure tests can be
     // completed in a reasonable amount of time.
-    if t_env.slow_platform || !cfg!(optimizations_enabled) {
+    let non_extensive_range = if t_env.slow_platform || !cfg!(optimizations_enabled) {
         (-0xf)..=0xff
     } else {
         (-0xff)..=0xffff
+    };
+
+    let extensive_range = (-0xfff)..=0xfffff;
+
+    match gen_kind {
+        GeneratorKind::Extensive => extensive_range,
+        GeneratorKind::Domain | GeneratorKind::Random => non_extensive_range,
     }
 }
 
@@ -233,7 +249,6 @@ pub fn check_near_count(_ctx: &CheckCtx) -> u64 {
 }
 
 /// Check whether extensive actions should be run or skipped.
-#[expect(dead_code, reason = "extensive tests have not yet been added")]
 pub fn skip_extensive_test(ctx: &CheckCtx) -> bool {
     let t_env = TestEnv::from_env(ctx);
     !t_env.should_run_extensive

crates/libm-test/tests/z_extensive/main.rs

@@ -0,0 +1,14 @@
+//! `main` is just a wrapper to handle configuration.
+
+#[cfg(not(feature = "test-multiprecision"))]
+fn main() {
+    eprintln!("multiprecision not enabled; skipping extensive tests");
+}
+
+#[cfg(feature = "test-multiprecision")]
+mod run;
+
+#[cfg(feature = "test-multiprecision")]
+fn main() {
+    run::run();
+}