chore(fuzz): Run fuzzing deterministically on CI

tooling/ast_fuzzer/fuzz/Cargo.toml

@@ -26,6 +26,7 @@ noir_ast_fuzzer = { path = ".." }
 [dev-dependencies]
 arbtest.workspace = true
 env_logger.workspace = true
+proptest.workspace = true
 
 
 [[bin]]

tooling/ast_fuzzer/fuzz/src/targets/acir_vs_brillig.rs

@@ -52,6 +52,7 @@ mod tests {
     /// ```
     #[test]
     fn fuzz_with_arbtest() {
-        crate::targets::tests::fuzz_with_arbtest(super::fuzz);
+        // TODO: Allow more tests when the 1510th case is fixed.
+        crate::targets::tests::fuzz_with_arbtest(super::fuzz, 1509);
     }
 }

tooling/ast_fuzzer/fuzz/src/targets/comptime_vs_brillig.rs

@@ -56,6 +56,7 @@ mod tests {
     /// ```
     #[test]
     fn fuzz_with_arbtest() {
-        crate::targets::tests::fuzz_with_arbtest(super::fuzz);
+        // TODO(#8870): Allow more tests when the bug is fixed (fails in the 128th case).
+        crate::targets::tests::fuzz_with_arbtest(super::fuzz, 127);
     }
 }

tooling/ast_fuzzer/fuzz/src/targets/min_vs_full.rs

@@ -56,19 +56,13 @@ pub fn fuzz(u: &mut Unstructured) -> eyre::Result<()> {
 
 #[cfg(test)]
 mod tests {
-    use crate::targets::tests::is_running_in_ci;
-
     /// ```ignore
     /// NOIR_ARBTEST_SEED=0x6819c61400001000 \
     /// NOIR_AST_FUZZER_SHOW_AST=1 \
     /// cargo test -p noir_ast_fuzzer_fuzz min_vs_full
     /// ```
     #[test]
     fn fuzz_with_arbtest() {
-        if is_running_in_ci() {
-            // TODO: Investigate second program constraint failures.
-            return;
-        }
-        crate::targets::tests::fuzz_with_arbtest(super::fuzz);
+        crate::targets::tests::fuzz_with_arbtest(super::fuzz, 2000);
     }
 }

tooling/ast_fuzzer/fuzz/src/targets/mod.rs

@@ -6,10 +6,16 @@ pub mod pass_vs_prev;
 
 #[cfg(test)]
 mod tests {
+
+    const TIMEOUT: Duration = Duration::from_secs(20);
+    const MIN_SIZE: u32 = 1 << 12;
+    const MAX_SIZE: u32 = 1 << 20;
+
     use std::time::Duration;
 
     use arbitrary::Unstructured;
     use color_eyre::eyre;
+    use proptest::prelude::*;
 
     pub fn seed_from_env() -> Option<u64> {
         let Ok(seed) = std::env::var("NOIR_ARBTEST_SEED") else { return None };
@@ -18,14 +24,7 @@ mod tests {
         Some(seed)
     }
 
-    /// We can use this to disable the proptests on CI until we fix known bugs.
-    ///
-    /// The tests should always be enabled locally. They can be run with:
-    ///
-    /// ```ignore
-    /// cargo test -p noir_ast_fuzzer_fuzz
-    /// ```
-    #[allow(unused)]
+    /// Check if we are running on CI.
     pub fn is_running_in_ci() -> bool {
         std::env::var("CI").is_ok()
     }
@@ -39,21 +38,72 @@ mod tests {
     /// NOIR_AST_FUZZER_SHOW_AST=1 \
     /// cargo test -p noir_ast_fuzzer_fuzz acir_vs_brillig
     /// ```
-    pub fn fuzz_with_arbtest(f: impl Fn(&mut Unstructured) -> eyre::Result<()>) {
+    ///
+    /// The `cases` determine how many tests to run on CI.
+    /// Tune this so that we can expect CI to be able to get through all cases in reasonable time.
+    pub fn fuzz_with_arbtest(f: impl Fn(&mut Unstructured) -> eyre::Result<()>, cases: u32) {
         let _ = env_logger::try_init();
 
-        let mut prop = arbtest::arbtest(|u| {
+        if let Some(seed) = seed_from_env() {
+            run_reproduce(f, seed);
+        } else if is_running_in_ci() {
+            run_deterministic(f, cases);
+        } else {
+            run_nondeterministic(f);
+        }
+    }
+
+    /// Reproduce the result of a single seed.
+    fn run_reproduce(f: impl Fn(&mut Unstructured) -> eyre::Result<()>, seed: u64) {
+        arbtest::arbtest(|u| {
             f(u).unwrap();
             Ok(())
         })
-        .budget(Duration::from_secs(20))
-        .size_min(1 << 12)
-        .size_max(1 << 20);
+        .seed(seed)
+        .run();
+    }
 
-        if let Some(seed) = seed_from_env() {
-            prop = prop.seed(seed);
-        }
+    /// Run the tests non-deterministically until the timeout.
+    ///
+    /// This is the local behavior.
+    fn run_nondeterministic(f: impl Fn(&mut Unstructured) -> eyre::Result<()>) {
+        arbtest::arbtest(|u| {
+            f(u).unwrap();
+            Ok(())
+        })
+        .size_min(MIN_SIZE)
+        .size_max(MAX_SIZE)
+        .budget(TIMEOUT)
+        .run();
+    }
+
+    /// Run multiple tests with a deterministic RNG.
+    ///
+    /// This is the behavior on CI.
+    fn run_deterministic(f: impl Fn(&mut Unstructured) -> eyre::Result<()>, cases: u32) {
+        let config = proptest::test_runner::Config {
+            cases,
+            failure_persistence: None,
+            max_shrink_iters: 0,
+            ..Default::default()
+        };
+        let rng = proptest::test_runner::TestRng::deterministic_rng(config.rng_algorithm);
+        let mut runner = proptest::test_runner::TestRunner::new_with_rng(config, rng);
+
+        runner
+            .run(&seed_strategy(), |seed| {
+                run_reproduce(&f, seed);
+                Ok(())
+            })
+            .unwrap();
+    }
 
-        prop.run();
+    /// Generate seeds for `arbtest` where the top 32 bits are random and the lower 32 bits represent the input size.
+    fn seed_strategy() -> proptest::strategy::BoxedStrategy<u64> {
+        (MIN_SIZE..MAX_SIZE)
+            .prop_flat_map(move |size| {
+                any::<u64>().prop_map(move |raw| (size as u64) | (raw << u32::BITS))
+            })
+            .boxed()
     }
 }

tooling/ast_fuzzer/fuzz/src/targets/orig_vs_morph.rs

@@ -363,6 +363,6 @@ mod tests {
     /// ```
     #[test]
     fn fuzz_with_arbtest() {
-        crate::targets::tests::fuzz_with_arbtest(super::fuzz);
+        crate::targets::tests::fuzz_with_arbtest(super::fuzz, 2000);
     }
 }

tooling/ast_fuzzer/fuzz/src/targets/pass_vs_prev.rs

@@ -71,6 +71,7 @@ mod tests {
     /// ```
     #[test]
     fn fuzz_with_arbtest() {
-        crate::targets::tests::fuzz_with_arbtest(super::fuzz);
+        // TODO: Allow more cases when the 3180th case is fixed.
+        crate::targets::tests::fuzz_with_arbtest(super::fuzz, 3179);
     }
 }