chore(fuzz): Handle overflows in comptime fuzzing

tooling/ast_fuzzer/fuzz/src/targets/acir_vs_brillig.rs

@@ -10,11 +10,9 @@ use noir_ast_fuzzer::rewrite::change_all_functions_into_unconstrained;
 
 pub fn fuzz(u: &mut Unstructured) -> eyre::Result<()> {
     let config = Config {
-        // We created enough bug tickets due to overflows
-        avoid_overflow: true,
-        // also with negative values
-        avoid_negative_int_literals: true,
-        // and it gets old to have to edit u128 to fit into u32 for the frontend to parse
+        // Overflows can be triggered easily, so in half the cases we avoid them,
+        // to make sure they don't mask other errors.
+        avoid_overflow: u.arbitrary()?,
         avoid_large_int_literals: true,
         ..Default::default()
     };

tooling/ast_fuzzer/fuzz/src/targets/comptime_vs_brillig.rs

@@ -13,24 +13,19 @@ use noir_ast_fuzzer::rewrite::change_all_functions_into_unconstrained;
 
 pub fn fuzz(u: &mut Unstructured) -> eyre::Result<()> {
     let config = Config {
-        // We created enough bug tickets due to overflows
-        // TODO(#8817): Comptime code fails to compile if there is an overflow, which causes a panic.
-        avoid_overflow: true,
-        // also with negative values
-        avoid_negative_int_literals: true,
-        // also divisions
-        avoid_err_by_zero: true,
-        // and it gets old to have to edit u128 to fit into u32 for the frontend to parse
+        // It's easy to overflow.
+        avoid_overflow: u.arbitrary()?,
+        // Avoid using large integers in for loops that the frontend would reject.
         avoid_large_int_literals: true,
+        // Also avoid negative integers, because the frontend rejects them for loops.
+        avoid_negative_int_literals: true,
         // Avoid break/continue
         avoid_loop_control: true,
-        // TODO(#8817): Comptime code fails to compile if there is an assertion failure, which causes a panic.
-        avoid_constrain: true,
         // Has to only use expressions valid in comptime
         comptime_friendly: true,
-        // Force brillig
+        // Force brillig, to generate loops that the interpreter can do but ACIR cannot.
         force_brillig: true,
-        // Use lower limits because of the interpreter.
+        // Use lower limits because of the interpreter, to avoid stack overflow
         max_loop_size: 5,
         max_recursive_calls: 5,
         ..Default::default()

tooling/ast_fuzzer/fuzz/src/targets/min_vs_full.rs

@@ -15,11 +15,9 @@ use noirc_evaluator::ssa::minimal_passes;
 pub fn fuzz(u: &mut Unstructured) -> eyre::Result<()> {
     let passes = minimal_passes();
     let config = Config {
-        // Try to avoid using overflowing operations; see below for the reason.
-        avoid_overflow: true,
-        // Avoid stuff that would be difficult to copy as Noir; we want to verify these failures with nargo.
+        // Overflows are easy to trigger.
+        avoid_overflow: u.arbitrary()?,
         avoid_large_int_literals: true,
-        avoid_negative_int_literals: true,
         ..Default::default()
     };
 
@@ -49,10 +47,7 @@ pub fn fuzz(u: &mut Unstructured) -> eyre::Result<()> {
 
     let result = inputs.exec()?;
 
-    // Unfortunately the minimal pipeline can fail on assertions of instructions that get eliminated from the final pipeline,
-    // so if the minimal version fails and the final succeeds, it is most likely because of some overflow in a variable that
-    // was ultimately unused. Therefore we only compare results if both succeeded, or if only the final failed.
-    if matches!(result, CompareResult::BothFailed(_, _) | CompareResult::LeftFailed(_, _)) {
+    if matches!(result, CompareResult::BothFailed(_, _)) {
         Ok(())
     } else {
         compare_results_compiled(&inputs, &result)

tooling/ast_fuzzer/fuzz/src/targets/mod.rs

@@ -46,7 +46,7 @@ mod tests {
             f(u).unwrap();
             Ok(())
         })
-        .budget(Duration::from_secs(10))
+        .budget(Duration::from_secs(20))
         .size_min(1 << 12)
         .size_max(1 << 20);
 

tooling/ast_fuzzer/fuzz/src/targets/orig_vs_morph.rs

@@ -16,7 +16,7 @@ pub fn fuzz(u: &mut Unstructured) -> eyre::Result<()> {
     let rules = rules::all();
     let max_rewrites = 10;
     let config = Config {
-        avoid_negative_int_literals: true,
+        avoid_overflow: u.arbitrary()?,
         avoid_large_int_literals: true,
         ..Default::default()
     };

tooling/ast_fuzzer/fuzz/src/targets/pass_vs_prev.rs

@@ -13,7 +13,7 @@ use noirc_evaluator::ssa::ssa_gen::Ssa;
 use noirc_evaluator::ssa::{SsaPass, primary_passes, ssa_gen};
 
 pub fn fuzz(u: &mut Unstructured) -> eyre::Result<()> {
-    let config = Config::default();
+    let config = Config { avoid_overflow: u.arbitrary()?, ..Config::default() };
 
     let inputs = CompareInterpreted::arb(u, config, |u, program| {
         let options = CompareOptions::arbitrary(u)?;

tooling/ast_fuzzer/src/compare/compiled.rs

@@ -80,6 +80,13 @@ impl Comparable for NargoError<FieldElement> {
         };
 
         match (ee1, ee2) {
+            (
+                AssertionFailed(ResolvedAssertionPayload::String(c), _, _),
+                AssertionFailed(_, _, _),
+            ) if c.contains("CustomDiagnostic") => {
+                // Looks like the workaround we have for comptime failures originating from overflows and similar assertion failures.
+                true
+            }
             (AssertionFailed(p1, _, _), AssertionFailed(p2, _, _)) => p1 == p2,
             (SolvingError(s1, _), SolvingError(s2, _)) => format!("{s1}") == format!("{s2}"),
             (SolvingError(s, _), AssertionFailed(p, _, _))

tooling/ast_fuzzer/src/compare/comptime.rs

@@ -7,6 +7,8 @@ use std::{cell::RefCell, collections::BTreeMap};
 use arbitrary::Unstructured;
 use bn254_blackbox_solver::Bn254BlackBoxSolver;
 use color_eyre::eyre::{self, WrapErr};
+use nargo::NargoError;
+use nargo::errors::ExecutionError;
 use nargo::{foreign_calls::DefaultForeignCallBuilder, parse_all};
 use noirc_abi::Abi;
 use noirc_driver::{
@@ -75,20 +77,10 @@ pub struct CompareComptime {
 impl CompareComptime {
     /// Execute the Noir code and the SSA, then compare the results.
     pub fn exec(&self) -> eyre::Result<CompareCompiledResult> {
-        log::debug!("comptime src:\n{}", self.source);
-        let (program1, output1) = match prepare_and_compile_snippet(
-            self.source.clone(),
-            self.force_brillig,
-            Vec::new(),
-        ) {
-            Ok(((program, output), _)) => (program, output),
-            Err(e) => panic!("failed to compile program:\n{}\n{e:?}", self.source),
-        };
-        let comptime_print = String::from_utf8(output1).expect("should be valid utf8 string");
-
         let blackbox_solver = Bn254BlackBoxSolver(false);
         let initial_witness = self.abi.encode(&BTreeMap::new(), None).wrap_err("abi::encode")?;
 
+        // Execute a compiled Program.
         let do_exec = |program| {
             let mut print = Vec::new();
 
@@ -108,9 +100,49 @@ impl CompareComptime {
             (res, print)
         };
 
-        let (res1, print1) = do_exec(&program1.program);
+        // Execute the 2nd (Brillig) program.
         let (res2, print2) = do_exec(&self.ssa.artifact.program);
 
+        // Try to compile the 1st (comptime) version from string.
+        log::debug!("comptime src:\n{}", self.source);
+        let (program1, output1) = match prepare_and_compile_snippet(
+            self.source.clone(),
+            self.force_brillig,
+            Vec::new(),
+        ) {
+            Ok(((program, output), _)) => (program, output),
+            Err(e) => {
+                // If the comptime code failed to compile, it could be because it executed the code
+                // and encountered an overflow, which would be a runtime error in Brillig.
+                let is_assertion = e.iter().any(|e| {
+                    e.secondaries.iter().any(|s| s.message == "Assertion failed")
+                        || e.message.contains("overflow")
+                        || e.message.contains("divide by zero")
+                });
+                if is_assertion {
+                    let msg = format!("{e:?}");
+                    let err = ExecutionError::AssertionFailed(
+                        acvm::pwg::ResolvedAssertionPayload::String(msg),
+                        vec![],
+                        None,
+                    );
+                    let res1 = Err(NargoError::ExecutionError(err));
+                    return CompareCompiledResult::new(
+                        &self.abi,
+                        (res1, "".to_string()),
+                        (res2, print2),
+                    );
+                } else {
+                    panic!("failed to compile program:\n{e:?}\n{}", self.source);
+                }
+            }
+        };
+        // Capture any println that happened during the compilation, which in these tests should be the whole program.
+        let comptime_print = String::from_utf8(output1).expect("should be valid utf8 string");
+
+        // Execute the 1st (comptime) program.
+        let (res1, print1) = do_exec(&program1.program);
+
         CompareCompiledResult::new(&self.abi, (res1, comptime_print + &print1), (res2, print2))
     }