feat: MessagePack serialisation for circuits

Cargo.toml

@@ -144,6 +144,10 @@ prost = "0.13"
 prost-build = "0.13"
 protoc-prebuilt = "0.3"
 
+# MessagePack
+rmp = "0.8.14"
+rmp-serde = "1.3.0"
+
 cfg-if = "1.0.0"
 dirs = "4"
 serde = { version = "1.0.136", features = ["derive"] }

acvm-repo/acir/Cargo.toml

@@ -27,6 +27,7 @@ flate2.workspace = true
 bincode.workspace = true
 base64.workspace = true
 prost.workspace = true
+rmp-serde.workspace = true
 serde-big-array = "0.5.1"
 strum = { workspace = true }
 strum_macros = { workspace = true }

acvm-repo/acir/src/circuit/mod.rs

@@ -4,10 +4,9 @@ pub mod opcodes;
 
 use crate::{
     native_types::{Expression, Witness},
-    proto::convert::ProtoSchema,
+    serialization::{bincode_deserialize, bincode_serialize},
 };
 use acir_field::AcirField;
-use noir_protobuf::ProtoCodec as _;
 pub use opcodes::Opcode;
 use thiserror::Error;
 
@@ -248,8 +247,9 @@ impl<F: AcirField> Circuit<F> {
 }
 
 impl<F: Serialize + AcirField> Program<F> {
+    /// Serialize and compress the [Program] into bytes.
     fn write<W: Write>(&self, writer: W) -> std::io::Result<()> {
-        let buf = self.bincode_serialize()?;
+        let buf = bincode_serialize(self)?;
         let mut encoder = flate2::write::GzEncoder::new(writer, Compression::default());
         encoder.write_all(&buf)?;
         encoder.finish()?;
@@ -273,38 +273,13 @@ impl<F: Serialize + AcirField> Program<F> {
     }
 }
 
-impl<F: Serialize + AcirField> Program<F> {
-    /// Serialize the program using `bincode`, which is what we have to use until Barretenberg can read another format.
-    pub(crate) fn bincode_serialize(&self) -> std::io::Result<Vec<u8>> {
-        bincode::serialize(self).map_err(std::io::Error::other)
-    }
-}
-
-impl<F: AcirField + for<'a> Deserialize<'a>> Program<F> {
-    pub(crate) fn bincode_deserialize(buf: &[u8]) -> std::io::Result<Self> {
-        bincode::deserialize(buf)
-            .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidInput, e))
-    }
-}
-
-#[allow(dead_code)] // TODO: Remove once we switch to protobuf
-impl<F: AcirField> Program<F> {
-    /// Serialize the program using `protobuf`, which is what we try to replace `bincode` with.
-    pub(crate) fn proto_serialize(&self) -> Vec<u8> {
-        ProtoSchema::<F>::serialize_to_vec(self)
-    }
-    pub(crate) fn proto_deserialize(buf: &[u8]) -> std::io::Result<Self> {
-        ProtoSchema::<F>::deserialize_from_vec(buf)
-            .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidInput, e))
-    }
-}
-
 impl<F: AcirField + for<'a> Deserialize<'a>> Program<F> {
+    /// Decompress and deserialize bytes into a [Program].
     fn read<R: Read>(reader: R) -> std::io::Result<Self> {
         let mut gz_decoder = flate2::read::GzDecoder::new(reader);
         let mut buf = Vec::new();
         gz_decoder.read_to_end(&mut buf)?;
-        let program = Self::bincode_deserialize(&buf)?;
+        let program = bincode_deserialize(&buf)?;
         Ok(program)
     }
 
@@ -523,6 +498,7 @@ mod tests {
 
         use crate::circuit::Program;
         use crate::native_types::{WitnessMap, WitnessStack};
+        use crate::serialization::*;
 
         // It's not possible to set the maximum size of collections via `ProptestConfig`, only an env var,
         // because e.g. the `VecStrategy` uses `Config::default().max_default_size_range`. On top of that,
@@ -576,8 +552,8 @@ mod tests {
         #[test]
         fn prop_program_proto_roundtrip() {
             run_with_max_size_range(100, |program: Program<TestField>| {
-                let bz = Program::proto_serialize(&program);
-                let de = Program::proto_deserialize(&bz)?;
+                let bz = proto_serialize(&program);
+                let de = proto_deserialize(&bz)?;
                 prop_assert_eq!(program, de);
                 Ok(())
             });
@@ -586,8 +562,18 @@ mod tests {
         #[test]
         fn prop_program_bincode_roundtrip() {
             run_with_max_size_range(100, |program: Program<TestField>| {
-                let bz = Program::bincode_serialize(&program)?;
-                let de = Program::bincode_deserialize(&bz)?;
+                let bz = bincode_serialize(&program)?;
+                let de = bincode_deserialize(&bz)?;
+                prop_assert_eq!(program, de);
+                Ok(())
+            });
+        }
+
+        #[test]
+        fn prop_program_msgpack_roundtrip() {
+            run_with_max_size_range(100, |(program, compact): (Program<TestField>, bool)| {
+                let bz = msgpack_serialize(&program, compact)?;
+                let de = msgpack_deserialize(&bz)?;
                 prop_assert_eq!(program, de);
                 Ok(())
             });
@@ -606,8 +592,8 @@ mod tests {
         #[test]
         fn prop_witness_stack_proto_roundtrip() {
             run_with_max_size_range(10, |witness: WitnessStack<TestField>| {
-                let bz = WitnessStack::proto_serialize(&witness);
-                let de = WitnessStack::proto_deserialize(&bz)?;
+                let bz = proto_serialize(&witness);
+                let de = proto_deserialize(&bz)?;
                 prop_assert_eq!(witness, de);
                 Ok(())
             });
@@ -616,8 +602,18 @@ mod tests {
         #[test]
         fn prop_witness_stack_bincode_roundtrip() {
             run_with_max_size_range(10, |witness: WitnessStack<TestField>| {
-                let bz = WitnessStack::bincode_serialize(&witness)?;
-                let de = WitnessStack::bincode_deserialize(&bz)?;
+                let bz = bincode_serialize(&witness)?;
+                let de = bincode_deserialize(&bz)?;
+                prop_assert_eq!(witness, de);
+                Ok(())
+            });
+        }
+
+        #[test]
+        fn prop_witness_stack_msgpack_roundtrip() {
+            run_with_max_size_range(10, |(witness, compact): (WitnessStack<TestField>, bool)| {
+                let bz = msgpack_serialize(&witness, compact)?;
+                let de = msgpack_deserialize(&bz)?;
                 prop_assert_eq!(witness, de);
                 Ok(())
             });
@@ -636,8 +632,8 @@ mod tests {
         #[test]
         fn prop_witness_map_proto_roundtrip() {
             run_with_max_size_range(10, |witness: WitnessMap<TestField>| {
-                let bz = WitnessMap::proto_serialize(&witness);
-                let de = WitnessMap::proto_deserialize(&bz)?;
+                let bz = proto_serialize(&witness);
+                let de = proto_deserialize(&bz)?;
                 prop_assert_eq!(witness, de);
                 Ok(())
             });
@@ -646,8 +642,18 @@ mod tests {
         #[test]
         fn prop_witness_map_bincode_roundtrip() {
             run_with_max_size_range(10, |witness: WitnessMap<TestField>| {
-                let bz = WitnessMap::bincode_serialize(&witness)?;
-                let de = WitnessMap::bincode_deserialize(&bz)?;
+                let bz = bincode_serialize(&witness)?;
+                let de = bincode_deserialize(&bz)?;
+                prop_assert_eq!(witness, de);
+                Ok(())
+            });
+        }
+
+        #[test]
+        fn prop_witness_map_msgpack_roundtrip() {
+            run_with_max_size_range(10, |(witness, compact): (WitnessMap<TestField>, bool)| {
+                let bz = msgpack_serialize(&witness, compact)?;
+                let de = msgpack_deserialize(&bz)?;
                 prop_assert_eq!(witness, de);
                 Ok(())
             });

acvm-repo/acir/src/lib.rs

@@ -8,6 +8,7 @@
 pub mod circuit;
 pub mod native_types;
 mod proto;
+mod serialization;
 
 pub use acir_field;
 pub use acir_field::{AcirField, FieldElement};
@@ -19,7 +20,7 @@
 mod reflection {
     //! Getting test failures? You've probably changed the ACIR serialization format.
     //!
     //! These tests generate C++ deserializers for [`ACIR bytecode`][super::circuit::Circuit]
     //! and the [`WitnessMap`] structs. These get checked against the C++ files committed to the `codegen` folder
     //! to see if changes have been to the serialization format. These are almost always a breaking change!
     //!

acvm-repo/acir/src/native_types/witness_map.rs

@@ -6,13 +6,12 @@
 
 use acir_field::AcirField;
 use flate2::Compression;
 use flate2::bufread::GzDecoder;
 use flate2::bufread::GzEncoder;
-use noir_protobuf::ProtoCodec as _;
 use serde::{Deserialize, Serialize};
 use thiserror::Error;
 
-use crate::{native_types::Witness, proto::convert::ProtoSchema};
+use crate::native_types::Witness;
 
 #[derive(Debug, Error)]
 enum SerializationError {
@@ -96,18 +95,6 @@
     }
 }
 
-#[allow(dead_code)]
-impl<F: AcirField> WitnessMap<F> {
-    pub(crate) fn proto_serialize(&self) -> Vec<u8> {
-        ProtoSchema::<F>::serialize_to_vec(self)
-    }
-
-    pub(crate) fn proto_deserialize(buf: &[u8]) -> Result<Self, WitnessMapError> {
-        ProtoSchema::<F>::deserialize_from_vec(buf)
-            .map_err(|e| SerializationError::Deserialize(e.to_string()).into())
-    }
-}
-
 impl<F: Serialize + AcirField> TryFrom<WitnessMap<F>> for Vec<u8> {
     type Error = WitnessMapError;
 

acvm-repo/acir/src/native_types/witness_stack.rs

@@ -2,14 +2,11 @@
 
 use acir_field::AcirField;
 use flate2::Compression;
 use flate2::bufread::GzDecoder;
 use flate2::bufread::GzEncoder;
-use noir_protobuf::ProtoCodec as _;
 use serde::{Deserialize, Serialize};
 use thiserror::Error;
 
-use crate::proto::convert::ProtoSchema;
-
 use super::WitnessMap;
 
 #[derive(Debug, Error)]
@@ -82,18 +79,6 @@
     }
 }
 
-#[allow(dead_code)]
-impl<F: AcirField> WitnessStack<F> {
-    pub(crate) fn proto_serialize(&self) -> Vec<u8> {
-        ProtoSchema::<F>::serialize_to_vec(self)
-    }
-
-    pub(crate) fn proto_deserialize(buf: &[u8]) -> Result<Self, WitnessStackError> {
-        ProtoSchema::<F>::deserialize_from_vec(buf)
-            .map_err(|e| SerializationError::Deserialize(e.to_string()).into())
-    }
-}
-
 impl<F: Serialize + AcirField> TryFrom<&WitnessStack<F>> for Vec<u8> {
     type Error = WitnessStackError;