ContextDeserialize and Beacon API Improvements (#7372)

* #7286
* BeaconAPI is not returning a versioned response when it should for some V1 endpoints
* these [strange functions with vX in the name that still accept `endpoint_version` arguments](https://github.com/sigp/lighthouse/blob/stable/beacon_node/http_api/src/produce_block.rs#L192)

This refactor is a prerequisite to get the fulu EF tests running.

Author: ethDreamer

Cargo.lock

@@ -53,6 +53,8 @@ members = [
     "common/warp_utils",
     "common/workspace_members",
 
+    "consensus/context_deserialize",
+    "consensus/context_deserialize_derive",
     "consensus/fixed_bytes",
     "consensus/fork_choice",
     "consensus/int_to_bytes",
@@ -127,6 +129,8 @@ clap = { version = "4.5.4", features = ["derive", "cargo", "wrap_help"] }
 # feature ourselves when desired.
 c-kzg = { version = "1", default-features = false }
 compare_fields_derive = { path = "common/compare_fields_derive" }
+context_deserialize = { path = "consensus/context_deserialize" }
+context_deserialize_derive = { path = "consensus/context_deserialize_derive" }
 criterion = "0.5"
 delay_map = "0.4"
 derivative = "2"

Cargo.toml

@@ -6115,7 +6115,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
                         payload_attributes: payload_attributes.into(),
                     },
                     metadata: Default::default(),
-                    version: Some(self.spec.fork_name_at_slot::<T::EthSpec>(prepare_slot)),
+                    version: self.spec.fork_name_at_slot::<T::EthSpec>(prepare_slot),
                 }));
             }
         }

beacon_node/beacon_chain/src/beacon_chain.rs

@@ -1,7 +1,7 @@
+use eth2::types::beacon_response::EmptyMetadata;
 use eth2::types::builder_bid::SignedBuilderBid;
-use eth2::types::fork_versioned_response::EmptyMetadata;
 use eth2::types::{
-    ContentType, EthSpec, ExecutionBlockHash, ForkName, ForkVersionDecode, ForkVersionDeserialize,
+    ContentType, ContextDeserialize, EthSpec, ExecutionBlockHash, ForkName, ForkVersionDecode,
     ForkVersionedResponse, PublicKeyBytes, SignedValidatorRegistrationData, Slot,
 };
 use eth2::types::{FullPayloadContents, SignedBlindedBeaconBlock};
@@ -119,7 +119,7 @@ impl BuilderHttpClient {
     }
 
     async fn get_with_header<
-        T: DeserializeOwned + ForkVersionDecode + ForkVersionDeserialize,
+        T: DeserializeOwned + ForkVersionDecode + for<'de> ContextDeserialize<'de, ForkName>,
         U: IntoUrl,
     >(
         &self,
@@ -147,15 +147,23 @@ impl BuilderHttpClient {
                 self.ssz_available.store(true, Ordering::SeqCst);
                 T::from_ssz_bytes_by_fork(&response_bytes, fork_name)
                     .map(|data| ForkVersionedResponse {
-                        version: Some(fork_name),
+                        version: fork_name,
                         metadata: EmptyMetadata {},
                         data,
                     })
                     .map_err(Error::InvalidSsz)
             }
             ContentType::Json => {
                 self.ssz_available.store(false, Ordering::SeqCst);
-                serde_json::from_slice(&response_bytes).map_err(Error::InvalidJson)
+                let mut de = serde_json::Deserializer::from_slice(&response_bytes);
+                let data =
+                    T::context_deserialize(&mut de, fork_name).map_err(Error::InvalidJson)?;
+
+                Ok(ForkVersionedResponse {
+                    version: fork_name,
+                    metadata: EmptyMetadata {},
+                    data,
+                })
             }
         }
     }

beacon_node/builder_client/src/lib.rs

@@ -463,12 +463,12 @@ where
                 let blobs = if block.message().body().has_blobs() {
                     debug!("Downloading finalized blobs");
                     if let Some(response) = remote
-                        .get_blobs::<E>(BlockId::Root(block_root), None)
+                        .get_blobs::<E>(BlockId::Root(block_root), None, &spec)
                         .await
                         .map_err(|e| format!("Error fetching finalized blobs from remote: {e:?}"))?
                     {
                         debug!("Downloaded finalized blobs");
-                        Some(response.data)
+                        Some(response.into_data())
                     } else {
                         warn!(
                             block_root = %block_root,

beacon_node/client/src/builder.rs

@@ -1980,7 +1980,7 @@ enum InvalidBuilderPayload {
         expected: Option<u64>,
     },
     Fork {
-        payload: Option<ForkName>,
+        payload: ForkName,
         expected: ForkName,
     },
     Signature {
@@ -2013,7 +2013,7 @@ impl fmt::Display for InvalidBuilderPayload {
                 write!(f, "payload block number was {} not {:?}", payload, expected)
             }
             InvalidBuilderPayload::Fork { payload, expected } => {
-                write!(f, "payload fork was {:?} not {}", payload, expected)
+                write!(f, "payload fork was {} not {}", payload, expected)
             }
             InvalidBuilderPayload::Signature { signature, pubkey } => write!(
                 f,
@@ -2116,7 +2116,7 @@ fn verify_builder_bid<E: EthSpec>(
             payload: header.block_number(),
             expected: block_number,
         }))
-    } else if bid.version != Some(current_fork) {
+    } else if bid.version != current_fork {
         Err(Box::new(InvalidBuilderPayload::Fork {
             payload: bid.version,
             expected: current_fork,

beacon_node/execution_layer/src/lib.rs

@@ -743,7 +743,7 @@ impl<E: EthSpec> MockBuilder<E> {
             .await
             .map_err(|_| "couldn't get head".to_string())?
             .ok_or_else(|| "missing head block".to_string())?
-            .data;
+            .into_data();
 
         let head_block_root = head_block_root.unwrap_or(head.canonical_root());
 
@@ -761,7 +761,7 @@ impl<E: EthSpec> MockBuilder<E> {
             .await
             .map_err(|_| "couldn't get finalized block".to_string())?
             .ok_or_else(|| "missing finalized block".to_string())?
-            .data
+            .data()
             .message()
             .body()
             .execution_payload()
@@ -774,7 +774,7 @@ impl<E: EthSpec> MockBuilder<E> {
             .await
             .map_err(|_| "couldn't get justified block".to_string())?
             .ok_or_else(|| "missing justified block".to_string())?
-            .data
+            .data()
             .message()
             .body()
             .execution_payload()
@@ -815,7 +815,7 @@ impl<E: EthSpec> MockBuilder<E> {
             .await
             .map_err(|_| "couldn't get state".to_string())?
             .ok_or_else(|| "missing state".to_string())?
-            .data;
+            .into_data();
 
         let prev_randao = head_state
             .get_randao_mix(head_state.current_epoch())
@@ -980,7 +980,7 @@ pub fn serve<E: EthSpec>(
                         .await
                         .map_err(|e| warp::reject::custom(Custom(e)))?;
                     let resp: ForkVersionedResponse<_> = ForkVersionedResponse {
-                        version: Some(fork_name),
+                        version: fork_name,
                         metadata: Default::default(),
                         data: payload,
                     };
@@ -1040,7 +1040,7 @@ pub fn serve<E: EthSpec>(
                     ),
                     eth2::types::Accept::Json | eth2::types::Accept::Any => {
                         let resp: ForkVersionedResponse<_> = ForkVersionedResponse {
-                            version: Some(fork_name),
+                            version: fork_name,
                             metadata: Default::default(),
                             data: signed_bid,
                         };

beacon_node/execution_layer/src/test_utils/mock_builder.rs

@@ -4,7 +4,7 @@ use crate::version::{add_consensus_version_header, V1, V2};
 use beacon_chain::{BeaconChain, BeaconChainTypes};
 use eth2::types::{self, EndpointVersion, Hash256, Slot};
 use std::sync::Arc;
-use types::fork_versioned_response::EmptyMetadata;
+use types::beacon_response::EmptyMetadata;
 use types::{CommitteeIndex, ForkVersionedResponse};
 use warp::{
     hyper::{Body, Response},
@@ -52,7 +52,7 @@ pub fn get_aggregate_attestation<T: BeaconChainTypes>(
 
     if endpoint_version == V2 {
         let fork_versioned_response = ForkVersionedResponse {
-            version: Some(fork_name),
+            version: fork_name,
             metadata: EmptyMetadata {},
             data: aggregate_attestation,
         };