Free the blobs

fix the types

rejig the new constants in params

add comment for cleanup

update reqresp

fix api package

rename blobs repo

commit the wip modifications

further appropriate renaming

further references update

further reference updates

continue refac

fix reqresp build

further refac

further refac

fix api

fix db interface

fix beacondb alloc

build

fix api

improve blob verificaion

correct validation call

fixes

fix the produce block/blobs flow

reduce diff

blob gossip validation

update validations

cleanup block vali

reduce diff

handle gossip of block and blob

fix test for timebeing

modify publishing flow

fix import flow

onsidecarbyrange fix and some type fixes

fix sidecars by root

prune blockinput cache

fix kzg interface

small renaming

interface rename

fix fetch blockmaybeblobs by range test

fix build lint issues for now

c-kzg version fix

FullOrBlindedBlobSidecar changes

fix tests

complete the blob publishing flow

fix test

get the single node run functional

get the gossip blob flow working

fix peer syncing using req/resp

fix sidecar by root check

refactor blobsidecars hotdb and remove archive

add blob gossip validation flow

fix topic

fix the validation condition

add blob validation and test various sync modes

fix tests

rebase fixes

enable deneb spec tests

make blobsbyroot multi block

fixes

cleanup defunt builder endpoint

archive blobs post finalization uptill the blob window

serve finalized blobs within the blob prune window

fix test

fix test

lookup in archive as well

cleanup and improvements

rebase fx

Add 4844 sim test and override the field elements per blob

update image

add blob test

add test run in package

start unknown sync and range sync

finalize the sims

change the signing flow

fix test types

fix tests

fix test

lint

update tx type and corresponding ethereumjs image

update c-kzg and use blobs bundle proof

fix test

fix test

merge getblobsbundle into getpayloadv3

update images

fix genesis config

rebase fixes

fix test

update images

fix tests

lint

fix the sidecar request count limit

fix test

lint

rebase fixes

cleanup

fix unit tests

update kzg to big endian

devnet 6 integration

fix passing setup arg

update path

fix tests

fix blobs sidecar by range response

cleanup blob import vals

reduce diff

Author: g11tech

.github/workflows/test-sim-merge.yml

@@ -17,8 +17,9 @@ env:
   NETHERMIND_IMAGE: nethermind/nethermind:1.14.3
   MERGEMOCK_IMAGE: g11tech/mergemock:latest
   GETH_WITHDRAWALS_IMAGE: g11tech/geth:withdrawalsfeb8
-  ETHEREUMJS_WITHDRAWALS_IMAGE: g11tech/ethereumjs:feb8
+  ETHEREUMJS_WITHDRAWALS_IMAGE: g11tech/ethereumjs:blobs-b6b63
   NETHERMIND_WITHDRAWALS_IMAGE: nethermindeth/nethermind:withdrawals_yolo
+  ETHEREUMJS_BLOBS_IMAGE: g11tech/ethereumjs:blobs-b6b63
 
 jobs:
   sim-merge-tests:
@@ -128,6 +129,17 @@ jobs:
       #     EL_BINARY_DIR: ${{ env.NETHERMIND_WITHDRAWALS_IMAGE }}
       #     EL_SCRIPT_DIR: netherminddocker
 
+      - name: Pull ethereumjs blobs
+        run: docker pull $ETHEREUMJS_BLOBS_IMAGE
+
+      - name: Test Lodestar <> ethereumjs blobs
+        run: yarn test:sim:blobs
+        working-directory: packages/beacon-node
+        env:
+          EL_BINARY_DIR: ${{ env.ETHEREUMJS_BLOBS_IMAGE }}
+          EL_SCRIPT_DIR: ethereumjsdocker
+          DEV_RUN: true
+
       - name: Upload debug log test files
         if: ${{ always() }}
         uses: actions/upload-artifact@v2

packages/beacon-node/package.json

@@ -85,6 +85,7 @@
     "test:sim:merge-interop": "mocha 'test/sim/merge-interop.test.ts'",
     "test:sim:mergemock": "mocha 'test/sim/mergemock.test.ts'",
     "test:sim:withdrawals": "mocha 'test/sim/withdrawal-interop.test.ts'",
+    "test:sim:blobs": "mocha 'test/sim/4844-interop.test.ts'",
     "download-spec-tests": "node --loader=ts-node/esm test/spec/downloadTests.ts",
     "check-spec-tests": "mocha test/spec/checkCoverage.ts",
     "test:spec-bls-general": "mocha --config .mocharc.spec.cjs 'test/spec/bls/**/*.test.ts' 'test/spec/general/**/*.test.ts'",

packages/beacon-node/src/api/impl/beacon/blocks/index.ts

@@ -4,13 +4,7 @@ import {SLOTS_PER_HISTORICAL_ROOT} from "@lodestar/params";
 import {sleep} from "@lodestar/utils";
 import {allForks, deneb} from "@lodestar/types";
 import {fromHexString, toHexString} from "@chainsafe/ssz";
-import {
-  BlockSource,
-  getBlockInput,
-  ImportBlockOpts,
-  BlockInput,
-  blobSidecarsToBlobsSidecar,
-} from "../../../../chain/blocks/types.js";
+import {BlockSource, getBlockInput, ImportBlockOpts, BlockInput} from "../../../../chain/blocks/types.js";
 import {promiseAllMaybeAsync} from "../../../../util/promises.js";
 import {isOptimisticBlock} from "../../../../util/forkChoice.js";
 import {BlockError, BlockErrorCode} from "../../../../chain/errors/index.js";
@@ -213,12 +207,7 @@ export function getBeaconBlockApi({
           config,
           signedBlock,
           BlockSource.api,
-          // The blobsSidecar will be replaced in the followup PRs with just blobs
-          blobSidecarsToBlobsSidecar(
-            config,
-            signedBlock,
-            signedBlobs.map((sblob) => sblob.message)
-          ),
+          signedBlobs.map((sblob) => sblob.message),
           null
         );
       } else {

packages/beacon-node/src/chain/blocks/types.ts

@@ -1,10 +1,10 @@
 import {CachedBeaconStateAllForks, computeEpochAtSlot, DataAvailableStatus} from "@lodestar/state-transition";
 import {MaybeValidExecutionStatus} from "@lodestar/fork-choice";
-import {allForks, deneb, Slot} from "@lodestar/types";
+import {allForks, deneb, Slot, RootHex} from "@lodestar/types";
 import {ForkSeq, MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS} from "@lodestar/params";
 import {ChainForkConfig} from "@lodestar/config";
-
-import {ckzg} from "../../util/kzg.js";
+import {toHexString} from "@chainsafe/ssz";
+import {pruneSetToMax} from "@lodestar/utils";
 
 export enum BlockInputType {
   preDeneb = "preDeneb",
@@ -21,7 +21,7 @@ export enum BlockSource {
 
 export type BlockInput = {block: allForks.SignedBeaconBlock; source: BlockSource; blockBytes: Uint8Array | null} & (
   | {type: BlockInputType.preDeneb}
-  | {type: BlockInputType.postDeneb; blobs: deneb.BlobsSidecar}
+  | {type: BlockInputType.postDeneb; blobs: deneb.BlobSidecars}
 );
 
 export function blockRequiresBlobs(config: ChainForkConfig, blockSlot: Slot, clockSlot: Slot): boolean {
@@ -32,26 +32,89 @@ export function blockRequiresBlobs(config: ChainForkConfig, blockSlot: Slot, clo
   );
 }
 
-// TODO DENEB: a helper function to convert blobSidecars to blobsSidecar, to be cleanup on BlockInput
-// migration
-export function blobSidecarsToBlobsSidecar(
-  config: ChainForkConfig,
-  signedBlock: allForks.SignedBeaconBlock,
-  blobSidecars: deneb.BlobSidecars
-): deneb.BlobsSidecar {
-  const beaconBlockSlot = signedBlock.message.slot;
-  const beaconBlockRoot = config.getForkTypes(beaconBlockSlot).BeaconBlock.hashTreeRoot(signedBlock.message);
-  const blobs = blobSidecars.map(({blob}) => blob);
-  const blobsSidecar = {
-    beaconBlockRoot,
-    beaconBlockSlot,
-    blobs,
-    kzgAggregatedProof: ckzg.computeAggregateKzgProof(blobs),
-  };
-  return blobsSidecar;
+export enum GossipedInputType {
+  block = "block",
+  blob = "blob",
 }
+type GossipedBlockInput = {serializedData: Uint8Array | null} & (
+  | {type: GossipedInputType.block; signedBlock: allForks.SignedBeaconBlock}
+  | {type: GossipedInputType.blob; signedBlob: deneb.SignedBlobSidecar}
+);
+type BlockInputCacheType = {block?: allForks.SignedBeaconBlock; blobs: Map<number, deneb.BlobSidecar>};
+
+const MAX_GOSSIPINPUT_CACHE = 5;
 
 export const getBlockInput = {
+  blockInputCache: new Map<RootHex, BlockInputCacheType>(),
+
+  getFullBlockInput(
+    config: ChainForkConfig,
+    gossipedInput: GossipedBlockInput
+  ):
+    | {blockInput: BlockInput; blockInputMeta: {pending: null; haveBlobs: number; expectedBlobs: number}}
+    | {blockInput: null; blockInputMeta: {pending: GossipedInputType.block; haveBlobs: number; expectedBlobs: null}}
+    | {blockInput: null; blockInputMeta: {pending: GossipedInputType.blob; haveBlobs: number; expectedBlobs: number}} {
+    let blockHex;
+    let blockCache;
+    if (gossipedInput.type === GossipedInputType.block) {
+      const {signedBlock} = gossipedInput;
+      blockHex = toHexString(
+        config.getForkTypes(signedBlock.message.slot).BeaconBlock.hashTreeRoot(signedBlock.message)
+      );
+      blockCache = this.blockInputCache.get(blockHex) ?? {blobs: new Map<number, deneb.BlobSidecar>()};
+      blockCache.block = signedBlock;
+    } else {
+      const {signedBlob} = gossipedInput;
+      blockHex = toHexString(signedBlob.message.blockRoot);
+      blockCache = this.blockInputCache.get(blockHex);
+
+      // If a new entry is going to be inserted, prune out old ones
+      if (blockCache === undefined) {
+        pruneSetToMax(this.blockInputCache, MAX_GOSSIPINPUT_CACHE);
+        blockCache = {blobs: new Map<number, deneb.BlobSidecar>()};
+      }
+      // TODO: freetheblobs check if its the same blob or a duplicate and throw/take actions
+      blockCache.blobs.set(signedBlob.message.index, signedBlob.message);
+    }
+    this.blockInputCache.set(blockHex, blockCache);
+    const {block: signedBlock} = blockCache;
+    if (signedBlock !== undefined) {
+      const {blobKzgCommitments} = (signedBlock as deneb.SignedBeaconBlock).message.body;
+      if (blobKzgCommitments.length < blockCache.blobs.size) {
+        throw Error(`Received more blobs=${blockCache.blobs.size} than commitments=${blobKzgCommitments.length}`);
+      }
+      if (blobKzgCommitments.length === blockCache.blobs.size) {
+        const blobSidecars = [];
+        for (let index = 0; index < blobKzgCommitments.length; index++) {
+          const blobSidecar = blockCache.blobs.get(index);
+          if (blobSidecar === undefined) {
+            throw Error("Missing blobSidecar");
+          }
+          blobSidecars.push(blobSidecar);
+        }
+        return {
+          // TODO freetheblobs: collate and add serialized data for the postDeneb blockinput
+          blockInput: getBlockInput.postDeneb(config, signedBlock, BlockSource.gossip, blobSidecars, null),
+          blockInputMeta: {pending: null, haveBlobs: blockCache.blobs.size, expectedBlobs: blobKzgCommitments.length},
+        };
+      } else {
+        return {
+          blockInput: null,
+          blockInputMeta: {
+            pending: GossipedInputType.blob,
+            haveBlobs: blockCache.blobs.size,
+            expectedBlobs: blobKzgCommitments.length,
+          },
+        };
+      }
+    } else {
+      return {
+        blockInput: null,
+        blockInputMeta: {pending: GossipedInputType.block, haveBlobs: blockCache.blobs.size, expectedBlobs: null},
+      };
+    }
+  },
+
   preDeneb(
     config: ChainForkConfig,
     block: allForks.SignedBeaconBlock,
@@ -73,7 +136,7 @@ export const getBlockInput = {
     config: ChainForkConfig,
     block: allForks.SignedBeaconBlock,
     source: BlockSource,
-    blobs: deneb.BlobsSidecar,
+    blobs: deneb.BlobSidecars,
     blockBytes: Uint8Array | null
   ): BlockInput {
     if (config.getForkSeq(block.message.slot) < ForkSeq.deneb) {
@@ -127,8 +190,8 @@ export type ImportBlockOpts = {
    * Metadata: `true` if all the signatures including the proposer signature have been verified
    */
   validSignatures?: boolean;
-  /** Set to true if already run `validateBlobsSidecar()` sucessfully on the blobs */
-  validBlobsSidecar?: boolean;
+  /** Set to true if already run `validateBlobSidecars()` sucessfully on the blobs */
+  validBlobSidecars?: boolean;
   /** Seen timestamp seconds */
   seenTimestampSec?: number;
   /** Set to true if persist block right at verification time */

packages/beacon-node/src/chain/blocks/verifyBlocksSanityChecks.ts

@@ -5,7 +5,7 @@ import {Slot, deneb} from "@lodestar/types";
 import {toHexString} from "@lodestar/utils";
 import {IClock} from "../../util/clock.js";
 import {BlockError, BlockErrorCode} from "../errors/index.js";
-import {validateBlobsSidecar} from "../validation/blobsSidecar.js";
+import {validateBlobSidecars} from "../validation/blobSidecar.js";
 import {BlockInput, BlockInputType, ImportBlockOpts} from "./types.js";
 
 /**
@@ -126,7 +126,7 @@ function maybeValidateBlobs(
   // TODO Deneb: Make switch verify it's exhaustive
   switch (blockInput.type) {
     case BlockInputType.postDeneb: {
-      if (opts.validBlobsSidecar) {
+      if (opts.validBlobSidecars) {
         return DataAvailableStatus.available;
       }
 
@@ -135,7 +135,7 @@ function maybeValidateBlobs(
       const {blobKzgCommitments} = (block as deneb.SignedBeaconBlock).message.body;
       const beaconBlockRoot = config.getForkTypes(blockSlot).BeaconBlock.hashTreeRoot(block.message);
       // TODO Deneb: This function throws un-typed errors
-      validateBlobsSidecar(blockSlot, beaconBlockRoot, blobKzgCommitments, blobs);
+      validateBlobSidecars(blockSlot, beaconBlockRoot, blobKzgCommitments, blobs);
 
       return DataAvailableStatus.available;
     }

packages/beacon-node/src/chain/blocks/writeBlockInputToDb.ts

@@ -1,4 +1,3 @@
-import {allForks, deneb} from "@lodestar/types";
 import {toHex} from "@lodestar/utils";
 import {BeaconChain} from "../chain.js";
 import {BlockInput, BlockInputType} from "./types.js";
@@ -31,13 +30,13 @@ export async function writeBlockInputToDb(this: BeaconChain, blocksInput: BlockI
     });
 
     if (type === BlockInputType.postDeneb) {
-      const {blobs} = blockInput;
+      const {blobs: blobSidecars} = blockInput;
       // NOTE: Old blobs are pruned on archive
-      fnPromises.push(this.db.blobsSidecar.add(blobs));
-      this.logger.debug("Persist blobsSidecar to hot DB", {
-        blobsLen: blobs.blobs.length,
-        slot: blobs.beaconBlockSlot,
-        root: toHex(blobs.beaconBlockRoot),
+      fnPromises.push(this.db.blobSidecars.add({blockRoot, slot: block.message.slot, blobSidecars}));
+      this.logger.debug("Persisted blobSidecars to hot DB", {
+        blobsLen: blobSidecars.length,
+        slot: block.message.slot,
+        root: blockRootHex,
       });
     }
   }
@@ -49,27 +48,26 @@ export async function writeBlockInputToDb(this: BeaconChain, blocksInput: BlockI
  * Prunes eagerly persisted block inputs only if not known to the fork-choice
  */
 export async function removeEagerlyPersistedBlockInputs(this: BeaconChain, blockInputs: BlockInput[]): Promise<void> {
-  const blockToRemove: allForks.SignedBeaconBlock[] = [];
-  const blobsToRemove: deneb.BlobsSidecar[] = [];
+  const blockToRemove = [];
+  const blobsToRemove = [];
 
   for (const blockInput of blockInputs) {
     const {block, type} = blockInput;
-    const blockRoot = toHex(this.config.getForkTypes(block.message.slot).BeaconBlock.hashTreeRoot(block.message));
-    if (!this.forkChoice.hasBlockHex(blockRoot)) {
+    const blockRoot = this.config.getForkTypes(block.message.slot).BeaconBlock.hashTreeRoot(block.message);
+    const blockRootHex = toHex(blockRoot);
+    if (!this.forkChoice.hasBlockHex(blockRootHex)) {
       blockToRemove.push(block);
 
       if (type === BlockInputType.postDeneb) {
-        blobsToRemove.push(blockInput.blobs);
-        this.db.blobsSidecar.remove(blockInput.blobs).catch((e) => {
-          this.logger.verbose("Error removing eagerly imported blobsSidecar", {blockRoot}, e);
-        });
+        const blobSidecars = blockInput.blobs;
+        blobsToRemove.push({blockRoot, slot: block.message.slot, blobSidecars});
       }
     }
   }
 
   await Promise.all([
     // TODO: Batch DB operations not with Promise.all but with level db ops
     this.db.block.batchRemove(blockToRemove),
-    this.db.blobsSidecar.batchRemove(blobsToRemove),
+    this.db.blobSidecars.batchRemove(blobsToRemove),
   ]);
 }

…de/src/chain/errors/blobsSidecarError.ts …ode/src/chain/errors/blobSidecarError.tspackages/beacon-node/src/chain/errors/blobsSidecarError.ts renamed to packages/beacon-node/src/chain/errors/blobSidecarError.ts packages/beacon-node/src/chain/errors/blobsSidecarError.ts renamed to packages/beacon-node/src/chain/errors/blobSidecarError.ts

@@ -1,7 +1,8 @@
 import {Slot} from "@lodestar/types";
 import {GossipActionError} from "./gossipValidation.js";
 
-export enum BlobsSidecarErrorCode {
+export enum BlobSidecarErrorCode {
+  INVALID_INDEX = "BLOBS_SIDECAR_ERROR_INVALID_INDEX",
   /** !bls.KeyValidate(block.body.blob_kzg_commitments[i]) */
   INVALID_KZG = "BLOBS_SIDECAR_ERROR_INVALID_KZG",
   /** !verify_kzg_commitments_against_transactions(block.body.execution_payload.transactions, block.body.blob_kzg_commitments) */
@@ -14,11 +15,12 @@ export enum BlobsSidecarErrorCode {
   INVALID_KZG_PROOF = "BLOBS_SIDECAR_ERROR_INVALID_KZG_PROOF",
 }
 
-export type BlobsSidecarErrorType =
-  | {code: BlobsSidecarErrorCode.INVALID_KZG; kzgIdx: number}
-  | {code: BlobsSidecarErrorCode.INVALID_KZG_TXS}
-  | {code: BlobsSidecarErrorCode.INCORRECT_SLOT; blockSlot: Slot; blobSlot: Slot}
-  | {code: BlobsSidecarErrorCode.INVALID_BLOB; blobIdx: number}
-  | {code: BlobsSidecarErrorCode.INVALID_KZG_PROOF};
+export type BlobSidecarErrorType =
+  | {code: BlobSidecarErrorCode.INVALID_INDEX; blobIdx: number; gossipIndex: number}
+  | {code: BlobSidecarErrorCode.INVALID_KZG; blobIdx: number}
+  | {code: BlobSidecarErrorCode.INVALID_KZG_TXS}
+  | {code: BlobSidecarErrorCode.INCORRECT_SLOT; blockSlot: Slot; blobSlot: Slot; blobIdx: number}
+  | {code: BlobSidecarErrorCode.INVALID_BLOB; blobIdx: number}
+  | {code: BlobSidecarErrorCode.INVALID_KZG_PROOF; blobIdx: number};
 
-export class BlobsSidecarError extends GossipActionError<BlobsSidecarErrorType> {}
+export class BlobSidecarError extends GossipActionError<BlobSidecarErrorType> {}

packages/beacon-node/src/chain/errors/index.ts

@@ -1,6 +1,6 @@
 export * from "./attestationError.js";
 export * from "./attesterSlashingError.js";
-export * from "./blobsSidecarError.js";
+export * from "./blobSidecarError.js";
 export * from "./blockError.js";
 export * from "./gossipValidation.js";
 export * from "./proposerSlashingError.js";

packages/beacon-node/src/chain/regen/interface.ts

@@ -9,6 +9,7 @@ export enum RegenCaller {
   processBlock = "processBlock",
   produceBlock = "produceBlock",
   validateGossipBlock = "validateGossipBlock",
+  validateGossipBlob = "validateGossipBlob",
   precomputeEpoch = "precomputeEpoch",
   produceAttestationData = "produceAttestationData",
   processBlocksInEpoch = "processBlocksInEpoch",

packages/beacon-node/src/chain/regen/regen.ts

@@ -3,8 +3,8 @@ import {
   CachedBeaconStateAllForks,
   computeEpochAtSlot,
   computeStartSlotAtEpoch,
-  DataAvailableStatus,
   ExecutionPayloadStatus,
+  DataAvailableStatus,
   processSlots,
   stateTransition,
 } from "@lodestar/state-transition";