From 5ec4dac0cd423d88fbce12917f6917fb3ea5bdfb Mon Sep 17 00:00:00 2001 From: Agnish Ghosh Date: Tue, 1 Apr 2025 17:27:51 +0530 Subject: [PATCH 01/27] prepare for getBlobsV2 --- tests/test_execution_types.nim | 15 +++++++++++++++ web3/execution_types.nim | 11 +++++++++++ 2 files changed, 26 insertions(+) diff --git a/tests/test_execution_types.nim b/tests/test_execution_types.nim index 2b23eb4f..12ba471c 100644 --- a/tests/test_execution_types.nim +++ b/tests/test_execution_types.nim @@ -57,6 +57,11 @@ suite "Execution types tests": blobs: @[Blob.conv(3)], ) + blobAndProof = BlobAndProofV1( + blob: Blob.conv(1), + proof: KzgProof.conv(2), + ) + response = GetPayloadResponse( executionPayload: payload, blockValue: Opt.some(1.u256), @@ -64,6 +69,8 @@ suite "Execution types tests": shouldOverrideBuilder: Opt.some(false), ) + response_blob = @[blobAndProof] + test "payload version": var badv31 = payload badv31.blobGasUsed = Opt.none(Quantity) @@ -122,6 +129,14 @@ suite "Execution types tests": check v32.blobsBundle == response.blobsBundle.get check v32.shouldOverrideBuilder == false + test "response blob version": + var v11 = response_blob + v11[0].blob = default(Blob) + v11[0].proof = default(KzgProof) + + check v11.version == Version.V1 + check response_blob.version == Version.V1 + test "ExecutionPayload roundtrip": let v3 = payload.V3 check v3 == v3.executionPayload.V3 diff --git a/web3/execution_types.nim b/web3/execution_types.nim index 79d79fc9..1abd3fde 100644 --- a/web3/execution_types.nim +++ b/web3/execution_types.nim @@ -56,6 +56,8 @@ type shouldOverrideBuilder*: Opt[bool] executionRequests*: Opt[seq[seq[byte]]] + GetBlobsResponse* = seq[BlobAndProofV1] + Version* {.pure.} = enum V1 V2 @@ -88,6 +90,9 @@ func version*(res: GetPayloadResponse): Version = else: Version.V1 +func version*(res: GetBlobsResponse): Version = + Version.V1 + func V1V2*(attr: PayloadAttributes): PayloadAttributesV1OrV2 = PayloadAttributesV1OrV2( timestamp: attr.timestamp, @@ -388,6 +393,9 @@ func V4*(res: GetPayloadResponse): GetPayloadV4Response = executionRequests: res.executionRequests.get, ) +func V1*(res: GetBlobsResponse): GetBlobsV1Response = + res + func getPayloadResponse*(x: ExecutionPayloadV1): GetPayloadResponse = GetPayloadResponse(executionPayload: x.executionPayload) @@ -413,3 +421,6 @@ func getPayloadResponse*(x: GetPayloadV4Response): GetPayloadResponse = shouldOverrideBuilder: Opt.some(x.shouldOverrideBuilder), executionRequests: Opt.some(x.executionRequests), ) + +func getBlobsResponse*(x: GetBlobsV1Response): GetBlobsResponse = + x From faf3ac609f7352b41c6b462b27741e461bcbe298 Mon Sep 17 00:00:00 2001 From: Agnish Ghosh Date: Tue, 1 Apr 2025 17:31:51 +0530 Subject: [PATCH 02/27] Revert "prepare for getBlobsV2" This reverts commit 5ec4dac0cd423d88fbce12917f6917fb3ea5bdfb. --- tests/test_execution_types.nim | 15 --------------- web3/execution_types.nim | 11 ----------- 2 files changed, 26 deletions(-) diff --git a/tests/test_execution_types.nim b/tests/test_execution_types.nim index 12ba471c..2b23eb4f 100644 --- a/tests/test_execution_types.nim +++ b/tests/test_execution_types.nim @@ -57,11 +57,6 @@ suite "Execution types tests": blobs: @[Blob.conv(3)], ) - blobAndProof = BlobAndProofV1( - blob: Blob.conv(1), - proof: KzgProof.conv(2), - ) - response = GetPayloadResponse( executionPayload: payload, blockValue: Opt.some(1.u256), @@ -69,8 +64,6 @@ suite "Execution types tests": shouldOverrideBuilder: Opt.some(false), ) - response_blob = @[blobAndProof] - test "payload version": var badv31 = payload badv31.blobGasUsed = Opt.none(Quantity) @@ -129,14 +122,6 @@ suite "Execution types tests": check v32.blobsBundle == response.blobsBundle.get check v32.shouldOverrideBuilder == false - test "response blob version": - var v11 = response_blob - v11[0].blob = default(Blob) - v11[0].proof = default(KzgProof) - - check v11.version == Version.V1 - check response_blob.version == Version.V1 - test "ExecutionPayload roundtrip": let v3 = payload.V3 check v3 == v3.executionPayload.V3 diff --git a/web3/execution_types.nim b/web3/execution_types.nim index 1abd3fde..79d79fc9 100644 --- a/web3/execution_types.nim +++ b/web3/execution_types.nim @@ -56,8 +56,6 @@ type shouldOverrideBuilder*: Opt[bool] executionRequests*: Opt[seq[seq[byte]]] - GetBlobsResponse* = seq[BlobAndProofV1] - Version* {.pure.} = enum V1 V2 @@ -90,9 +88,6 @@ func version*(res: GetPayloadResponse): Version = else: Version.V1 -func version*(res: GetBlobsResponse): Version = - Version.V1 - func V1V2*(attr: PayloadAttributes): PayloadAttributesV1OrV2 = PayloadAttributesV1OrV2( timestamp: attr.timestamp, @@ -393,9 +388,6 @@ func V4*(res: GetPayloadResponse): GetPayloadV4Response = executionRequests: res.executionRequests.get, ) -func V1*(res: GetBlobsResponse): GetBlobsV1Response = - res - func getPayloadResponse*(x: ExecutionPayloadV1): GetPayloadResponse = GetPayloadResponse(executionPayload: x.executionPayload) @@ -421,6 +413,3 @@ func getPayloadResponse*(x: GetPayloadV4Response): GetPayloadResponse = shouldOverrideBuilder: Opt.some(x.shouldOverrideBuilder), executionRequests: Opt.some(x.executionRequests), ) - -func getBlobsResponse*(x: GetBlobsV1Response): GetBlobsResponse = - x From 3fffb22183fbf8e09e4f7e2cc92f3d4dc480bcc3 Mon Sep 17 00:00:00 2001 From: Agnish Ghosh Date: Sun, 6 Apr 2025 17:47:03 +0530 Subject: [PATCH 03/27] accomodate getBlobsV2 --- tests/test_json_marshalling.nim | 1 + web3/conversions.nim | 1 + web3/engine_api.nim | 8 ++++++++ web3/engine_api_types.nim | 6 ++++++ web3/primitives.nim | 3 +++ 5 files changed, 19 insertions(+) diff --git a/tests/test_json_marshalling.nim b/tests/test_json_marshalling.nim index 97f11a80..6d5ec1a7 100644 --- a/tests/test_json_marshalling.nim +++ b/tests/test_json_marshalling.nim @@ -204,6 +204,7 @@ suite "JSON-RPC Quantity": checkRandomObject(ExecutionPayloadV3) checkRandomObject(BlobsBundleV1) checkRandomObject(BlobAndProofV1) + checkRandomObject(BlobAndProofV2) checkRandomObject(ExecutionPayloadBodyV1) checkRandomObject(PayloadAttributesV1) checkRandomObject(PayloadAttributesV2) diff --git a/web3/conversions.nim b/web3/conversions.nim index b71fe050..fff14306 100644 --- a/web3/conversions.nim +++ b/web3/conversions.nim @@ -60,6 +60,7 @@ ExecutionPayloadV1OrV2.useDefaultSerializationIn JrpcConv ExecutionPayloadV3.useDefaultSerializationIn JrpcConv BlobsBundleV1.useDefaultSerializationIn JrpcConv BlobAndProofV1.useDefaultSerializationIn JrpcConv +BlobAndProofV2.useDefaultSerializationFor JrpcConv ExecutionPayloadBodyV1.useDefaultSerializationIn JrpcConv PayloadAttributesV1.useDefaultSerializationIn JrpcConv PayloadAttributesV2.useDefaultSerializationIn JrpcConv diff --git a/web3/engine_api.nim b/web3/engine_api.nim index 00b53976..755ea64f 100644 --- a/web3/engine_api.nim +++ b/web3/engine_api.nim @@ -39,6 +39,7 @@ createRpcSigsFromNim(RpcClient): proc engine_getPayloadBodiesByHashV1(hashes: seq[Hash32]): seq[Opt[ExecutionPayloadBodyV1]] proc engine_getPayloadBodiesByRangeV1(start: Quantity, count: Quantity): seq[Opt[ExecutionPayloadBodyV1]] proc engine_getBlobsV1(blob_versioned_hashes: seq[VersionedHash]): GetBlobsV1Response + proc engine_getBlobsV2(blob_versioned_hashes: seq[VersionedHash]): GetBlobsV2Response # https://github.com/ethereum/execution-apis/blob/9301c0697e4c7566f0929147112f6d91f65180f6/src/engine/common.md proc engine_exchangeCapabilities(methods: seq[string]): seq[string] @@ -115,6 +116,13 @@ template getBlobs*( Future[GetBlobsV1Response] = engine_getBlobsV1(rpcClient, blob_versioned_hashes) +template getBlobs*( + rpcClient: RpcClient, + T: type GetBlobsV2Response, + blob_versioned_hashes: seq[VersionedHash]): + Future[GetBlobsV2Response] = + engine_getBlobsV2(rpcClient, blob_versioned_hashes) + template newPayload*( rpcClient: RpcClient, payload: ExecutionPayloadV1): Future[PayloadStatusV1] = diff --git a/web3/engine_api_types.nim b/web3/engine_api_types.nim index b27d8ccd..bf0245d3 100644 --- a/web3/engine_api_types.nim +++ b/web3/engine_api_types.nim @@ -131,6 +131,10 @@ type blob*: Blob proof*: KzgProof + BlobAndProofV2* = object + blob*: Blob + proofs*: array[cellsPerExternalBlob, KzgProof] + # https://github.com/ethereum/execution-apis/blob/v1.0.0-beta.4/src/engine/shanghai.md#executionpayloadbodyv1 # For optional withdrawals field, see: # https://github.com/ethereum/execution-apis/blob/v1.0.0-beta.4/src/engine/shanghai.md#engine_getpayloadbodiesbyhashv1 @@ -223,6 +227,8 @@ type GetBlobsV1Response* = seq[BlobAndProofV1] + GetBlobsV2Response* = seq[BlobAndProofV2] + SomeGetPayloadResponse* = ExecutionPayloadV1 | GetPayloadV2Response | diff --git a/web3/primitives.nim b/web3/primitives.nim index 06a98bed..e66ab7e0 100644 --- a/web3/primitives.nim +++ b/web3/primitives.nim @@ -29,6 +29,9 @@ const # https://github.com/ethereum/execution-apis/blob/c4089414bbbe975bbc4bf1ccf0a3d31f76feb3e1/src/engine/cancun.md#blobsbundlev1 fieldElementsPerBlob = 4096 + # https://github.com/0x00101010/execution-apis/blob/eip-7594/src/engine/osaka.md#blobandproofv2 + cellsPerExternalBlob* = 128 + type # https://github.com/ethereum/execution-apis/blob/c4089414bbbe975bbc4bf1ccf0a3d31f76feb3e1/src/schemas/base-types.yaml From 97784b5533f31fd84fd64176d6141693ef23bfcd Mon Sep 17 00:00:00 2001 From: Agnish Ghosh Date: Sun, 6 Apr 2025 17:50:57 +0530 Subject: [PATCH 04/27] Revert "accomodate getBlobsV2" This reverts commit 3fffb22183fbf8e09e4f7e2cc92f3d4dc480bcc3. --- tests/test_json_marshalling.nim | 1 - web3/conversions.nim | 1 - web3/engine_api.nim | 8 -------- web3/engine_api_types.nim | 6 ------ web3/primitives.nim | 3 --- 5 files changed, 19 deletions(-) diff --git a/tests/test_json_marshalling.nim b/tests/test_json_marshalling.nim index 6d5ec1a7..97f11a80 100644 --- a/tests/test_json_marshalling.nim +++ b/tests/test_json_marshalling.nim @@ -204,7 +204,6 @@ suite "JSON-RPC Quantity": checkRandomObject(ExecutionPayloadV3) checkRandomObject(BlobsBundleV1) checkRandomObject(BlobAndProofV1) - checkRandomObject(BlobAndProofV2) checkRandomObject(ExecutionPayloadBodyV1) checkRandomObject(PayloadAttributesV1) checkRandomObject(PayloadAttributesV2) diff --git a/web3/conversions.nim b/web3/conversions.nim index fff14306..b71fe050 100644 --- a/web3/conversions.nim +++ b/web3/conversions.nim @@ -60,7 +60,6 @@ ExecutionPayloadV1OrV2.useDefaultSerializationIn JrpcConv ExecutionPayloadV3.useDefaultSerializationIn JrpcConv BlobsBundleV1.useDefaultSerializationIn JrpcConv BlobAndProofV1.useDefaultSerializationIn JrpcConv -BlobAndProofV2.useDefaultSerializationFor JrpcConv ExecutionPayloadBodyV1.useDefaultSerializationIn JrpcConv PayloadAttributesV1.useDefaultSerializationIn JrpcConv PayloadAttributesV2.useDefaultSerializationIn JrpcConv diff --git a/web3/engine_api.nim b/web3/engine_api.nim index 755ea64f..00b53976 100644 --- a/web3/engine_api.nim +++ b/web3/engine_api.nim @@ -39,7 +39,6 @@ createRpcSigsFromNim(RpcClient): proc engine_getPayloadBodiesByHashV1(hashes: seq[Hash32]): seq[Opt[ExecutionPayloadBodyV1]] proc engine_getPayloadBodiesByRangeV1(start: Quantity, count: Quantity): seq[Opt[ExecutionPayloadBodyV1]] proc engine_getBlobsV1(blob_versioned_hashes: seq[VersionedHash]): GetBlobsV1Response - proc engine_getBlobsV2(blob_versioned_hashes: seq[VersionedHash]): GetBlobsV2Response # https://github.com/ethereum/execution-apis/blob/9301c0697e4c7566f0929147112f6d91f65180f6/src/engine/common.md proc engine_exchangeCapabilities(methods: seq[string]): seq[string] @@ -116,13 +115,6 @@ template getBlobs*( Future[GetBlobsV1Response] = engine_getBlobsV1(rpcClient, blob_versioned_hashes) -template getBlobs*( - rpcClient: RpcClient, - T: type GetBlobsV2Response, - blob_versioned_hashes: seq[VersionedHash]): - Future[GetBlobsV2Response] = - engine_getBlobsV2(rpcClient, blob_versioned_hashes) - template newPayload*( rpcClient: RpcClient, payload: ExecutionPayloadV1): Future[PayloadStatusV1] = diff --git a/web3/engine_api_types.nim b/web3/engine_api_types.nim index bf0245d3..b27d8ccd 100644 --- a/web3/engine_api_types.nim +++ b/web3/engine_api_types.nim @@ -131,10 +131,6 @@ type blob*: Blob proof*: KzgProof - BlobAndProofV2* = object - blob*: Blob - proofs*: array[cellsPerExternalBlob, KzgProof] - # https://github.com/ethereum/execution-apis/blob/v1.0.0-beta.4/src/engine/shanghai.md#executionpayloadbodyv1 # For optional withdrawals field, see: # https://github.com/ethereum/execution-apis/blob/v1.0.0-beta.4/src/engine/shanghai.md#engine_getpayloadbodiesbyhashv1 @@ -227,8 +223,6 @@ type GetBlobsV1Response* = seq[BlobAndProofV1] - GetBlobsV2Response* = seq[BlobAndProofV2] - SomeGetPayloadResponse* = ExecutionPayloadV1 | GetPayloadV2Response | diff --git a/web3/primitives.nim b/web3/primitives.nim index e66ab7e0..06a98bed 100644 --- a/web3/primitives.nim +++ b/web3/primitives.nim @@ -29,9 +29,6 @@ const # https://github.com/ethereum/execution-apis/blob/c4089414bbbe975bbc4bf1ccf0a3d31f76feb3e1/src/engine/cancun.md#blobsbundlev1 fieldElementsPerBlob = 4096 - # https://github.com/0x00101010/execution-apis/blob/eip-7594/src/engine/osaka.md#blobandproofv2 - cellsPerExternalBlob* = 128 - type # https://github.com/ethereum/execution-apis/blob/c4089414bbbe975bbc4bf1ccf0a3d31f76feb3e1/src/schemas/base-types.yaml From 57a1daa1d5806ec5e6cae9a41efdc6aa22a0c514 Mon Sep 17 00:00:00 2001 From: Agnish Ghosh <80243668+agnxsh@users.noreply.github.com> Date: Mon, 7 Apr 2025 03:31:57 +0530 Subject: [PATCH 05/27] getBlobsV2 (#199) * conflict resolved * getBlobsV2 * fix * revert random object check on BlobAndProofV2 * have a rand function for BlobAndProofV2 * another fix * addressed reviews * update link --- tests/test_json_marshalling.nim | 7 +++++++ web3/conversions.nim | 1 + web3/engine_api.nim | 8 ++++++++ web3/engine_api_types.nim | 7 +++++++ web3/primitives.nim | 3 +++ 5 files changed, 26 insertions(+) diff --git a/tests/test_json_marshalling.nim b/tests/test_json_marshalling.nim index 97f11a80..177840a3 100644 --- a/tests/test_json_marshalling.nim +++ b/tests/test_json_marshalling.nim @@ -89,6 +89,12 @@ proc rand[T](_: type seq[T]): seq[T] = for i in 0..<3: result[i] = rand(T) +proc rand[T](_: type openArray[T]): array[cellsPerExternalBlob, T] = + var a: array[cellsPerExternalBlob, T] + for i in 0.. Date: Mon, 7 Apr 2025 03:14:33 +0000 Subject: [PATCH 06/27] use spec names for spec constants (#200) --- tests/test_json_marshalling.nim | 4 ++-- web3/engine_api_types.nim | 2 +- web3/primitives.nim | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_json_marshalling.nim b/tests/test_json_marshalling.nim index 177840a3..562de39c 100644 --- a/tests/test_json_marshalling.nim +++ b/tests/test_json_marshalling.nim @@ -89,8 +89,8 @@ proc rand[T](_: type seq[T]): seq[T] = for i in 0..<3: result[i] = rand(T) -proc rand[T](_: type openArray[T]): array[cellsPerExternalBlob, T] = - var a: array[cellsPerExternalBlob, T] +proc rand[T](_: type openArray[T]): array[CELLS_PER_EXT_BLOB, T] = + var a: array[CELLS_PER_EXT_BLOB, T] for i in 0.. Date: Tue, 15 Apr 2025 15:04:07 +0700 Subject: [PATCH 07/27] Fusaka: Add EIP-7873 fields to TransactionObject/TransactionArgs (#198) * Fusaka: Add EIP-7873 fields to TransactionObject/TransactionArgs * Fix: fields should be Optional * Fix transaction signing --- web3/eth_api_types.nim | 4 ++ web3/transaction_signing.nim | 87 ++++++++++++++++++++++-------------- 2 files changed, 58 insertions(+), 33 deletions(-) diff --git a/web3/eth_api_types.nim b/web3/eth_api_types.nim index c694888c..198edbb9 100644 --- a/web3/eth_api_types.nim +++ b/web3/eth_api_types.nim @@ -71,6 +71,9 @@ type # EIP-7702 authorizationList*: Opt[seq[Authorization]] + # EIP-7873 + initCodes*: Opt[seq[seq[byte]]] + ## A block header object BlockHeader* = ref object number*: Quantity @@ -166,6 +169,7 @@ type maxFeePerBlobGas*: Opt[UInt256] # EIP-4844 blobVersionedHashes*: Opt[seq[VersionedHash]] # EIP-4844 authorizationList*: Opt[seq[Authorization]] # EIP-7702 + initCodes*: Opt[seq[seq[byte]]] # EIP-7873 ReceiptObject* = ref object # A transaction receipt object, or null when no receipt was found: transactionHash*: Hash32 # hash of the transaction. diff --git a/web3/transaction_signing.nim b/web3/transaction_signing.nim index 0744e9e0..f575d017 100644 --- a/web3/transaction_signing.nim +++ b/web3/transaction_signing.nim @@ -13,13 +13,13 @@ import func encodeTransactionLegacy(s: TransactionArgs, pk: PrivateKey): seq[byte] = var tr = Transaction(txType: TxLegacy) - tr.gasLimit = s.gas.get.GasInt - tr.gasPrice = s.gasPrice.get.GasInt + tr.gasLimit = s.gas.get(0.Quantity).GasInt + tr.gasPrice = s.gasPrice.get(0.Quantity).GasInt tr.to = s.to if s.value.isSome: - tr.value = s.value.get - tr.nonce = uint64(s.nonce.get) + tr.value = s.value.value + tr.nonce = uint64(s.nonce.get(0.Quantity)) tr.payload = s.payload tr.signature = if s.chainId.isSome(): @@ -31,67 +31,84 @@ func encodeTransactionLegacy(s: TransactionArgs, pk: PrivateKey): seq[byte] = func encodeTransactionEip2930(s: TransactionArgs, pk: PrivateKey): seq[byte] = var tr = Transaction(txType: TxEip2930) - tr.gasLimit = s.gas.get.GasInt - tr.gasPrice = s.gasPrice.get.GasInt + tr.gasLimit = s.gas.get(0.Quantity).GasInt + tr.gasPrice = s.gasPrice.get(0.Quantity).GasInt tr.to = s.to if s.value.isSome: - tr.value = s.value.get - tr.nonce = uint64(s.nonce.get) + tr.value = s.value.value + tr.nonce = uint64(s.nonce.get(0.Quantity)) tr.payload = s.payload tr.chainId = s.chainId.get tr.signature = tr.sign(pk, true) - tr.accessList = s.accessList.get + tr.accessList = s.accessList.value rlp.encode(tr) func encodeTransactionEip1559(s: TransactionArgs, pk: PrivateKey): seq[byte] = var tr = Transaction(txType: TxEip1559) - tr.gasLimit = s.gas.get.GasInt - tr.maxPriorityFeePerGas = s.maxPriorityFeePerGas.get.GasInt - tr.maxFeePerGas = s.maxFeePerGas.get.GasInt + tr.gasLimit = s.gas.get(0.Quantity).GasInt + tr.maxPriorityFeePerGas = s.maxPriorityFeePerGas.get(0.Quantity).GasInt + tr.maxFeePerGas = s.maxFeePerGas.get(0.Quantity).GasInt tr.to = s.to if s.value.isSome: - tr.value = s.value.get - tr.nonce = uint64(s.nonce.get) + tr.value = s.value.value + tr.nonce = uint64(s.nonce.get(0.Quantity)) tr.payload = s.payload - tr.chainId = s.chainId.get + tr.chainId = s.chainId.get(0.u256) tr.signature = tr.sign(pk, true) if s.accessList.isSome: - tr.accessList = s.accessList.get + tr.accessList = s.accessList.value rlp.encode(tr) func encodeTransactionEip4844(s: TransactionArgs, pk: PrivateKey): seq[byte] = var tr = Transaction(txType: TxEip4844) - tr.gasLimit = s.gas.get.GasInt - tr.maxPriorityFeePerGas = s.maxPriorityFeePerGas.get.GasInt - tr.maxFeePerGas = s.maxFeePerGas.get.GasInt + tr.gasLimit = s.gas.get(0.Quantity).GasInt + tr.maxPriorityFeePerGas = s.maxPriorityFeePerGas.get(0.Quantity).GasInt + tr.maxFeePerGas = s.maxFeePerGas.get(0.Quantity).GasInt tr.to = s.to if s.value.isSome: - tr.value = s.value.get - tr.nonce = uint64(s.nonce.get) + tr.value = s.value.value + tr.nonce = uint64(s.nonce.get(0.Quantity)) tr.payload = s.payload - tr.chainId = s.chainId.get + tr.chainId = s.chainId.get(0.u256) tr.signature = tr.sign(pk, true) if s.accessList.isSome: - tr.accessList = s.accessList.get - tr.maxFeePerBlobGas = s.maxFeePerBlobGas.get - tr.versionedHashes = s.blobVersionedHashes.get + tr.accessList = s.accessList.value + tr.maxFeePerBlobGas = s.maxFeePerBlobGas.get(0.u256) + tr.versionedHashes = s.blobVersionedHashes.value rlp.encode(tr) func encodeTransactionEip7702(s: TransactionArgs, pk: PrivateKey): seq[byte] = var tr = Transaction(txType: TxEip7702) - tr.gasLimit = s.gas.get.GasInt - tr.maxPriorityFeePerGas = s.maxPriorityFeePerGas.get.GasInt - tr.maxFeePerGas = s.maxFeePerGas.get.GasInt + tr.gasLimit = s.gas.get(0.Quantity).GasInt + tr.maxPriorityFeePerGas = s.maxPriorityFeePerGas.get(0.Quantity).GasInt + tr.maxFeePerGas = s.maxFeePerGas.get(0.Quantity).GasInt tr.to = s.to if s.value.isSome: - tr.value = s.value.get - tr.nonce = uint64(s.nonce.get) + tr.value = s.value.value + tr.nonce = uint64(s.nonce.get(0.Quantity)) tr.payload = s.payload - tr.chainId = s.chainId.get + tr.chainId = s.chainId.get(0.u256) + tr.signature = tr.sign(pk, true) + if s.accessList.isSome: + tr.accessList = s.accessList.value + tr.authorizationList = s.authorizationList.value + rlp.encode(tr) + +func encodeTransactionEip7873(s: TransactionArgs, pk: PrivateKey): seq[byte] = + var tr = Transaction(txType: TxEip7873) + tr.gasLimit = s.gas.get(0.Quantity).GasInt + tr.maxPriorityFeePerGas = s.maxPriorityFeePerGas.get(0.Quantity).GasInt + tr.maxFeePerGas = s.maxFeePerGas.get(0.Quantity).GasInt + tr.to = s.to + if s.value.isSome: + tr.value = s.value.value + tr.nonce = uint64(s.nonce.get(0.Quantity)) + tr.payload = s.payload + tr.chainId = s.chainId.get(0.u256) tr.signature = tr.sign(pk, true) if s.accessList.isSome: - tr.accessList = s.accessList.get - tr.authorizationList = s.authorizationList.get + tr.accessList = s.accessList.value + tr.initCodes = s.initCodes.value rlp.encode(tr) func encodeTransaction*(s: TransactionArgs, pk: PrivateKey, txType: TxType): seq[byte] = @@ -106,8 +123,12 @@ func encodeTransaction*(s: TransactionArgs, pk: PrivateKey, txType: TxType): seq encodeTransactionEip4844(s, pk) of TxEip7702: encodeTransactionEip7702(s, pk) + of TxEip7873: + encodeTransactionEip7873(s, pk) func txType(s: TransactionArgs): TxType = + if s.initCodes.isSome: + return TxEip7873 if s.authorizationList.isSome: return TxEip7702 if s.blobVersionedHashes.isSome: From 1c15d1f9d606271eaa5eb0e7e340323b6439127a Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Tue, 22 Apr 2025 13:06:30 +0200 Subject: [PATCH 08/27] Bump nim-eth minimum to 0.7.0 (#203) `TxEip7873` requires nim-eth to be at least 0.7.0. ``` ./nim-web3/web3/transaction_signing.nim(98, 32) Error: undeclared identifier: 'TxEip7873' ``` --- web3.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web3.nimble b/web3.nimble index 80dc2bf8..27ad5c04 100644 --- a/web3.nimble +++ b/web3.nimble @@ -18,7 +18,7 @@ requires "nim >= 2.0.0" requires "chronicles" requires "chronos" requires "bearssl" -requires "eth" +requires "eth >= 0.7.0" requires "faststreams" requires "json_rpc" requires "json_serialization" From d35250bef5549a5535b996fe06e72748a2a5ca4a Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Tue, 22 Apr 2025 13:29:50 +0200 Subject: [PATCH 09/27] v0.6.0 (#204) - getBlobsV2 - use spec names for spec constants - Fusaka: Add EIP-7873 fields to TransactionObject/TransactionArgs - Bump nim-eth minimum to 0.7.0 --- web3.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web3.nimble b/web3.nimble index 27ad5c04..1ce74120 100644 --- a/web3.nimble +++ b/web3.nimble @@ -8,7 +8,7 @@ # those terms. mode = ScriptMode.Verbose -version = "0.5.0" +version = "0.6.0" author = "Status Research & Development GmbH" description = "These are the humble beginnings of library similar to web3.[js|py]" license = "MIT or Apache License 2.0" From 8460999c15f37eec61d09296ae81bac577daf3d9 Mon Sep 17 00:00:00 2001 From: Agnish Ghosh <80243668+agnxsh@users.noreply.github.com> Date: Wed, 23 Apr 2025 16:23:15 +0530 Subject: [PATCH 10/27] init payloadV5 (#201) * init payloadV5 * fix other v5 stuff --- tests/test_json_marshalling.nim | 1 + web3/conversions.nim | 2 ++ web3/engine_api.nim | 7 +++++++ web3/engine_api_types.nim | 12 ++++++++++++ web3/execution_types.nim | 27 ++++++++++++++++++++++++++- 5 files changed, 48 insertions(+), 1 deletion(-) diff --git a/tests/test_json_marshalling.nim b/tests/test_json_marshalling.nim index 562de39c..54b9866d 100644 --- a/tests/test_json_marshalling.nim +++ b/tests/test_json_marshalling.nim @@ -209,6 +209,7 @@ suite "JSON-RPC Quantity": checkRandomObject(ExecutionPayloadV1OrV2) checkRandomObject(ExecutionPayloadV3) checkRandomObject(BlobsBundleV1) + checkRandomObject(BlobsBundleV2) checkRandomObject(BlobAndProofV1) checkRandomObject(BlobAndProofV2) checkRandomObject(ExecutionPayloadBodyV1) diff --git a/web3/conversions.nim b/web3/conversions.nim index abd03e58..abc4a65a 100644 --- a/web3/conversions.nim +++ b/web3/conversions.nim @@ -59,6 +59,7 @@ ExecutionPayloadV2.useDefaultSerializationIn JrpcConv ExecutionPayloadV1OrV2.useDefaultSerializationIn JrpcConv ExecutionPayloadV3.useDefaultSerializationIn JrpcConv BlobsBundleV1.useDefaultSerializationIn JrpcConv +BlobsBundleV2.useDefaultSerializationIn JrpcConv BlobAndProofV1.useDefaultSerializationIn JrpcConv BlobAndProofV2.useDefaultSerializationIn JrpcConv ExecutionPayloadBodyV1.useDefaultSerializationIn JrpcConv @@ -73,6 +74,7 @@ GetPayloadV2Response.useDefaultSerializationIn JrpcConv GetPayloadV2ResponseExact.useDefaultSerializationIn JrpcConv GetPayloadV3Response.useDefaultSerializationIn JrpcConv GetPayloadV4Response.useDefaultSerializationIn JrpcConv +GetPayloadV5Response.useDefaultSerializationIn JrpcConv ClientVersionV1.useDefaultSerializationIn JrpcConv #------------------------------------------------------------------------------ diff --git a/web3/engine_api.nim b/web3/engine_api.nim index 755ea64f..25a4a64d 100644 --- a/web3/engine_api.nim +++ b/web3/engine_api.nim @@ -36,6 +36,7 @@ createRpcSigsFromNim(RpcClient): proc engine_getPayloadV2_exact(payloadId: Bytes8): GetPayloadV2ResponseExact proc engine_getPayloadV3(payloadId: Bytes8): GetPayloadV3Response proc engine_getPayloadV4(payloadId: Bytes8): GetPayloadV4Response + proc engine_getPayloadV5(payloadId: Bytes8): GetPayloadV5Response proc engine_getPayloadBodiesByHashV1(hashes: seq[Hash32]): seq[Opt[ExecutionPayloadBodyV1]] proc engine_getPayloadBodiesByRangeV1(start: Quantity, count: Quantity): seq[Opt[ExecutionPayloadBodyV1]] proc engine_getBlobsV1(blob_versioned_hashes: seq[VersionedHash]): GetBlobsV1Response @@ -109,6 +110,12 @@ template getPayload*( payloadId: Bytes8): Future[GetPayloadV4Response] = engine_getPayloadV4(rpcClient, payloadId) +template getPayload*( + rpcClient: RpcClient, + T: type GetPayloadV5Response, + payloadId: Bytes8): Future[GetPayloadV5Response] = + engine_getPayloadV5(rpcClient, payloadId) + template getBlobs*( rpcClient: RpcClient, T: type GetBlobsV1Response, diff --git a/web3/engine_api_types.nim b/web3/engine_api_types.nim index 69778222..a6ef13ae 100644 --- a/web3/engine_api_types.nim +++ b/web3/engine_api_types.nim @@ -127,6 +127,11 @@ type proofs*: seq[KzgProof] blobs*: seq[Blob] + BlobsBundleV2* = object + commitments*: seq[KzgCommitment] + proofs*: seq[KzgProof] + blobs*: seq[Blob] + # https://github.com/ethereum/execution-apis/blob/40088597b8b4f48c45184da002e27ffc3c37641f/src/engine/cancun.md#blobandproofv1 BlobAndProofV1* = object blob*: Blob @@ -226,6 +231,13 @@ type shouldOverrideBuilder*: bool executionRequests*: seq[seq[byte]] + GetPayloadV5Response* = object + executionPayload*: ExecutionPayloadV3 + blockValue*: UInt256 + blobsBundle*: BlobsBundleV2 + shouldOverrideBuilder*: bool + executionRequests*: seq[seq[byte]] + GetBlobsV1Response* = seq[BlobAndProofV1] GetBlobsV2Response* = seq[BlobAndProofV2] diff --git a/web3/execution_types.nim b/web3/execution_types.nim index 79d79fc9..3df02a0b 100644 --- a/web3/execution_types.nim +++ b/web3/execution_types.nim @@ -53,6 +53,7 @@ type executionPayload*: ExecutionPayload blockValue*: Opt[UInt256] blobsBundle*: Opt[BlobsBundleV1] + blobsBundleV2*: Opt[BlobsBundleV2] shouldOverrideBuilder*: Opt[bool] executionRequests*: Opt[seq[seq[byte]]] @@ -61,6 +62,7 @@ type V2 V3 V4 + V5 func version*(payload: ExecutionPayload): Version = if payload.blobGasUsed.isSome or payload.excessBlobGas.isSome: @@ -79,7 +81,10 @@ func version*(attr: PayloadAttributes): Version = Version.V1 func version*(res: GetPayloadResponse): Version = - if res.executionRequests.isSome: + if res.blobsBundleV2.isSome and + res.blobsBundleV2.get.proofs.len == (CELLS_PER_EXT_BLOB * res.blobsBundleV2.get.blobs.len): + Version.V5 + elif res.executionRequests.isSome: Version.V4 elif res.blobsBundle.isSome or res.shouldOverrideBuilder.isSome: Version.V3 @@ -388,6 +393,15 @@ func V4*(res: GetPayloadResponse): GetPayloadV4Response = executionRequests: res.executionRequests.get, ) +func V5*(res: GetPayloadResponse): GetPayloadV5Response = + GetPayloadV5Response( + executionPayload: res.executionPayload.V3, + blockValue: res.blockValue.get, + blobsBundle: res.blobsBundleV2.get(BlobsBundleV2()), + shouldOverrideBuilder: res.shouldOverrideBuilder.get(false), + executionRequests: res.executionRequests.get, + ) + func getPayloadResponse*(x: ExecutionPayloadV1): GetPayloadResponse = GetPayloadResponse(executionPayload: x.executionPayload) @@ -402,6 +416,7 @@ func getPayloadResponse*(x: GetPayloadV3Response): GetPayloadResponse = executionPayload: x.executionPayload.executionPayload, blockValue: Opt.some(x.blockValue), blobsBundle: Opt.some(x.blobsBundle), + blobsBundleV2: Opt.none(BlobsBundleV2), shouldOverrideBuilder: Opt.some(x.shouldOverrideBuilder) ) @@ -413,3 +428,13 @@ func getPayloadResponse*(x: GetPayloadV4Response): GetPayloadResponse = shouldOverrideBuilder: Opt.some(x.shouldOverrideBuilder), executionRequests: Opt.some(x.executionRequests), ) + +func getPayloadResponse*(x: GetPayloadV5Response): GetPayloadResponse = + GetPayloadResponse( + executionPayload: x.executionPayload.executionPayload, + blockValue: Opt.some(x.blockValue), + blobsBundle: Opt.none(BlobsBundleV1), + blobsBundleV2: Opt.some(x.blobsBundle), + shouldOverrideBuilder: Opt.some(x.shouldOverrideBuilder), + executionRequests: Opt.some(x.executionRequests), + ) From 7de20af8e4d3ae61fb67028ff0295f790268f706 Mon Sep 17 00:00:00 2001 From: tersec Date: Fri, 16 May 2025 14:26:52 +0000 Subject: [PATCH 11/27] rm EIP-7873 support (#206) --- web3/eth_api_types.nim | 4 ---- web3/transaction_signing.nim | 21 --------------------- 2 files changed, 25 deletions(-) diff --git a/web3/eth_api_types.nim b/web3/eth_api_types.nim index 198edbb9..c694888c 100644 --- a/web3/eth_api_types.nim +++ b/web3/eth_api_types.nim @@ -71,9 +71,6 @@ type # EIP-7702 authorizationList*: Opt[seq[Authorization]] - # EIP-7873 - initCodes*: Opt[seq[seq[byte]]] - ## A block header object BlockHeader* = ref object number*: Quantity @@ -169,7 +166,6 @@ type maxFeePerBlobGas*: Opt[UInt256] # EIP-4844 blobVersionedHashes*: Opt[seq[VersionedHash]] # EIP-4844 authorizationList*: Opt[seq[Authorization]] # EIP-7702 - initCodes*: Opt[seq[seq[byte]]] # EIP-7873 ReceiptObject* = ref object # A transaction receipt object, or null when no receipt was found: transactionHash*: Hash32 # hash of the transaction. diff --git a/web3/transaction_signing.nim b/web3/transaction_signing.nim index f575d017..6205a5f1 100644 --- a/web3/transaction_signing.nim +++ b/web3/transaction_signing.nim @@ -94,23 +94,6 @@ func encodeTransactionEip7702(s: TransactionArgs, pk: PrivateKey): seq[byte] = tr.authorizationList = s.authorizationList.value rlp.encode(tr) -func encodeTransactionEip7873(s: TransactionArgs, pk: PrivateKey): seq[byte] = - var tr = Transaction(txType: TxEip7873) - tr.gasLimit = s.gas.get(0.Quantity).GasInt - tr.maxPriorityFeePerGas = s.maxPriorityFeePerGas.get(0.Quantity).GasInt - tr.maxFeePerGas = s.maxFeePerGas.get(0.Quantity).GasInt - tr.to = s.to - if s.value.isSome: - tr.value = s.value.value - tr.nonce = uint64(s.nonce.get(0.Quantity)) - tr.payload = s.payload - tr.chainId = s.chainId.get(0.u256) - tr.signature = tr.sign(pk, true) - if s.accessList.isSome: - tr.accessList = s.accessList.value - tr.initCodes = s.initCodes.value - rlp.encode(tr) - func encodeTransaction*(s: TransactionArgs, pk: PrivateKey, txType: TxType): seq[byte] = case txType of TxLegacy: @@ -123,12 +106,8 @@ func encodeTransaction*(s: TransactionArgs, pk: PrivateKey, txType: TxType): seq encodeTransactionEip4844(s, pk) of TxEip7702: encodeTransactionEip7702(s, pk) - of TxEip7873: - encodeTransactionEip7873(s, pk) func txType(s: TransactionArgs): TxType = - if s.initCodes.isSome: - return TxEip7873 if s.authorizationList.isSome: return TxEip7702 if s.blobVersionedHashes.isSome: From d08873193533e0662b012210aca1c8739a11e3c0 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Wed, 21 May 2025 13:06:22 +0200 Subject: [PATCH 12/27] Bump nim-eth minimum to 0.8.0 (#207) `TxEip7873` has been removed from nim-eth. --- web3.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web3.nimble b/web3.nimble index 1ce74120..44d9fa1f 100644 --- a/web3.nimble +++ b/web3.nimble @@ -18,7 +18,7 @@ requires "nim >= 2.0.0" requires "chronicles" requires "chronos" requires "bearssl" -requires "eth >= 0.7.0" +requires "eth >= 0.8.0" requires "faststreams" requires "json_rpc" requires "json_serialization" From 85af8f228a7032d60572685513607fdb2d587a53 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Wed, 21 May 2025 13:28:15 +0200 Subject: [PATCH 13/27] v0.7.0 (#208) - init payloadV5 - rm EIP-7873 support - Bump nim-eth minimum to 0.8.0 --- web3.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web3.nimble b/web3.nimble index 44d9fa1f..bccdd3aa 100644 --- a/web3.nimble +++ b/web3.nimble @@ -8,7 +8,7 @@ # those terms. mode = ScriptMode.Verbose -version = "0.6.0" +version = "0.7.0" author = "Status Research & Development GmbH" description = "These are the humble beginnings of library similar to web3.[js|py]" license = "MIT or Apache License 2.0" From 6231ca330534c88805ded82810b4dd40f9d43323 Mon Sep 17 00:00:00 2001 From: andri lim Date: Fri, 27 Jun 2025 23:55:51 +0700 Subject: [PATCH 14/27] Replace PooledTransaction usage from test with BlobTx (#210) * Replace PooledTransaction usage from test Since PooledTransaction has been removed from nim-eth. We only need a minimum rlp reader to pass the test. And make clear the distinction between PooledTransaction hash and inner tx hash. * revert import --- tests/helpers/handlers.nim | 7 ++++--- tests/helpers/min_blobtx_rlp.nim | 29 +++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 3 deletions(-) create mode 100644 tests/helpers/min_blobtx_rlp.nim diff --git a/tests/helpers/handlers.nim b/tests/helpers/handlers.nim index 59ae8f60..7e19081c 100644 --- a/tests/helpers/handlers.nim +++ b/tests/helpers/handlers.nim @@ -13,7 +13,8 @@ import json_rpc/rpcserver, ../../web3/conversions, ../../web3/eth_api_types, - ../../web3/primitives as w3 + ../../web3/primitives as w3, + ./min_blobtx_rlp type Hash32 = w3.Hash32 @@ -29,8 +30,8 @@ proc installHandlers*(server: RpcServer) = return SyncingStatus(syncing: false) server.rpc("eth_sendRawTransaction") do(x: JsonString, data: seq[byte]) -> Hash32: - let tx = rlp.decode(data, PooledTransaction) - let h = rlpHash(tx) + let tx = rlp.decode(data, BlobTransaction) + let h = computeRlpHash(tx.tx) return Hash32(h.data) server.rpc("eth_getTransactionReceipt") do(x: JsonString, data: Hash32) -> ReceiptObject: diff --git a/tests/helpers/min_blobtx_rlp.nim b/tests/helpers/min_blobtx_rlp.nim new file mode 100644 index 00000000..5050b736 --- /dev/null +++ b/tests/helpers/min_blobtx_rlp.nim @@ -0,0 +1,29 @@ +{.push raises: [].} + +import + eth/common/transactions_rlp {.all.} + +type + # Pretend to be a PooledTransaction + # for testing purpose + BlobTransaction* = object + tx*: Transaction + +proc readTxTyped(rlp: var Rlp, tx: var BlobTransaction) {.raises: [RlpError].} = + let + txType = rlp.readTxType() + numFields = + if txType == TxEip4844: + rlp.listLen + else: + 1 + if numFields == 4 or numFields == 5: + rlp.tryEnterList() # spec: rlp([tx_payload, blobs, commitments, proofs]) + rlp.readTxPayload(tx.tx, txType) + # ignore BlobBundle + +proc read*(rlp: var Rlp, T: type BlobTransaction): T {.raises: [RlpError].} = + if rlp.isList: + rlp.readTxLegacy(result.tx) + else: + rlp.readTxTyped(result) From 440c62829d3ad41ebb91ab4b3bb9d374865c7c54 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Tue, 22 Jul 2025 08:22:12 +0200 Subject: [PATCH 15/27] chore: improve readme (#212) * Improve the readme * Add nodemon globally installation instructions * Fix typo * Add fetch submodules instruction --- README.md | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index dd76c1cf..5da16502 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ The humble beginnings of a Nim library similar to web3.[js|py] ## Installation -You can install the developement version of the library through nimble with the following command +You can install the development version of the library through nimble with the following command ```console nimble install https://github.com/status-im/nim-web3@#master @@ -17,7 +17,21 @@ nimble install https://github.com/status-im/nim-web3@#master ## Development -You should first run `./simulator.sh` which runs `hardhat node` +First, fetch the submodules: + +```bash +git submodule update --init --recursive +``` + +Install nodemon globally, hardhat locally and create a `hardhat.config.js` file: + +```bash +npm install -g nodemon +npm install hardhat +echo "module.exports = {};" > hardhat.config.js +``` + +Then you should run `./simulator.sh` which runs `hardhat node` This creates a local simulated Ethereum network on your local machine and the tests will use this for their E2E processing From 6d46159259421f0105bbf86cfd4302e83c1fece6 Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Tue, 22 Jul 2025 12:35:27 +0200 Subject: [PATCH 16/27] json: adapt to array streaming API changes (#211) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Juan M Gómez --- .github/workflows/ci.yml | 5 +++++ ci-test.sh | 2 +- web3/conversions.nim | 26 +++++++++++++++++--------- 3 files changed, 23 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 24b12697..efd4d141 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -162,6 +162,11 @@ jobs: QUICK_AND_DIRTY_COMPILER=1 QUICK_AND_DIRTY_NIMBLE=1 CC=gcc \ bash build_nim.sh nim csources dist/nimble NimBinaries echo '${{ github.workspace }}/nim/bin' >> $GITHUB_PATH + - name: Setup Nimble + uses: nim-lang/setup-nimble-action@v1 + with: + nimble-version: 'latest' + repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Use gcc 14 # Should be removed when ubuntu-latest is 26.04 diff --git a/ci-test.sh b/ci-test.sh index c561c0a8..5486b2f3 100755 --- a/ci-test.sh +++ b/ci-test.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash set -ex npm install hardhat diff --git a/web3/conversions.nim b/web3/conversions.nim index abc4a65a..855fcf8f 100644 --- a/web3/conversions.nim +++ b/web3/conversions.nim @@ -170,11 +170,17 @@ func valid(hex: string): bool = if x notin HexDigits: return false true +when not declared(json_serialization.streamElement): # json_serialization < 0.3.0 + template streamElement(w: var JsonWriter, s, body: untyped) = + template s: untyped = w.stream + body + proc writeHexValue(w: var JsonWriter, v: openArray[byte]) {.gcsafe, raises: [IOError].} = - w.stream.write "\"0x" - w.stream.writeHex v - w.stream.write "\"" + w.streamElement(s): + s.write "\"0x" + s.writeHex v + s.write "\"" #------------------------------------------------------------------------------ # Well, both rpc and chronicles share the same encoding of these types @@ -209,9 +215,10 @@ proc writeValue*[F: CommonJsonFlavors](w: var JsonWriter[F], v: RlpEncodedBytes) proc writeValue*[F: CommonJsonFlavors]( w: var JsonWriter[F], v: Quantity ) {.gcsafe, raises: [IOError].} = - w.stream.write "\"0x" - w.stream.toHex(distinctBase v) - w.stream.write "\"" + w.streamElement(s): + s.write "\"0x" + s.toHex(distinctBase v) + s.write "\"" proc readValue*[F: CommonJsonFlavors](r: var JsonReader[F], val: var DynamicBytes) {.gcsafe, raises: [IOError, JsonReaderError].} = @@ -299,9 +306,10 @@ proc readValue*[F: CommonJsonFlavors](r: var JsonReader[F], val: var UInt256) proc writeValue*(w: var JsonWriter[JrpcConv], v: uint64) {.gcsafe, raises: [IOError].} = - w.stream.write "\"0x" - w.stream.toHex(v) - w.stream.write "\"" + w.streamElement(s): + s.write "\"0x" + s.toHex(v) + s.write "\"" proc readValue*(r: var JsonReader[JrpcConv], val: var uint64) {.gcsafe, raises: [IOError, JsonReaderError].} = From 304afd1897bc62f7078d10810683f1f7af5f24f1 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Wed, 30 Jul 2025 17:10:24 +0200 Subject: [PATCH 17/27] On JSON-RPC, authorization serializes v as yParity (#209) * On JSON-RPC, authorization serializes v as yParity JSON uses yParity for authorization instead of v like in transaction, because v for transaction has additional information mixed in in legacy case (chain ID and magic value); this legacy case is not relevant for authorizations. Existing logic ignored yParity value from JSON and uses default initialized v = 0 value. Create our own web3 type, same as how we do it for transactions and receipts, and use correct yParity key. * Fix deps * Use flavor automatic serialization * eth_api seq[string] * adjust JSON marshalling tests (#202) Co-authored-by: Jacek Sieka * Custom serializer for TransactionArgs: oneof input or data field * cleanup --------- Co-authored-by: jangko Co-authored-by: tersec Co-authored-by: Jacek Sieka --- tests/test_execution_api.nim | 4 ++- tests/test_json_marshalling.nim | 35 +++++++++---------- web3.nimble | 7 ++-- web3/conversions.nim | 60 ++++++++++++++++++++++++++------- web3/eth_api.nim | 2 +- 5 files changed, 70 insertions(+), 38 deletions(-) diff --git a/tests/test_execution_api.nim b/tests/test_execution_api.nim index 9dfa8f18..d0710dbf 100644 --- a/tests/test_execution_api.nim +++ b/tests/test_execution_api.nim @@ -21,6 +21,8 @@ const {.push raises: [].} +JrpcConv.automaticSerialization(JsonValueRef, true) + func compareValue(lhs, rhs: JsonValueRef): bool func compareObject(lhs, rhs: JsonValueRef): bool = @@ -222,7 +224,7 @@ suite "Test eth api": let res = waitFor client.eth_getBlockReceipts("latest") check res.isSome waitFor client.close() - + waitFor srv.stop() waitFor srv.closeWait() diff --git a/tests/test_json_marshalling.nim b/tests/test_json_marshalling.nim index 54b9866d..8440255a 100644 --- a/tests/test_json_marshalling.nim +++ b/tests/test_json_marshalling.nim @@ -36,9 +36,9 @@ proc rand(_: type uint64): uint64 = uint64.fromBytesBE(res) proc rand[T: Quantity](_: type T): T = - var res: array[8, byte] + var res: array[sizeof(T), byte] discard randomBytes(res) - T(uint64.fromBytesBE(res)) + T(distinctBase(T).fromBytesBE(res)) proc rand[T: ChainId](_: type T): T = var res: array[8, byte] @@ -51,7 +51,7 @@ proc rand(_: type RlpEncodedBytes): RlpEncodedBytes = proc rand(_: type TypedTransaction): TypedTransaction = discard randomBytes(distinctBase result) -proc rand(_: type string): string = +func rand(_: type string): string = "random bytes" proc rand(_: type bool): bool = @@ -130,11 +130,8 @@ template checkRandomObject(T: type) = suite "JSON-RPC Quantity": test "Valid": - template checkType(typeName: typedesc): untyped = - for (validStr, validValue) in [ - ("0x0", typeName 0), - ("0x123", typeName 291), - ("0x1234", typeName 4660)]: + template checkType(typeName: typedesc, tests: auto): untyped = + for (validStr, validValue) in tests: let validJson = JrpcConv.encode(validStr) res = JrpcConv.decode(validJson, typeName) @@ -142,16 +139,15 @@ suite "JSON-RPC Quantity": resUInt256Ref = JrpcConv.decode(validJson, ref UInt256) check: - JrpcConv.decode(validJson, typeName) == validValue - JrpcConv.encode(validValue) == validJson - res == validValue - resUInt256 == validValue.distinctBase.u256 - resUInt256Ref[] == validValue.distinctBase.u256 + JrpcConv.decode(validJson, typeName) == typeName validValue + JrpcConv.encode(typeName validValue) == validJson + res == typeName(validValue) + resUInt256 == typeName(validValue).distinctBase.u256 + resUInt256Ref[] == typeName(validValue).distinctBase.u256 - checkType(Quantity) - checkType(Quantity) + checkType(Quantity, [("0x0", 0), ("0x123", 291), ("0x1234", 4660)]) - test "Invalid Quantity/Quantity/UInt256/ref UInt256": + test "Invalid Quantity/UInt256/ref UInt256": # TODO once https://github.com/status-im/nimbus-eth2/pull/3850 addressed, # re-add "0x0400" test case as invalid. for invalidStr in [ @@ -168,7 +164,6 @@ suite "JSON-RPC Quantity": except CatchableError: check: false - checkInvalids(Quantity) checkInvalids(Quantity) checkInvalids(UInt256) checkInvalids(ref UInt256) @@ -266,7 +261,7 @@ suite "JSON-RPC Quantity": test "AccessListResult": let z = AccessListResult() let w = JrpcConv.encode(z) - check w == """{"accessList":[],"gasUsed":"0x0"}""" + check w == """{"accessList":[],"error":null,"gasUsed":"0x0"}""" test "AccessListResult with error": let z = AccessListResult( @@ -276,8 +271,8 @@ suite "JSON-RPC Quantity": check w == """{"accessList":[],"error":"error","gasUsed":"0x0"}""" test "Authorization": - let z = Authorization() + let z = Authorization(yParity: 3, nonce: 11) let w = JrpcConv.encode(z) - check w == """{"chainId":"0x0","address":"0x0000000000000000000000000000000000000000","nonce":"0x0","v":"0x0","r":"0x0","s":"0x0"}""" + check w == """{"chainId":"0x0","address":"0x0000000000000000000000000000000000000000","nonce":"0xb","yParity":"0x3","r":"0x0","s":"0x0"}""" let x = JrpcConv.decode(w, Authorization) check x == z diff --git a/web3.nimble b/web3.nimble index bccdd3aa..1464afd2 100644 --- a/web3.nimble +++ b/web3.nimble @@ -18,10 +18,11 @@ requires "nim >= 2.0.0" requires "chronicles" requires "chronos" requires "bearssl" -requires "eth >= 0.8.0" +requires "eth >= 0.9.0" requires "faststreams" -requires "json_rpc" -requires "json_serialization" +requires "json_rpc >= 0.5.2" +requires "serialization >= 0.4.4" +requires "json_serialization >= 0.4.2" requires "nimcrypto" requires "stew" requires "stint" diff --git a/web3/conversions.nim b/web3/conversions.nim index 855fcf8f..0424e4b2 100644 --- a/web3/conversions.nim +++ b/web3/conversions.nim @@ -7,12 +7,14 @@ # This file may not be copied, modified, or distributed except according to # those terms. +{.push gcsafe, raises: [].} + import std/strutils, stew/byteutils, faststreams/textio, json_rpc/jsonmarshal, - json_serialization/stew/results, + json_serialization/pkg/results, json_serialization, ./primitives, ./engine_api_types, @@ -28,6 +30,18 @@ export export eth_types_json_serialization except Topic +#------------------------------------------------------------------------------ +# JrpcConv configuration +#------------------------------------------------------------------------------ + +JrpcConv.automaticSerialization(string, true) +JrpcConv.automaticSerialization(JsonString, true) +JrpcConv.automaticSerialization(ref, true) +JrpcConv.automaticSerialization(seq, true) +JrpcConv.automaticSerialization(bool, true) +JrpcConv.automaticSerialization(float64, true) +JrpcConv.automaticSerialization(array, true) + #------------------------------------------------------------------------------ # eth_api_types #------------------------------------------------------------------------------ @@ -40,7 +54,7 @@ LogObject.useDefaultSerializationIn JrpcConv StorageProof.useDefaultSerializationIn JrpcConv ProofResponse.useDefaultSerializationIn JrpcConv FilterOptions.useDefaultSerializationIn JrpcConv -TransactionArgs.useDefaultSerializationIn JrpcConv +TransactionArgs.useDefaultReaderIn JrpcConv FeeHistoryResult.useDefaultSerializationIn JrpcConv Authorization.useDefaultSerializationIn JrpcConv @@ -85,8 +99,6 @@ ExecutionPayload.useDefaultSerializationIn JrpcConv PayloadAttributes.useDefaultSerializationIn JrpcConv GetPayloadResponse.useDefaultSerializationIn JrpcConv -{.push gcsafe, raises: [].} - #------------------------------------------------------------------------------ # Private helpers #------------------------------------------------------------------------------ @@ -122,7 +134,7 @@ template toHexImpl(hex, pos: untyped) = dec pos hex[pos] = c - for _ in 0 ..< 16: + for _ in 0 ..< maxDigits: prepend(hexChars[int(n and 0xF)]) if n == 0: break n = n shr 4 @@ -139,7 +151,7 @@ func getEnumStringTable(enumType: typedesc): Table[string, enumType] res[$value] = value res -proc toHex(s: OutputStream, x: uint64) {.gcsafe, raises: [IOError].} = +proc toHex(s: OutputStream, x: uint8|uint64) {.gcsafe, raises: [IOError].} = toHexImpl(hex, pos) write s, hex.toOpenArray(pos, static(hex.len - 1)) @@ -256,13 +268,14 @@ proc readValue*[F: CommonJsonFlavors](r: var JsonReader[F], val: var RlpEncodedB # skip empty hex val = RlpEncodedBytes hexToSeqByte(hexStr) -proc readValue*[F: CommonJsonFlavors](r: var JsonReader[F], val: var Quantity) - {.gcsafe, raises: [IOError, JsonReaderError].} = +proc readValue*[F: CommonJsonFlavors]( + r: var JsonReader[F], val: var Quantity +) {.gcsafe, raises: [IOError, JsonReaderError].} = let hexStr = r.parseString() if hexStr.invalidQuantityPrefix: r.raiseUnexpectedValue("Quantity value has invalid leading 0") wrapValueError: - val = Quantity strutils.fromHex[uint64](hexStr) + val = typeof(val) strutils.fromHex[typeof(distinctBase(val))](hexStr) proc readValue*[F: CommonJsonFlavors](r: var JsonReader[F], val: var PayloadExecutionStatus) {.gcsafe, raises: [IOError, JsonReaderError].} = @@ -304,20 +317,20 @@ proc readValue*[F: CommonJsonFlavors](r: var JsonReader[F], val: var UInt256) # Exclusive to JrpcConv #------------------------------------------------------------------------------ -proc writeValue*(w: var JsonWriter[JrpcConv], v: uint64) +proc writeValue*(w: var JsonWriter[JrpcConv], v: uint64 | uint8) {.gcsafe, raises: [IOError].} = w.streamElement(s): s.write "\"0x" s.toHex(v) s.write "\"" -proc readValue*(r: var JsonReader[JrpcConv], val: var uint64) +proc readValue*(r: var JsonReader[JrpcConv], val: var (uint8 | uint64)) {.gcsafe, raises: [IOError, JsonReaderError].} = let hexStr = r.parseString() if hexStr.invalidQuantityPrefix: r.raiseUnexpectedValue("Uint64 value has invalid leading 0") wrapValueError: - val = strutils.fromHex[uint64](hexStr) + val = strutils.fromHex[typeof(val)](hexStr) proc writeValue*(w: var JsonWriter[JrpcConv], v: seq[byte]) {.gcsafe, raises: [IOError].} = @@ -422,8 +435,29 @@ proc writeValue*(w: var JsonWriter[JrpcConv], v: Opt[seq[ReceiptObject]]) else: w.writeValue JsonString("null") +proc writeValue*(w: var JsonWriter[JrpcConv], v: TransactionArgs) + {.gcsafe, raises: [IOError].} = + mixin writeValue + var + noInput = true + noData = true + + w.beginObject() + for k, val in fieldPairs(v): + when k == "input": + if v.input.isSome and noData: + w.writeMember(k, val) + noInput = false + elif k == "data": + if v.data.isSome and noInput: + w.writeMember(k, val) + noData = false + else: + w.writeMember(k, val) + w.endObject() + func `$`*(v: Quantity): string {.inline.} = - encodeQuantity(v.uint64) + encodeQuantity(distinctBase(v)) func `$`*(v: TypedTransaction): string {.inline.} = "0x" & distinctBase(v).toHex diff --git a/web3/eth_api.nim b/web3/eth_api.nim index 372b9f5f..2c91375a 100644 --- a/web3/eth_api.nim +++ b/web3/eth_api.nim @@ -72,7 +72,7 @@ createRpcSigsFromNim(RpcClient): proc eth_chainId(): UInt256 proc eth_getWork(): seq[UInt256] - proc eth_submitWork(nonce: int64, powHash: Hash32, mixDigest: Hash32): bool + proc eth_submitWork(nonce: uint64, powHash: Hash32, mixDigest: Hash32): bool proc eth_submitHashrate(hashRate: UInt256, id: UInt256): bool proc eth_subscribe(name: string, options: FilterOptions): string proc eth_subscribe(name: string): string From 81ee8ce479d86acb73be7c4f365328e238d9b4a3 Mon Sep 17 00:00:00 2001 From: Advaita Saha Date: Wed, 6 Aug 2025 07:57:36 +0530 Subject: [PATCH 18/27] eth_config endpoint addition (#217) * eth_config basic setup * Add custom serialization and add hash/forkid fields * remove importutils import * Don't update to pkg/results yet * import njs/pkg/results * update to Uint256 * update activation time to Quantity * allow nil return * adjust to new eip changes * use JsonNumber * add timestamp * shift to generalized primtive Number * incorporate suggestions --------- Co-authored-by: jangko --- tests/test_json_marshalling.nim | 7 +++++++ web3/conversions.nim | 37 +++++++++++++++++++++++++++++++++ web3/eth_api.nim | 1 + web3/eth_api_types.nim | 26 +++++++++++++++++++++++ web3/primitives.nim | 4 ++++ 5 files changed, 75 insertions(+) diff --git a/tests/test_json_marshalling.nim b/tests/test_json_marshalling.nim index 8440255a..aaaa4555 100644 --- a/tests/test_json_marshalling.nim +++ b/tests/test_json_marshalling.nim @@ -40,6 +40,11 @@ proc rand[T: Quantity](_: type T): T = discard randomBytes(res) T(distinctBase(T).fromBytesBE(res)) +proc rand[T: Number](_: type T): T = + var res: array[sizeof(T), byte] + discard randomBytes(res) + T(distinctBase(T).fromBytesBE(res)) + proc rand[T: ChainId](_: type T): T = var res: array[8, byte] discard randomBytes(res) @@ -222,6 +227,8 @@ suite "JSON-RPC Quantity": checkRandomObject(PayloadAttributes) checkRandomObject(GetPayloadResponse) + checkRandomObject(EthConfigObject) + test "check blockId": let a = RtBlockIdentifier(kind: bidNumber, number: 77.Quantity) let x = JrpcConv.encode(a) diff --git a/web3/conversions.nim b/web3/conversions.nim index 0424e4b2..00189603 100644 --- a/web3/conversions.nim +++ b/web3/conversions.nim @@ -62,6 +62,9 @@ BlockHeader.useDefaultSerializationIn JrpcConv BlockObject.useDefaultSerializationIn JrpcConv TransactionObject.useDefaultSerializationIn JrpcConv ReceiptObject.useDefaultSerializationIn JrpcConv +BlobScheduleObject.useDefaultSerializationIn JrpcConv +ConfigObject.useDefaultSerializationIn JrpcConv +EthConfigObject.useDefaultSerializationIn JrpcConv #------------------------------------------------------------------------------ # engine_api_types @@ -252,6 +255,16 @@ proc readValue*(r: var JsonReader[JrpcConv], val: var Hash32) wrapValueError: val = fromHex(Hash32, r.parseString()) +proc writeValue*(w: var JsonWriter[JrpcConv], v: Number) + {.gcsafe, raises: [IOError].} = + w.streamElement(s): + s.writeText distinctBase(v) + +proc readValue*(r: var JsonReader[JrpcConv], val: var Number) + {.gcsafe, raises: [IOError, JsonReaderError].} = + wrapValueError: + val = r.parseInt(uint64).Number + proc readValue*[F: CommonJsonFlavors](r: var JsonReader[F], val: var TypedTransaction) {.gcsafe, raises: [IOError, JsonReaderError].} = wrapValueError: @@ -313,6 +326,16 @@ proc readValue*[F: CommonJsonFlavors](r: var JsonReader[F], val: var UInt256) wrapValueError: val = hexStr.parse(StUint[256], 16) +proc readValue*[F: CommonJsonFlavors](r: var JsonReader[F], val: var seq[PrecompilePair]) + {.gcsafe, raises: [IOError, SerializationError].} = + for k,v in readObject(r, string, Address): + val.add PrecompilePair(name: k, address: v) + +proc readValue*[F: CommonJsonFlavors](r: var JsonReader[F], val: var seq[SystemContractPair]) + {.gcsafe, raises: [IOError, SerializationError].} = + for k,v in readObject(r, string, Address): + val.add SystemContractPair(name: k, address: v) + #------------------------------------------------------------------------------ # Exclusive to JrpcConv #------------------------------------------------------------------------------ @@ -435,6 +458,20 @@ proc writeValue*(w: var JsonWriter[JrpcConv], v: Opt[seq[ReceiptObject]]) else: w.writeValue JsonString("null") +proc writeValue*(w: var JsonWriter[JrpcConv], v: seq[PrecompilePair]) + {.gcsafe, raises: [IOError].} = + w.beginObject() + for x in v: + w.writeMember(x.name, x.address) + w.endObject() + +proc writeValue*(w: var JsonWriter[JrpcConv], v: seq[SystemContractPair]) + {.gcsafe, raises: [IOError].} = + w.beginObject() + for x in v: + w.writeMember(x.name, x.address) + w.endObject() + proc writeValue*(w: var JsonWriter[JrpcConv], v: TransactionArgs) {.gcsafe, raises: [IOError].} = mixin writeValue diff --git a/web3/eth_api.nim b/web3/eth_api.nim index 2c91375a..4a8064d0 100644 --- a/web3/eth_api.nim +++ b/web3/eth_api.nim @@ -70,6 +70,7 @@ createRpcSigsFromNim(RpcClient): proc eth_getFilterLogs(filterId: string): JsonNode proc eth_getLogs(filterOptions: FilterOptions): seq[LogObject] proc eth_chainId(): UInt256 + proc eth_config(): EthConfigObject proc eth_getWork(): seq[UInt256] proc eth_submitWork(nonce: uint64, powHash: Hash32, mixDigest: Hash32): bool diff --git a/web3/eth_api_types.nim b/web3/eth_api_types.nim index c694888c..9b821669 100644 --- a/web3/eth_api_types.nim +++ b/web3/eth_api_types.nim @@ -262,6 +262,32 @@ type blobGasUsedRatio*: seq[float64] reward*: Opt[seq[FeeHistoryReward]] + BlobScheduleObject* = object + baseFeeUpdateFraction*: Number + max*: Number + target*: Number + + PrecompilePair* = object + address*: Address + name*: string + + SystemContractPair* = object + name*: string + address*: Address + + ConfigObject* = object + activationTime*: Number + blobSchedule*: BlobScheduleObject + chainId*: UInt256 + forkId*: Bytes4 + precompiles*: seq[PrecompilePair] + systemContracts*: seq[SystemContractPair] + + EthConfigObject* = ref object + current*: ConfigObject + next*: Opt[ConfigObject] + last*: Opt[ConfigObject] + func blockId*(n: uint64): RtBlockIdentifier = RtBlockIdentifier(kind: bidNumber, number: Quantity n) diff --git a/web3/primitives.nim b/web3/primitives.nim index 638c7a51..538764e6 100644 --- a/web3/primitives.nim +++ b/web3/primitives.nim @@ -43,6 +43,10 @@ type # Quantity is use in lieu of an ordinary `uint64` to avoid the default # format that comes with json_serialization + Number* = distinct uint64 + # Number is use in lieu of an ordinary `uint64` to avoid the default + # format that comes with json_serialization + Blob* = FixedBytes[FIELD_ELEMENTS_PER_BLOB * 32] template `==`*[minLen, maxLen](a, b: DynamicBytes[minLen, maxLen]): bool = From 48fb2d4a215c20326b0cb945913b1d614a0564b9 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Thu, 28 Aug 2025 14:51:34 +0200 Subject: [PATCH 19/27] feat: abi encoder / decoder (#216) * Add nimbledeps to gitignore * Add abi encoder * Add encoding tests * Add AbiEncoder * Add encoding tests * Fix async signature * Add encoding tests to all tests * Do not bubble the exception * Add proper error handling * Use faststream to simplify internal logic * Partial implementation of the decoding * Fix tuple encoding and try to provide better docs / tests * Update abi util to to take tuple in consideration * Add deprecation notice * Refactor the dynamic array / seq decoding * Add decoding * Add ABI encoder / decoder to the readme * Fix indent * Export decoding * Restore previous encoding functions and mark them deprecated * Remove compiler hints and warnings * Use inputStream signature and use readAll * Add custom types * Cleanup * Add support for serialization * Add support for dontSerialize pragma * Add readValue compatibility * Add readme for serialization * Use enumInstanceSerializedFields to get the serialized fields instead of checking manually the pragma * Use Abi serialization as main interface * Update README * Update deprecation message and export abi_serialization * Update deprecation message and export abi_serialization * Add tests for string * Test encode empty string * Encode object as tuple * Remove ref object * Ensure that object are encoded as tuple * Decode object as tuple * Remove compilation hint * Remove unused import * Replace ABI encoding and decoding by the new encoding method * Improve serialization test * Add test more 3 slots * Remove unused variables * Use existing type T to get the arity * Use advance function to point on the right position instead of copying the whole buffer * Use local encoder instead of calling Abi.encode * Use SomeRange type in testing * Add overflow detection when decoding int and provide clearer comments * Re organize comments * Fix indentation * Add raises pragma * Remove int / uint support * Use toOpenArray method and replace StUint usage by uint. * Use used pragma * Reject range with int / uint value * Apply toOpenArray function * Remove unused function * Check hardhat version on ci * Update ci with hardhat 3 * Remove the intermediate data seq * Remove comment * Remove comment * Use explicit return --- .gitignore | 1 + README.md | 52 ++++ ci-test.sh | 6 +- tests/all_tests.nim | 6 +- tests/test_abi_serialization.nim | 67 +++++ tests/test_abi_utils.nim | 15 ++ tests/test_contract_dsl.nim | 1 + tests/test_decoding.nim | 248 +++++++++++++++++ tests/test_encoding.nim | 312 ++++++++++++++++++++++ tests/test_string_decoder.nim | 19 +- web3.nim | 9 +- web3/abi_serialization.nim | 39 +++ web3/abi_utils.nim | 48 ++++ web3/contract_dsl.nim | 43 +-- web3/decoding.nim | 439 +++++++++++++++++++++++++++++++ web3/encoding.nim | 361 ++++++++++++++++--------- 16 files changed, 1510 insertions(+), 156 deletions(-) create mode 100644 tests/test_abi_serialization.nim create mode 100644 tests/test_abi_utils.nim create mode 100644 tests/test_decoding.nim create mode 100644 tests/test_encoding.nim create mode 100644 web3/abi_serialization.nim create mode 100644 web3/abi_utils.nim create mode 100644 web3/decoding.nim diff --git a/.gitignore b/.gitignore index 8b816f54..c33254f3 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ build/ *.exe *.dll nimble.paths +nimbledeps node_modules nohup.out diff --git a/README.md b/README.md index 5da16502..914cf800 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,56 @@ Then you should run `./simulator.sh` which runs `hardhat node` This creates a local simulated Ethereum network on your local machine and the tests will use this for their E2E processing +### ABI encoder / decoder + +Implements encoding of parameters according to the Ethereum +[Contract ABI Specification][1] using nim-serialization interface. + +Usage +----- + +```nim +import + serialization, + stint, + web3/[encoding, decoding] + +# encode unsigned integers, booleans, enums +Abi.encode(42'u8) + +# encode uint256 +Abi.encode(42.u256) + +# encode byte arrays and sequences +Abi.encode([1'u8, 2'u8, 3'u8]) +Abi.encode(@[1'u8, 2'u8, 3'u8]) + +# encode tuples +Abi.encode( (42'u8, @[1'u8, 2'u8, 3'u8], true) ) + +# decode values of different types +Abi.decode(bytes, uint8) +Abi.decode(bytes, UInt256) +Abi.decode(bytes, array[3, uint8]) +Abi.decode(bytes, seq[uint8]) + +# decode tuples +Abi.decode(bytes, (uint32, bool, seq[byte])) + +# custom type +type Contract = object + a: uint64 + b {.dontSerialize.}: string + c: bool + +let encoded = Abi.encode(x) +let decoded = Abi.decode(encoded, Contract) + +# encoded.a == decoded.a +# encoded.b == "" +# encoded.c == decoded.c +``` + ## License Licensed and distributed under either of @@ -46,3 +96,5 @@ or - Apache License, Version 2.0, ([LICENSE-APACHEv2](LICENSE-APACHEv2) or http://www.apache.org/licenses/LICENSE-2.0) at your option. This file may not be copied, modified, or distributed except according to those terms. + +[1]: https://docs.soliditylang.org/en/latest/abi-spec.html \ No newline at end of file diff --git a/ci-test.sh b/ci-test.sh index 5486b2f3..bffa05b4 100755 --- a/ci-test.sh +++ b/ci-test.sh @@ -1,8 +1,10 @@ #!/bin/bash set -ex -npm install hardhat -touch hardhat.config.js +npm install hardhat@3 +npm pkg set type="module" +echo "export default {};" > hardhat.config.js +npx hardhat --version nohup npx hardhat node & nimble install -y --depsOnly diff --git a/tests/all_tests.nim b/tests/all_tests.nim index 62b8a8a1..f14cc209 100644 --- a/tests/all_tests.nim +++ b/tests/all_tests.nim @@ -19,5 +19,9 @@ import test_signed_tx, test_execution_types, test_string_decoder, + test_abi_utils, + test_encoding, + test_decoding, + test_abi_serialization, test_contract_dsl, - test_execution_api + test_execution_api \ No newline at end of file diff --git a/tests/test_abi_serialization.nim b/tests/test_abi_serialization.nim new file mode 100644 index 00000000..6a04e4d0 --- /dev/null +++ b/tests/test_abi_serialization.nim @@ -0,0 +1,67 @@ +import + std/unittest, + serialization, + std/random, + ../web3/encoding, + ../web3/decoding, + ../web3/eth_api_types, + ../web3/abi_serialization + +type Contract = object + a: uint64 + b {.dontSerialize.}: string + c: bool + d: string + +type StorageDeal = object + client: array[20, byte] + provider: array[20, byte] + cid: array[32, byte] + size: uint64 + duration: uint64 + pricePerByte: UInt256 + signature: array[65, byte] + metadata: string + +proc randomBytes[N: static int](): array[N, byte] = + var a: array[N, byte] + for b in a.mitems: + b = rand(byte) + return a + +suite "ABI serialization": + test "encode and decode custom type": + let x = Contract(a: 1, b: "SECRET", c: true, d: "hello") + + let encoded = Abi.encode(x) + check encoded == Abi.encode((x.a, x.c, x.d)) + + let decoded = Abi.decode(encoded, Contract) + check decoded.a == x.a + check decoded.b == "" + check decoded.c == x.c + check decoded.d == x.d + + test "encode and decode complex type": + let deal = StorageDeal( + client: randomBytes[20](), + provider: randomBytes[20](), + cid: randomBytes[32](), + size: 1024'u64, + duration: 365'u64, + pricePerByte: 1000.u256, + signature: randomBytes[65](), + metadata: "Sample metadata for storage deal" + ) + + let encoded = Abi.encode(deal) + let decoded = Abi.decode(encoded, StorageDeal) + + check decoded.client == deal.client + check decoded.provider == deal.provider + check decoded.cid == deal.cid + check decoded.size == deal.size + check decoded.duration == deal.duration + check decoded.pricePerByte == deal.pricePerByte + check decoded.signature == deal.signature + check decoded.metadata == deal.metadata \ No newline at end of file diff --git a/tests/test_abi_utils.nim b/tests/test_abi_utils.nim new file mode 100644 index 00000000..ef236035 --- /dev/null +++ b/tests/test_abi_utils.nim @@ -0,0 +1,15 @@ + +import + std/unittest, + ../web3/abi_utils + +suite "ABI utils": + test "can determine whether types are dynamic or static": + check static isStatic(uint8) + check static isDynamic(seq[byte]) + check static isStatic(array[2, array[2, byte]]) + check static isDynamic(array[2, seq[byte]]) + check static isStatic((uint8, bool)) + check static isDynamic((uint8, seq[byte])) + check static isStatic((uint8, (bool, uint8))) + check static isDynamic((uint8, (bool, seq[byte]))) \ No newline at end of file diff --git a/tests/test_contract_dsl.nim b/tests/test_contract_dsl.nim index 72c179ab..c9ad1679 100644 --- a/tests/test_contract_dsl.nim +++ b/tests/test_contract_dsl.nim @@ -11,6 +11,7 @@ import pkg/unittest2, stew/byteutils, stint, + ../web3/encoding, ../web3/contract_dsl type diff --git a/tests/test_decoding.nim b/tests/test_decoding.nim new file mode 100644 index 00000000..491345b0 --- /dev/null +++ b/tests/test_decoding.nim @@ -0,0 +1,248 @@ +import + std/[unittest, random], + stew/[endians2], + stint, + serialization, + ../web3/eth_api_types, + ../web3/decoding, + ../web3/encoding, + ../web3/abi_serialization, + ./helpers/primitives_utils + +randomize() + +type SomeDistinctType = distinct uint16 + +func `==`*(a, b: SomeDistinctType): bool = + uint16(a) == uint16(b) + +suite "ABI decoding": + proc checkDecode[T](value: T) = + let encoded = Abi.encode(value) + check Abi.decode(encoded, T) == value + + proc randomBytes[N: static int](): array[N, byte] = + var a: array[N, byte] + for b in a.mitems: + b = rand(byte) + return a + + proc checkDecode(T: type) = + checkDecode(T.default) + checkDecode(T.low) + checkDecode(T.high) + + test "decodes uint8, uint16, 32, 64": + checkDecode(uint8) + checkDecode(uint16) + checkDecode(uint32) + checkDecode(uint64) + + test "decodes int8, int16, int32, int64": + checkDecode(int8) + checkDecode(int16) + checkDecode(int32) + checkDecode(int64) + + test "fails when trying to decode overfow data": + try: + let encoded = Abi.encode(int16.high) + discard Abi.decode(encoded, int8) + fail() + except SerializationError: + discard + + test "fails to decode when reading past end": + var encoded = Abi.encode(uint8.fromBytes(randomBytes[8](), bigEndian)) + encoded.delete(encoded.len-1) + + try: + discard Abi.decode(encoded, uint8) + fail() + except SerializationError as decoded: + check decoded.msg == "reading past end of bytes" + + test "fails to decode when trailing bytes remain": + var encoded = Abi.encode(uint8.fromBytes(randomBytes[8](), bigEndian)) + encoded.add(uint8.fromBytes(randomBytes[8](), bigEndian)) + + try: + discard Abi.decode(encoded, uint8) + fail() + except SerializationError as decoded: + check decoded.msg == "unread trailing bytes found" + + test "fails to decode when padding does not consist of zeroes with unsigned value": + var encoded = Abi.encode(uint8.fromBytes(randomBytes[8](), bigEndian)) + encoded[3] = 42'u8 + + try: + discard Abi.decode(encoded, uint8) + fail() + except SerializationError as decoded: + check decoded.msg == "invalid padding found" + + test "fails to decode when padding does not consist of zeroes": + var encoded = Abi.encode(8.int8) + encoded[3] = 42'u8 + + try: + discard Abi.decode(encoded, int8) + fail() + except SerializationError as decoded: + check decoded.msg == "invalid padding found" + + test "decodes booleans": + checkDecode(false) + checkDecode(true) + + test "fails to decode boolean when value is not 0 or 1": + let encoded = Abi.encode(2'u8) + + try: + discard Abi.decode(encoded, bool) + fail() + except SerializationError as decoded: + check decoded.msg == "invalid boolean value" + + test "decodes ranges": + type SomeRange = range[0x0000'u16..0xAAAA'u16] + checkDecode(SomeRange) + + test "fails to decode when value not in range": + type SomeRange = range[0x0000'u16..0xAAAA'u16] + let encoded = Abi.encode(0xFFFF'u16) + + try: + discard Abi.decode(encoded, SomeRange) + fail() + except SerializationError as decoded: + check decoded.msg == "value not in range" + + test "decodes enums": + type SomeEnum = enum + one = 1 + two = 2 + checkDecode(one) + checkDecode(two) + + test "fails to decode enum when encountering invalid value": + type SomeEnum = enum + one = 1 + two = 2 + let encoded = Abi.encode(3'u8) + + try: + discard Abi.decode(encoded, SomeEnum) + fail() + except SerializationError as decoded: + check decoded.msg == "invalid enum value" + + test "decodes stints": + checkDecode(UInt128) + checkDecode(UInt256) + checkDecode(Int128) + checkDecode(Int256) + + test "decodes addresses": + checkDecode(address(3)) + + test "decodes byte arrays": + checkDecode([1'u8, 2'u8, 3'u8]) + checkDecode(randomBytes[32]()) + checkDecode(randomBytes[33]()) + checkDecode(randomBytes[65]()) + + test "fails to decode array when padding does not consist of zeroes": + var arr = randomBytes[33]() + var encoded = Abi.encode(arr) + encoded[62] = 42'u8 + + try: + discard Abi.decode(encoded, type(arr)) + fail() + except SerializationError as decoded: + check decoded.msg == "invalid padding found" + + test "decodes byte sequences": + checkDecode(@[1'u8, 2'u8, 3'u8]) + checkDecode(@(randomBytes[32]())) + checkDecode(@(randomBytes[33]())) + + test "fails to decode seq when padding does not consist of zeroes": + var value = @(randomBytes[64]()) + value[62] = 42'u8 + + try: + discard Abi.decode(value, seq[byte]) + fail() + except SerializationError as decoded: + check decoded.msg == "invalid padding found" + + test "decodes sequences": + let seq1 = @(randomBytes[33]()) + let seq2 = @(randomBytes[32]()) + let value = @[seq1, seq2] + checkDecode(value) + + test "decodes arrays with static elements": + checkDecode([randomBytes[32](), randomBytes[32]()]) + + test "decodes arrays with dynamic elements": + let seq1 = @(randomBytes[32]()) + let seq2 = @(randomBytes[32]()) + checkDecode([seq1, seq2]) + + test "decodes arrays with string": + checkDecode(["hello", "world"]) + + test "decodes strings": + checkDecode("hello!☺") + + test "decodes distinct types as their base type": + checkDecode(SomeDistinctType(0xAABB'u16)) + + test "decodes tuples": + let a = true + let b = @[1'u8, 2'u8, 3'u8] + let c = 0xAABBCCDD'u32 + let d = @[4'u8, 5'u8, 6'u8] + checkDecode( (a, b, c, d) ) + + test "decodes nested tuples": + let a = true + let b = @[1'u8, 2'u8, 3'u8] + let c = 0xAABBCCDD'u32 + let d = @[4'u8, 5'u8, 6'u8] + checkDecode( (a, b, (c, d)) ) + + test "reads elements after dynamic tuple": + let a = @[1'u8, 2'u8, 3'u8] + let b = 0xAABBCCDD'u32 + checkDecode( ((a,), b) ) + + test "reads elements after static tuple": + let a = 0x123'u16 + let b = 0xAABBCCDD'u32 + checkDecode( ((a,), b) ) + + test "reads static tuple inside dynamic tuple": + let a = @[1'u8, 2'u8, 3'u8] + let b = 0xAABBCCDD'u32 + checkDecode( (a, (b,)) ) + + test "reads empty tuples": + checkDecode( ((),) ) + + test "reads empty tuple": + checkDecode( (), ) + + test "encodes strings": + let encoded = Abi.encode("hello") + check Abi.decode(encoded, string) == "hello" + + test "encodes empty strings": + let encoded = Abi.encode("") + let decoded = Abi.decode(encoded, string) + check decoded == "" + check len(decoded) == 0 \ No newline at end of file diff --git a/tests/test_encoding.nim b/tests/test_encoding.nim new file mode 100644 index 00000000..c02864c3 --- /dev/null +++ b/tests/test_encoding.nim @@ -0,0 +1,312 @@ +import + std/unittest, + std/sequtils, + std/random, + stint, + stew/[byteutils], + serialization, + ../web3/encoding, + ../web3/eth_api_types, + ../web3/abi_serialization, + ./helpers/primitives_utils + +suite "ABI encoding": + proc zeroes(amount: int): seq[byte] = + newSeq[byte](amount) + + proc randomBytes[N: static int](): array[N, byte] = + var a: array[N, byte] + for b in a.mitems: + b = rand(byte) + return a + + test "encodes uint8": + check Abi.encode(42'u8) == 31.zeroes & 42'u8 # [0, 0, ..., 0, 2a] (2a = 42) + + test "encodes booleans": + check Abi.encode(false) == 31.zeroes & 0'u8 # [0, 0, ..., 0, 0] + check Abi.encode(true) == 31.zeroes & 1'u8 # [0, 0, ..., 0, 1] + + test "encodes uint16, 32, 64": + check Abi.encode(0xABCD'u16) == + 30.zeroes & 0xAB'u8 & 0xCD'u8 + check Abi.encode(0x11223344'u32) == + 28.zeroes & 0x11'u8 & 0x22'u8 & 0x33'u8 & 0x44'u8 + check Abi.encode(0x1122334455667788'u64) == + 24.zeroes & + 0x11'u8 & 0x22'u8 & 0x33'u8 & 0x44'u8 & + 0x55'u8 & 0x66'u8 & 0x77'u8 & 0x88'u8 + + test "encodes int8, 16, 32, 64": + check Abi.encode(1'i8) == 31.zeroes & 0x01'u8 # [0, 0, ..., 0, 1] + check Abi.encode(1'i16) == 31.zeroes & 0x01'u8 # [0, 0, ..., 0, 1] + check Abi.encode(1'i32) == 31.zeroes & 0x01'u8 # [0, 0, ..., 0, 1] + check Abi.encode(1'i64) == 31.zeroes & 0x01'u8 # [0, 0, ..., 0, 1] + + check Abi.encode(-1'i8) == 0xFF'u8.repeat(32) # [255, 255, ..., 255] (signed value) + check Abi.encode(-1'i16) == 0xFF'u8.repeat(32) # [255, 255, ..., 255] (signed value) + check Abi.encode(-1'i32) == 0xFF'u8.repeat(32) # [255, 255, ..., 255] (signed value) + check Abi.encode(-1'i64) == 0xFF'u8.repeat(32) # [255, 255, ..., 255] (signed value) + + test "encodes ranges": + type SomeRange = range[0x0000'u16..0xAAAA'u16] + check Abi.encode(SomeRange(0x1122)) == 30.zeroes & 0x11'u8 & 0x22'u8 + + test "encodes enums": + type SomeEnum = enum + one = 1 + two = 2 + check Abi.encode(one) == 31.zeroes & 1'u8 # [0, 0, ..., 0, 1] + check Abi.encode(two) == 31.zeroes & 2'u8 # [0, 0, ..., 0, 2] + + test "encodes stints": + let uint256 = UInt256.fromBytes(randomBytes[32](), bigEndian) + let uint128 = UInt128.fromBytes(randomBytes[32](), bigEndian) + check Abi.encode(uint256) == @(uint256.toBytesBE) + check Abi.encode(uint128) == 16.zeroes & @(uint128.toBytesBE) + + check Abi.encode(1.i256) == 31.zeroes & 0x01'u8 # [0, 0, ..., 0, 1] + check Abi.encode(1.i128) == 31.zeroes & 0x01'u8 # [0, 0, ..., 0, 1] + + check Abi.encode(-1.i256) == 0xFF'u8.repeat(32) # [255, 255, ..., 255] (signed value) + check Abi.encode(-1.i128) == 0xFF'u8.repeat(32) # [255, 255, ..., 255] (signed value) + + test "encodes addresses": + let address = address(3) + check Abi.encode(address) == 12.zeroes & @(array[20, byte](address)) + + test "encodes hashes": + let hash = txhash(3) + check Abi.encode(hash) == @(array[32, byte](hash)) + + test "encodes FixedBytes": + let bytes3 = FixedBytes[3]([1'u8, 2'u8, 3'u8]) + check Abi.encode(bytes3) == @[1'u8, 2'u8, 3'u8] & 29.zeroes # Fixed array are right-padded with zeroes + + let bytes32 = FixedBytes[32](randomBytes[32]()) + check Abi.encode(bytes32) == @(bytes32.data) + + let bytes33 = FixedBytes[33](randomBytes[33]()) + check Abi.encode(bytes33) == @(bytes33.data) & 31.zeroes # Right-padded with another 32 zeroes + + test "encodes byte arrays": + let bytes3 = [1'u8, 2'u8, 3'u8] + check Abi.encode(bytes3) == @bytes3 & 29.zeroes # Fixed array are right-padded with zeroes. + + let bytes32 = randomBytes[32]() + check Abi.encode(bytes32) == @bytes32 + + let bytes33 =randomBytes[33]() + check Abi.encode(bytes33) == @bytes33 & 31.zeroes # Right-padded with another 32 zeroes + + test "encodes byte sequences": + let bytes3 = @[1'u8, 2'u8, 3'u8] + let bytes3len = Abi.encode(bytes3.len.uint64) + check Abi.encode(bytes3) == bytes3len & bytes3 & 29.zeroes + check Abi.encode(bytes3) == + 31.zeroes & 3'u8 & # [0, 0, ..., 0, 3] (length) + bytes3 & 29.zeroes # [1, 2, 3, 0, ..., 0] (data) + + let bytes32 = @(randomBytes[32]()) + let bytes32len = Abi.encode(bytes32.len.uint64) + check Abi.encode(bytes32) == bytes32len & bytes32 + + let bytes33 = @(randomBytes[33]()) + let bytes33len = Abi.encode(bytes33.len.uint64) + check Abi.encode(bytes33) == bytes33len & bytes33 & 31.zeroes + + test "encodes empty seq of seq": + let v: seq[seq[int]] = @[] + check Abi.encode(v) == Abi.encode(0'u64) + check Abi.encode(v) == 32.zeroes # Encode the size only (zero) + + test "encodes tuples": + let a = true + let b = @[1'u8, 2'u8, 3'u8] + let c = 0xAABBCCDD'u32 + let d = @[4'u8, 5'u8, 6'u8] + check Abi.encode( (a, b, c, d) ) == + Abi.encode(a) & + Abi.encode(4 * 32'u8) & + Abi.encode(c) & + Abi.encode(6 * 32'u8) & + Abi.encode(b) & + Abi.encode(d) + check Abi.encode( (a, b, c, d) ) == + 31.zeroes & 1'u8 & # boolean value + 31.zeroes & 128'u8 & # offset to b (4 (bool, offset, int, offset) * 32 bytes) + 28.zeroes & 0xAA'u8 & 0xBB & 0xCC & 0xDD & + 31.zeroes & 192'u8 & # offset to d ((4 + b length + b data) * 32 bytes) + 31.zeroes & 3'u8 & b & 29.zeroes & # b (length + data) + 31.zeroes & 3'u8 & d & 29.zeroes # d (length + data) + + test "encodes nested tuples": + let a = true + let b = @[1'u8, 2'u8, 3'u8] + let c = 0xAABBCCDD'u32 + let d = @[4'u8, 5'u8, 6'u8] + check Abi.encode( (a, b, (c, d)) ) == + Abi.encode(a) & + Abi.encode(3 * 32'u8) & # offset of b in outer tuple + Abi.encode(5 * 32'u8) & # offset of inner tuple in outer tuple + Abi.encode(b) & + Abi.encode(c) & + Abi.encode(2 * 32'u8) & # offset of d in inner tuple + Abi.encode(d) + check Abi.encode( (a, b, (c, d)) ) == + 31.zeroes & 1'u8 & # boolean value + 31.zeroes & 96'u8 & # offset to b ((bool, offset, tuple) * 32 bytes) + 31.zeroes & 160'u8 & # offset to tuple ((3 + b length + b data) * 32 bytes) + 31.zeroes & 3'u8 & b & 29.zeroes & # b (length + data) + 28.zeroes & 0xAA'u8 & 0xBB & 0xCC & 0xDD & + 31.zeroes & 64'u8 & # offset to d ((static + offset) * 32 bytes) + 31.zeroes & 3'u8 & d & 29.zeroes # d (length + data) + + test "encodes tuple with only dynamic fields": + let t = (@[1'u8, 2'u8], @[3'u8, 4'u8]) + check Abi.encode(t) == + Abi.encode(2 * 32'u64) & + Abi.encode(4 * 32'u64) & + Abi.encode(@[1'u8, 2'u8]) & + Abi.encode(@[3'u8, 4'u8]) + check Abi.encode(t) == + 31.zeroes & 64'u8 & # offset to first + 31.zeroes & 128'u8 & # offset to second (first offset + length encoding + data length) + 31.zeroes & 2'u8 & @[1'u8, 2'u8] & 30.zeroes & # first element (length + data) + 31.zeroes & 2'u8 & @[3'u8, 4'u8] & 30.zeroes # second element (length + data) + + test "encodes tuple with empty dynamic fields": + var empty: seq[byte] = @[] + let t = (empty, empty) + check Abi.encode(t) == + Abi.encode(2 * 32'u64) & + Abi.encode(2 * 32'u64 + Abi.encode(empty).len.uint64) & + Abi.encode(empty) & + Abi.encode(empty) + check Abi.encode(t) == + 31.zeroes & 64'u8 & # offset to first + 31.zeroes & 96'u8 & # offset to second (first offset + length encoding + data length) + 32.zeroes & # empty sequence + 32.zeroes # empty sequence + + test "encodes tuple with static and empty dynamic": + var empty: seq[byte] = @[] + let t = (42'u8, empty) + check Abi.encode(t) == + Abi.encode(42'u8) & + Abi.encode(2 * 32'u64) & + Abi.encode(empty) + check Abi.encode(t) == + 31.zeroes & 42'u8 & # int left-padded with zeroes + 31.zeroes & 64'u8 & # offset to empty (static + offset) + 32.zeroes # empty sequence + + test "encodes arrays": + let a, b = randomBytes[32]() + check Abi.encode([a, b]) == + Abi.encode((a, b)) # Encode as tuple because fixed arrays are static. + + test "encodes openArray": + let a = [1'u8, 2'u8, 3'u8, 4'u8, 5'u8] + check encode(a[1..3]) == + Abi.encode(3'u64) & @[2'u8, 3'u8, 4'u8] & 29.zeroes # [2, 3, 4, ..., 0, 0] + + test "encodes sequences": + let a, b = @[randomBytes[32]()] + # + check Abi.encode(@[a, b]) == + Abi.encode(2'u64) & # sequence length + Abi.encode( (a, b) ) # encode as tuple because sequences are dynamic. + + test "encodes sequence as dynamic element": + let s = @[42.u256, 43.u256] + check Abi.encode( (s,) ) == + Abi.encode(32'u8) & # offset in tuple + Abi.encode(s) + + test "encodes nested sequence": + let nestedSeq = @[ @[1'u8, 2'u8], @[3'u8, 4'u8, 5'u8] ] + check Abi.encode(nestedSeq) == + Abi.encode(2'u64) & + Abi.encode(2 * 32'u64) & + Abi.encode(4 * 32'u64) & + Abi.encode(@[1'u8, 2'u8]) & + Abi.encode(@[3'u8, 4'u8, 5'u8]) + check Abi.encode(nestedSeq) == + 31.zeroes & 2'u8 & # sequence length + 31.zeroes & 64'u8 & # offset to first item (2 offsets) + 31.zeroes & 128'u8 & # offset to second item (first offset + length + data length) + 31.zeroes & 2'u8 & @[1'u8, 2'u8] & 30.zeroes & # first item (length + data) + 31.zeroes & 3'u8 & @[3'u8, 4'u8, 5'u8] & 29.zeroes # second item (length + data) + + test "encodes seq of empty seqs": + let empty: seq[int] = @[] + let v: seq[seq[int]] = @[ empty, empty ] + check Abi.encode(v) == + Abi.encode(2'u64) & + Abi.encode(2 * 32'u64) & + Abi.encode(2 * 32'u64 + Abi.encode(empty).len.uint64) & + Abi.encode(empty) & + Abi.encode(empty) + check Abi.encode(v) == + 31.zeroes & 2'u8 & # sequence length + 31.zeroes & 64'u8 & # offset to first item (2 offsets) + 31.zeroes & 96'u8 & # offset to second item (first offset + zero length) + 32.zeroes & # empty sequence + 32.zeroes # empty sequence + + test "encodes DynamicBytes": + let bytes3 = DynamicBytes(@[1'u8, 2'u8, 3'u8]) + check Abi.encode(bytes3) == + Abi.encode(3'u64) & # data length right-padded with zeroes + bytes3.data & 29.zeroes + + let bytes32 = DynamicBytes(@(randomBytes[32]())) + check Abi.encode(bytes32) == + Abi.encode(32'u64) & # data length + bytes32.data + + let bytes33 = DynamicBytes(@(randomBytes[33]())) + check Abi.encode(bytes33) == + Abi.encode(33'u64) & # data length right-padded with zeroes + bytes33.data & 31.zeroes + + test "encodes array of static elements as static element": + let a = [[42'u8], [43'u8]] + check Abi.encode( (a,) ) == + Abi.encode(a) # The tuple encoding does not add offset for static elements (fixed length array). + + test "encodes array of dynamic elements as dynamic element": + let a = [@[42'u8], @[43'u8]] + check Abi.encode( (a,) ) == + Abi.encode(32'u8) & # offset in tuple + Abi.encode(a) + + test "encodes strings as UTF-8 byte sequence": + check Abi.encode("hello!☺") == Abi.encode("hello!☺".toBytes) + + test "encodes empty strings": + let encoded = Abi.encode("") + check encoded == Abi.encode(0'u64) + check encoded == 32.zeroes + + test "encodes distinct types as their base type": + type SomeDistinctType = distinct uint16 + let value = 0xAABB'u16 + check Abi.encode(SomeDistinctType(value)) == Abi.encode(value) + + test "encodes zero values": + check Abi.encode(UInt256.zero) == 32.zeroes + check Abi.encode(UInt256.zero) == 32.zeroes + check Abi.encode(@[0'u8, 0'u8]) == + Abi.encode(2'u64) & @[0'u8, 0'u8] & 30.zeroes + + test "encodes large arrays": + let largeArray = newSeqWith(100, 42'u8) + let encoded = Abi.encode(largeArray) + check encoded[0..31] == Abi.encode(100'u64) + + test "encodes very long strings": + let longString = "a".repeat(1000) + let encoded = Abi.encode(longString) + check encoded[0..31] == Abi.encode(1000'u64) \ No newline at end of file diff --git a/tests/test_string_decoder.nim b/tests/test_string_decoder.nim index 776549f6..dccc5847 100644 --- a/tests/test_string_decoder.nim +++ b/tests/test_string_decoder.nim @@ -1,7 +1,8 @@ import pkg/unittest2, stew/byteutils, - ../web3/encoding, + serialization, + ../web3/decoding, ../web3/primitives type @@ -25,12 +26,16 @@ suite "String decoders": signature = init SignatureBytes index = init Int64LeBytes - var offset = 0 - offset += decode(logData, 0, offset, pubkey) - offset += decode(logData, 0, offset, withdrawalCredentials) - offset += decode(logData, 0, offset, amount) - offset += decode(logData, 0, offset, signature) - offset += decode(logData, 0, offset, index) + type Tuple = tuple[ + pubkey: PubKeyBytes, + withdrawalCredentials: WithdrawalCredentialsBytes, + amount: Int64LeBytes, + signature: SignatureBytes, + index: Int64LeBytes + ] + + (pubkey, withdrawalCredentials, amount, signature, index) = + Abi.decode(logData, Tuple) assert($pubkey == "0xb2f5263a3454de3a9116b0edaa3cfbb2795a99482ee268b7aed5b15b532d9b20c34b67c82877ba1326326f3ae6cc5ad3") assert($withdrawalCredentials == "0x010000000000000000000000a3f7076718fa4fed91b5830a45489053eb367afb") diff --git a/web3.nim b/web3.nim index 57c6a5c7..0cc83e96 100644 --- a/web3.nim +++ b/web3.nim @@ -15,7 +15,7 @@ import json_rpc/private/jrpc_sys, eth/common/keys, chronos/apps/http/httpclient, - web3/[eth_api_types, conversions, transaction_signing, encoding, contract_dsl], + web3/[eth_api_types, conversions, transaction_signing, encoding, decoding, contract_dsl], web3/eth_api from eth/common/eth_types import ChainId @@ -25,6 +25,7 @@ export eth_api_types, conversions, encoding, + decoding, contract_dsl, HttpClientFlag, HttpClientFlags, @@ -374,7 +375,7 @@ proc call*[T]( let response = await callAux(c.sender.web3, c.sender.contractAddress, c.sender.web3.defaultAccount, c.data, value, gas, blockNumber) if response.len > 0: - discard decode(response, 0, 0, result) + result = Abi.decode(response, T) else: raise newException(CatchableError, "No response from the Web3 provider") @@ -455,7 +456,9 @@ proc createImmutableContractInvocation*( sender.web3, sender.contractAddress, sender.defaultAccount, data, sender.value, sender.gas, sender.blockNumber) if response.len > 0: - discard decode(response, 0, 0, result) + # All data are encoded as tuple + let (value) = Abi.decode(response, (ReturnType,)) + return value else: raise newException(CatchableError, "No response from the Web3 provider") diff --git a/web3/abi_serialization.nim b/web3/abi_serialization.nim new file mode 100644 index 00000000..00fad1c7 --- /dev/null +++ b/web3/abi_serialization.nim @@ -0,0 +1,39 @@ + +import serialization +import faststreams + +{.push raises: [].} + +serializationFormat Abi, + mimeType = "application/ethereum‑abi" + +type + AbiReader* = object + stream: InputStream + + AbiWriter* = object + stream: OutputStream + +proc new*(T: type AbiReader, stream: InputStream): T = + T(stream: stream) + +proc new*(T: type AbiWriter): T = + T(stream: memoryOutput()) + +proc init*(T: type AbiWriter, s: OutputStream): T = + T(stream: s) + +proc init*(T: type AbiReader, s: InputStream): AbiReader = + AbiReader(stream: s) + +proc getStream*(r: AbiWriter): OutputStream = + r.stream + +proc getStream*(r: AbiReader): InputStream = + r.stream + +proc write*(w: AbiWriter, bytes: seq[byte]) {.raises: [IOError]} = + w.stream.write bytes + +Abi.setReader AbiReader +Abi.setWriter AbiWriter, PreferredOutput = seq[byte] \ No newline at end of file diff --git a/web3/abi_utils.nim b/web3/abi_utils.nim new file mode 100644 index 00000000..7aece381 --- /dev/null +++ b/web3/abi_utils.nim @@ -0,0 +1,48 @@ + +from ./primitives import DynamicBytes + +{.push raises: [].} + +const abiSlotSize* = 32 + +type + AbiSignedInt* = int8|int16|int32|int64 + AbiUnsignedInt* = uint8|uint16|uint32|uint64 + +func isDynamicObject*(T: typedesc): bool + +func isDynamic*(T: type): bool = + when T is seq | openArray | string | DynamicBytes: + return true + elif T is array: + when typeof(default(T)[0]) is byte: + return T.len > abiSlotSize + else: + type t = typeof(default(T)[0]) + return isDynamic(t) + elif T is tuple: + for v in fields(default(T)): + if isDynamic(typeof(v)): + return true + return false + elif T is object: + return isDynamicObject(T) + else: + return false + +func isDynamicType*(a: typedesc): bool = + when a is seq | openArray | string | DynamicBytes: + true + elif a is object: + return isDynamicObject(a) + else: + false + +func isDynamicObject*(T: typedesc): bool {.compileTime.} = + for v in fields(default(T)): + if isDynamicType(typeof(v)): + return true + return false + +func isStatic*(T: type): bool {.compileTime.} = + not isDynamic(T) \ No newline at end of file diff --git a/web3/contract_dsl.nim b/web3/contract_dsl.nim index b39115b6..42453b00 100644 --- a/web3/contract_dsl.nim +++ b/web3/contract_dsl.nim @@ -1,7 +1,7 @@ import std/[macros, strutils], json_serialization, - ./[encoding, eth_api_types], + ./[encoding, decoding, eth_api_types], ./conversions, stint, stew/byteutils @@ -184,7 +184,7 @@ proc genFunction(cname: NimNode, functionObject: FunctionObject): NimNode = funcParamsTuple.add(ident input.name) result = quote do: - proc `procName`*[TSender](`senderName`: ContractInstance[`cname`, TSender]): auto = + proc `procName`*[TSender](`senderName`: ContractInstance[`cname`, TSender]): auto {.raises: [SerializationError].} = discard for input in functionObject.inputs: result[3].add nnkIdentDefs.newTree( @@ -197,13 +197,13 @@ proc genFunction(cname: NimNode, functionObject: FunctionObject): NimNode = mixin createImmutableContractInvocation return createImmutableContractInvocation( `senderName`.sender, `output`, - static(keccak256(`signature`).data[0..<4]) & encode(`funcParamsTuple`)) + static(keccak256(`signature`).data[0..<4]) & Abi.encode(`funcParamsTuple`)) else: result[6] = quote do: mixin createMutableContractInvocation return createMutableContractInvocation( `senderName`.sender, `output`, - static(keccak256(`signature`).data[0..<4]) & encode(`funcParamsTuple`)) + static(keccak256(`signature`).data[0..<4]) & Abi.encode(`funcParamsTuple`)) proc `&`(a, b: openArray[byte]): seq[byte] = let sza = a.len @@ -234,7 +234,7 @@ proc genConstructor(cname: NimNode, constructorObject: ConstructorObject): NimNo ) result[6] = quote do: mixin createContractDeployment - return createContractDeployment(`sender`, `cname`, `contractCode` & encode(`funcParamsTuple`)) + return createContractDeployment(`sender`, `cname`, `contractCode` & Abi.encode(`funcParamsTuple`)) proc genEvent(cname: NimNode, eventObject: EventObject): NimNode = if not eventObject.anonymous: @@ -249,12 +249,11 @@ proc genEvent(cname: NimNode, eventObject: EventObject): NimNode = i = 1 call = nnkCall.newTree(callbackIdent) callWithRawData = nnkCall.newTree(callbackIdent) - offset = ident "offset" argParseBody.add quote do: let `eventData` = JrpcConv.decode(`jsonIdent`.string, EventData) - var offsetInited = false + let tupleType = nnkTupleTy.newTree() for input in eventObject.inputs: let param = nnkIdentDefs.newTree( @@ -267,23 +266,31 @@ proc genEvent(cname: NimNode, eventObject: EventObject): NimNode = let argument = genSym(nskVar) kind = input.typ + if input.indexed: argParseBody.add quote do: - var `argument`: `kind` - discard decode(`eventData`.topics[`i`].data, 0, 0, `argument`) + var `argument`: `kind` = Abi.decode(`eventData`.topics[`i`].data, `kind`) i += 1 + + call.add argument + callWithRawData.add argument else: - if not offsetInited: - argParseBody.add quote do: - var `offset` = 0 + # Generate the tuple type based on the non indexed inputs + tupleType.add nnkIdentDefs.newTree(ident(input.name), input.typ, newEmptyNode()) + + # Decode the tuple in on shot + let decodedIdent = genSym(nskLet, "decoded") + argParseBody.add quote do: + let `decodedIdent` = Abi.decode(`eventData`.data, `tupleType`) - offsetInited = true + # Loop on the inputs again to add the arguments to the calls + for input in eventObject.inputs: + if not input.indexed: + let fieldName = ident(input.name) + let argument = nnkDotExpr.newTree(decodedIdent, fieldName) + call.add argument + callWithRawData.add argument - argParseBody.add quote do: - var `argument`: `kind` - `offset` += decode(`eventData`.data, 0, `offset`, `argument`) - call.add argument - callWithRawData.add argument let eventName = eventObject.name cbident = ident eventName diff --git a/web3/decoding.nim b/web3/decoding.nim new file mode 100644 index 00000000..298eb070 --- /dev/null +++ b/web3/decoding.nim @@ -0,0 +1,439 @@ +import + std/typetraits, + stint, + faststreams/inputs, + stew/[byteutils, endians2, assign2], + serialization, + ./abi_serialization, + ./eth_api_types, + ./abi_utils + +{.push raises: [].} + +export abi_serialization, abi_utils + +type + AbiDecoder* = object + input: InputStream + + UInt = AbiUnsignedInt | StUint + +template basetype(Range: type range): untyped = + when Range isnot AbiUnsignedInt: {.error: "only uint ranges supported".} + elif sizeof(Range) == sizeof(uint8): uint8 + elif sizeof(Range) == sizeof(uint16): uint16 + elif sizeof(Range) == sizeof(uint32): uint32 + elif sizeof(Range) == sizeof(uint64): uint64 + else: + {.error "unsupported range type".} + +proc finish(decoder: var AbiDecoder) {.raises: [SerializationError].} = + try: + if decoder.input.readable: + raise newException(SerializationError, "unread trailing bytes found") + except IOError as e: + raise newException(SerializationError, "Failed to finish decoding: " & e.msg) + +proc read(decoder: var AbiDecoder, size = abiSlotSize): seq[byte] {.raises: [SerializationError].} = + ## We want to make sure that even if the data is smaller than the ABI slot size, + ## it will occupy at least one slot size. That's why we have: + ## size + abiSlotSize - 1 + ## Then we divide it by abiSlotSize to get the number of slots needed. + ## And finally, we multiply it by abiSlotSize to get the total size in bytes. + var buf = newSeq[byte]((size + abiSlotSize - 1) div abiSlotSize * abiSlotSize) + + try: + if not decoder.input.readInto(buf): + raise newException(SerializationError, "reading past end of bytes") + except IOError as e: + raise newException(SerializationError, "Failed to read bytes: " & e.msg) + + return buf + +template checkLeftPadding(buf: openArray[byte], padding: int, expected: uint8) = + for i in 0 ..< padding: + if buf[i] != expected: + raise newException(SerializationError, "invalid padding found") + +template checkRightPadding(buf: openArray[byte], paddingStart: int, paddingEnd: int) = + for i in paddingStart ..< paddingEnd: + if buf[i] != 0x00'u8: + raise newException(SerializationError, "invalid padding found") + +proc decode(_: var AbiDecoder, _: type int): int {.error: + "ABI: plain 'int' is forbidden. Use int8/16/32/64 or Int256."} + +proc decode(_: var AbiDecoder, _: type uint): uint {.error: + "ABI: plain 'uint' is forbidden. Use uint8/16/32/64 or UInt256."} + +proc decode(decoder: var AbiDecoder, T: type UInt): T {.raises: [SerializationError].} = + var buf = decoder.read(sizeof(T)) + + let padding = abiSlotSize - sizeof(T) + checkLeftPadding(buf, padding, 0x00'u8) + + return T.fromBytesBE(buf.toOpenArray(padding, abiSlotSize-1)) + +proc decode(decoder: var AbiDecoder, T: type StInt): T {.raises: [SerializationError].} = + var buf = decoder.read(sizeof(T)) + + let padding = abiSlotSize - sizeof(T) + let value = T.fromBytesBE(buf.toOpenArray(padding, abiSlotSize-1)) + + let b = if value.isNegative: 0xFF'u8 else: 0x00'u8 + checkLeftPadding(buf, padding, b) + + return value + +proc decode(decoder: var AbiDecoder, T: type AbiSignedInt): T {.raises: [SerializationError].} = + var buf = decoder.read(sizeof(T)) + + let padding = abiSlotSize - sizeof(T) + + let unsigned = T.toUnsigned.fromBytesBE(buf.toOpenArray(padding, abiSlotSize - 1)) + let value = cast[T](unsigned) + + let expectedPadding = if value < 0: 0xFF'u8 else: 0x00'u8 + checkLeftPadding(buf, padding, expectedPadding) + + return value + +proc decode(decoder: var AbiDecoder, T: type bool): T {.raises: [SerializationError].} = + case decoder.decode(uint8) + of 0: false + of 1: true + else: raise newException(SerializationError, "invalid boolean value") + +proc decode(decoder: var AbiDecoder, T: type range): T {.raises: [SerializationError].} = + let value = decoder.decode(basetype(T)) + if value in T.low..T.high: + T(value) + else: + raise newException(SerializationError, "value not in range") + +proc decode(decoder: var AbiDecoder, T: type enum): T {.raises: [SerializationError].}= + let value = decoder.decode(uint64) + if value in T.low.uint64..T.high.uint64: + T(value) + else: + raise newException(SerializationError, "invalid enum value") + +proc decode(decoder: var AbiDecoder, T: type Address): T {.raises: [SerializationError].} = + var bytes: array[sizeof(T), byte] + let padding = abiSlotSize - sizeof(T) + + try: + bytes[0.. pos: + decoder.input.advance(offsets[i].int - pos) + res[i] = decoder.decode(T) + + return res + else: + ## When T is static, ABI layout looks like: + ## +----------------------------+ + ## | size of the dynamic array | <-- 32 (optional ONLY for dynamic arrays) + ## +----------------------------+ + ## | element 0 | <-- 32 + ## +----------------------------+ + ## | element 1 | <-- 32 + ## +----------------------------+ + ## | ... | + ## +----------------------------+ + ## | element N-1 | + ## +----------------------------+ + let len = if size.isNone: decoder.decode(uint64) else: size.get() + result = newSeq[T](len) + for i in 0.. pos: + decoder.input.advance(offsets[i].int - pos) + field = decoder.decode(typeof(field)) + + inc i + + discard offsets + + return res + +proc decodeObject(decoder: var AbiDecoder, T: type): T {.raises: [SerializationError].} = + ## When T is a object, ABI layout looks like the typle: + ## +----------------------------+ + ## | static field 0 or offset | <-- 32 + ## +----------------------------+ + ## | static field 1 or offset | <-- 32 + ## +----------------------------+ + ## | ... | + ## +----------------------------+ + ## | static field N-1 or offset | <-- 32 + ## +----------------------------+ + ## | dynamic field 0 data | <-- at offset + ## +----------------------------+ + ## | dynamic field 1 data | <-- at offset + ## +----------------------------+ + ## | ... | + ## +----------------------------+ + var resultObj: T + var offsets = newSeq[uint64](totalSerializedFields(T)) + + # Decode static fields first and get the offsets for dynamic fields. + var i = 0 + resultObj.enumInstanceSerializedFields(_, fieldValue): + when isDynamic(typeof(fieldValue)): + offsets[i] = decoder.decode(uint64) + else: + fieldValue = decoder.decode(typeof(fieldValue)) + inc i + + # Decode dynamic fields using the offsets. + i = 0 + resultObj.enumInstanceSerializedFields(_, fieldValue): + when isDynamic(typeof(fieldValue)): + let pos = decoder.input.pos() + if offsets[i].int > pos: + decoder.input.advance(offsets[i].int - pos) + fieldValue = decoder.decode(typeof(fieldValue)) + inc i + + return resultObj + +proc decode*(decoder: var AbiDecoder, T: type): T {.raises: [SerializationError]} = + ## This method should not be used directly. + ## It is needed because `genFunction` create tuple + ## with object instead of creating a flat tuple with + ## object fields. + when T is object: + let value = decoder.decodeObject(T) + return value + else: + let value = decoder.decode(T) + decoder.finish() + return value + +proc readValue*[T](r: var AbiReader, value: T): T {.raises: [SerializationError]} = + try: + readValue[T](r, T) + except SerializationError as e: + raise newException(SerializationError, e.msg) + +proc readValue*[T](r: var AbiReader, _: typedesc[T]): T {.raises: [SerializationError]} = + var resultObj: T + var decoder = AbiDecoder(input: r.getStream) + type StInts = StInt | StUint + + when T is object and T is not StInts: + resultObj = decodeObject(decoder, T) + else: + resultObj = decoder.decode(T) + + decoder.finish() + result = resultObj + +# Keep the old encode functions for compatibility +func decode*(input: openArray[byte], baseOffset, offset: int, to: var StUint): int {.deprecated: "use Abi.decode instead"} = + const meaningfulLen = to.bits div 8 + let offset = offset + baseOffset + to = type(to).fromBytesBE(input.toOpenArray(offset, offset + meaningfulLen - 1)) + meaningfulLen + +func decode*[N](input: openArray[byte], baseOffset, offset: int, to: var StInt[N]): int {.deprecated: "use Abi.decode instead"} = + const meaningfulLen = N div 8 + let offset = offset + baseOffset + to = type(to).fromBytesBE(input.toOpenArray(offset, offset + meaningfulLen - 1)) + meaningfulLen + +func decodeFixed(input: openArray[byte], baseOffset, offset: int, to: var openArray[byte]): int {.deprecated: "use Abi.decode instead"} = + let meaningfulLen = to.len + var padding = to.len mod 32 + if padding != 0: + padding = 32 - padding + let offset = baseOffset + offset + padding + if to.len != 0: + assign(to, input.toOpenArray(offset, offset + meaningfulLen - 1)) + meaningfulLen + padding + +func decode*[N](input: openArray[byte], baseOffset, offset: int, to: var FixedBytes[N]): int {.inline, deprecated: "use Abi.decode instead".} = + decodeFixed(input, baseOffset, offset, array[N, byte](to)) + +func decode*(input: openArray[byte], baseOffset, offset: int, to: var Address): int {.inline, deprecated: "use Abi.decode instead".} = + decodeFixed(input, baseOffset, offset, array[20, byte](to)) + +func decode*(input: openArray[byte], baseOffset, offset: int, to: var seq[byte]): int {.deprecated: "use Abi.decode instead"} = + var dataOffsetBig, dataLenBig: UInt256 + result = decode(input, baseOffset, offset, dataOffsetBig) + let dataOffset = dataOffsetBig.truncate(int) + discard decode(input, baseOffset, dataOffset, dataLenBig) + let dataLen = dataLenBig.truncate(int) + let actualDataOffset = baseOffset + dataOffset + 32 + to = input[actualDataOffset ..< actualDataOffset + dataLen] + +func decode*(input: openArray[byte], baseOffset, offset: int, to: var string): int {.deprecated: "use Abi.decode instead"} = + var dataOffsetBig, dataLenBig: UInt256 + result = decode(input, baseOffset, offset, dataOffsetBig) + let dataOffset = dataOffsetBig.truncate(int) + discard decode(input, baseOffset, dataOffset, dataLenBig) + let dataLen = dataLenBig.truncate(int) + let actualDataOffset = baseOffset + dataOffset + 32 + to = string.fromBytes(input.toOpenArray(actualDataOffset, actualDataOffset + dataLen - 1)) + +func decode*(input: openArray[byte], baseOffset, offset: int, to: var DynamicBytes): int {.inline, deprecated: "use Abi.decode instead".} = + var s: seq[byte] + result = decode(input, baseOffset, offset, s) + # TODO: Check data len, and raise? + to = typeof(to)(move(s)) + +func decode*(input: openArray[byte], baseOffset, offset: int, obj: var object): int {.deprecated: "use Abi.decode instead"} + +func decode*[T](input: openArray[byte], baseOffset, offset: int, to: var seq[T]): int {.inline, deprecated: "use Abi.decode instead".} = + var dataOffsetBig, dataLenBig: UInt256 + result = decode(input, baseOffset, offset, dataOffsetBig) + let dataOffset = dataOffsetBig.truncate(int) + discard decode(input, baseOffset, dataOffset, dataLenBig) + # TODO: Check data len, and raise? + let dataLen = dataLenBig.truncate(int) + to.setLen(dataLen) + let baseOffset = baseOffset + dataOffset + 32 + var offset = 0 + for i in 0 ..< dataLen: + offset += decode(input, baseOffset, offset, to[i]) + +func decode*(input: openArray[byte], baseOffset, offset: int, to: var bool): int {.deprecated: "use Abi.decode instead"} = + var i: Int256 + result = decode(input, baseOffset, offset, i) + to = not i.isZero() + +func decode*(input: openArray[byte], baseOffset, offset: int, obj: var object): int {.deprecated: "use Abi.decode instead"} = + when isDynamicObject(typeof(obj)): + var dataOffsetBig: UInt256 + result = decode(input, baseOffset, offset, dataOffsetBig) + let dataOffset = dataOffsetBig.truncate(int) + let offset = baseOffset + dataOffset + var offset2 = 0 + for k, field in fieldPairs(obj): + let sz = decode(input, offset, offset2, field) + offset2 += sz + else: + var offset = offset + for field in fields(obj): + let sz = decode(input, baseOffset, offset, field) + offset += sz + result += sz + +# Obsolete +func decode*(input: string, offset: int, to: var DynamicBytes): int {.inline, deprecated: "Use decode(openArray[byte], ...) instead".} = + decode(hexToSeqByte(input), 0, offset div 2, to) * 2 diff --git a/web3/encoding.nim b/web3/encoding.nim index 89e07e9b..cdd663c9 100644 --- a/web3/encoding.nim +++ b/web3/encoding.nim @@ -8,26 +8,235 @@ # those terms. import - std/macros, - stint, ./eth_api_types, stew/[assign2, byteutils] + std/[sequtils, macros], + faststreams/outputs, + stint, + stew/[byteutils, endians2], + serialization, + ./eth_api_types, + ./abi_utils, + ./abi_serialization -func encode*[bits: static[int]](x: StUint[bits]): seq[byte] = - @(x.toBytesBE()) +{.push raises: [].} -func encode*[bits: static[int]](x: StInt[bits]): seq[byte] = - @(x.toBytesBE()) +export abi_serialization, abi_utils + +type + AbiEncoder* = object + output: OutputStream + +func finish(encoder: var AbiEncoder): seq[byte] = + encoder.output.getOutput(seq[byte]) + +proc write(encoder: var AbiEncoder, bytes: openArray[byte]) {.raises: [SerializationError]} = + try: + encoder.output.write(bytes) + except IOError as e: + raise newException(SerializationError, "Failed to write bytes: " & e.msg) + +proc padleft(encoder: var AbiEncoder, bytes: openArray[byte], padding: byte = 0'u8) {.raises: [SerializationError]} = + let padSize = abiSlotSize - bytes.len + if padSize > 0: + encoder.write(repeat(padding, padSize)) + encoder.write(bytes) + +proc padright(encoder: var AbiEncoder, bytes: openArray[byte], padding: byte = 0'u8) {.raises: [SerializationError]} = + ## When padding right, the byte length may exceed abiSlotSize. + ## So we first apply a modulo operation to compute the remainder. + ## If the result is 0, we apply a second modulo to avoid adding + ## a full slot of padding. + encoder.write(bytes) + let padSize = (abiSlotSize - (bytes.len mod abiSlotSize)) mod abiSlotSize + + if padSize > 0: + encoder.write(repeat(padding, padSize)) + +proc encode(encoder: var AbiEncoder, value: AbiUnsignedInt | StUint) {.raises: [SerializationError]} = + encoder.padleft(value.toBytesBE) + +proc encode(encoder: var AbiEncoder, value: AbiSignedInt | StInt) {.raises: [SerializationError]} = + when typeof(value) is StInt: + let unsignedValue = cast[StInt[(type value).bits]](value) + let isNegative = value.isNegative + else: + let unsignedValue = cast[(type value).toUnsigned](value) + let isNegative = value < 0 + + let bytes = unsignedValue.toBytesBE + let padding = if isNegative: 0xFF'u8 else: 0x00'u8 + encoder.padleft(bytes, padding) + +proc encode(encoder: var AbiEncoder, value: bool) {.raises: [SerializationError]} = + encoder.padleft([if value: 1'u8 else: 0'u8]) + +proc encode(encoder: var AbiEncoder, value: enum) {.raises: [SerializationError]} = + encoder.encode(uint64(ord(value))) + +proc encode(encoder: var AbiEncoder, value: Address) {.raises: [SerializationError]} = + encoder.padleft(array[20, byte](value)) + +proc encode(encoder: var AbiEncoder, value: Bytes32) {.raises: [SerializationError]} = + encoder.padleft(array[32, byte](value)) + +proc encode[I](encoder: var AbiEncoder, bytes: array[I, byte]) {.raises: [SerializationError]} = + encoder.padright(bytes) + +proc encode(encoder: var AbiEncoder, bytes: seq[byte]) {.raises: [SerializationError]} = + encoder.encode(bytes.len.uint64) + encoder.padright(bytes) + +proc encode(encoder: var AbiEncoder, value: string) {.raises: [SerializationError]} = + encoder.encode(value.toBytes) + +proc encode(encoder: var AbiEncoder, value: distinct) {.raises: [SerializationError]} = + type Base = distinctBase(typeof(value)) + encoder.encode(Base(value)) + +proc encode[T](encoder: var AbiEncoder, value: seq[T]) {.raises: [SerializationError].} +proc encode[T: tuple](encoder: var AbiEncoder, tupl: T) {.raises: [SerializationError]} + +template encodeCollection(encoder: var AbiEncoder, value: untyped) = + ## When T is dynamic, ABI layout looks like: + ## + ## +----------------------------+ + ## | offset to element 0 | <-- 32 + ## +----------------------------+ + ## | offset to element 1 | <-- 32 + size of encoded element 0 + ## +----------------------------+ + ## | ... | + ## +----------------------------+ + ## | encoded element 0 | <-- item at offset 0 + ## +----------------------------+ + ## | encoded element 1 | <-- item at offset 1 + ## +----------------------------+ + ## | ... | + ## +----------------------------+ + if isDynamic(typeof(value[0])): + var blocks: seq[seq[byte]] = @[] + var offset = value.len * abiSlotSize + + # Encode offset first + for element in value: + var e = AbiEncoder(output: memoryOutput()) + e.encode(element) + let bytes = e.finish() + blocks.add(bytes) + + encoder.encode(offset.uint64) + offset += bytes.len + + # Then encode the data + for data in blocks: + encoder.write(data) + else: + ## When T is static, ABI layout looks like: + ## + ## +----------------------------+ + ## | element 0 | <-- 32 + ## +----------------------------+ + ## | element 1 | <-- 32 + ## +----------------------------+ + ## | ... | + ## +----------------------------+ + ## | element N-1 | + ## +----------------------------+ + for element in value: + encoder.encode(element) -func decode*(input: openArray[byte], baseOffset, offset: int, to: var StUint): int = - const meaningfulLen = to.bits div 8 - let offset = offset + baseOffset - to = type(to).fromBytesBE(input.toOpenArray(offset, offset + meaningfulLen - 1)) - meaningfulLen +proc encode[T, I](encoder: var AbiEncoder, value: array[I, T]) {.raises: [SerializationError].} = + ## Fixed array does not include the length in the ABI encoding. + encodeCollection(encoder, value) -func decode*[N](input: openArray[byte], baseOffset, offset: int, to: var StInt[N]): int = - const meaningfulLen = N div 8 - let offset = offset + baseOffset - to = type(to).fromBytesBE(input.toOpenArray(offset, offset + meaningfulLen - 1)) - meaningfulLen +proc encode[T](encoder: var AbiEncoder, value: seq[T]) {.raises: [SerializationError].} = + ## Sequences are dynamic by definition, so we always encode their length first. + encoder.encode(value.len.uint64) + + encodeCollection(encoder, value) + +proc encodeField(field: auto): seq[byte] {.raises: [SerializationError].} = + var e = AbiEncoder(output: memoryOutput()) + e.encode(field) + return e.finish() + +proc encode[T: tuple](encoder: var AbiEncoder, tupl: T) {.raises: [SerializationError]} = + ## Tuple can contain both static and dynamic elements. + ## When the data is dynamic, the offset to the data is encoded first. + ## + ## Example: (static, dynamic, dynamic) + ## + ## +------------------------------+ + ## | element 1 | + ## +------------------------------+ + ## | offset to element 2 | + ## +------------------------------+ + ## | offset to element 3 | + ## +------------------------------+ + ## | element 2 | + ## +------------------------------+ + ## | element 3 | + ## +------------------------------+ + when isStatic(T): + for field in tupl.fields: + encoder.encode(field) + else: + var offset {.used.} = T.arity * abiSlotSize + var cursor = encoder.output.delayFixedSizeWrite(offset) + + for field in tupl.fields: + when isDynamic(typeof(field)): + let bytes = encodeField(field) + encoder.write(bytes) + + # Encode the offset of the dynamic data + cursor.write(encodeField(offset.uint64)) + offset += bytes.len + else: + cursor.write(encodeField(field)) + + cursor.finalize() + +proc writeValue*[T](w: var AbiWriter, value: T) {.raises: [SerializationError]} = + var encoder = AbiEncoder(output: memoryOutput()) + type StInts = StInt | StUint + + when T is range: + when T.low is int or T.low is uint: + {.error: "Ranges with int or uint bounds are not supported. Use explicit types like int8, int16, uint8, etc.".} + + when T is object and T is not StInts: + var offset {.used.} = totalSerializedFields(T) * abiSlotSize + var cursor = encoder.output.delayFixedSizeWrite(offset) + + value.enumInstanceSerializedFields(_, fieldValue): + when isDynamic(typeof(fieldValue)): + let bytes = encodeField(fieldValue) + encoder.write(bytes) + + # Encode the offset of the dynamic data + cursor.write(encodeField(offset.uint64)) + offset += bytes.len + else: + cursor.write(encodeField(fieldValue)) + + cursor.finalize() + + try: + w.write encoder.finish() + except IOError as e: + raise newException(SerializationError, "Failed to write value: " & e.msg) + else: + try: + encoder.encode(value) + w.write encoder.finish() + except IOError as e: + raise newException(SerializationError, "Failed to write value: " & e.msg) + +# Keep the old encode functions for compatibility +func encode*[bits: static[int]](x: StUint[bits]): seq[byte] {.deprecated: "use Abi.encode instead" .} = + @(x.toBytesBE()) + +func encode*[bits: static[int]](x: StInt[bits]): seq[byte] {.deprecated: "use Abi.encode instead" .} = + @(x.toBytesBE()) func encodeFixed(a: openArray[byte]): seq[byte] = var padding = a.len mod 32 @@ -35,25 +244,9 @@ func encodeFixed(a: openArray[byte]): seq[byte] = result.setLen(padding) # Zero fill padding result.add(a) -func encode*[N: static int](b: FixedBytes[N]): seq[byte] = encodeFixed(b.data) -func encode*(b: Address): seq[byte] = encodeFixed(b.data) -func encode*[N](b: array[N, byte]): seq[byte] {.inline.} = encodeFixed(b) - -func decodeFixed(input: openArray[byte], baseOffset, offset: int, to: var openArray[byte]): int = - let meaningfulLen = to.len - var padding = to.len mod 32 - if padding != 0: - padding = 32 - padding - let offset = baseOffset + offset + padding - if to.len != 0: - assign(to, input.toOpenArray(offset, offset + meaningfulLen - 1)) - meaningfulLen + padding - -func decode*[N](input: openArray[byte], baseOffset, offset: int, to: var FixedBytes[N]): int {.inline.} = - decodeFixed(input, baseOffset, offset, array[N, byte](to)) - -func decode*(input: openArray[byte], baseOffset, offset: int, to: var Address): int {.inline.} = - decodeFixed(input, baseOffset, offset, array[20, byte](to)) +func encode*[N: static int](b: FixedBytes[N]): seq[byte] {.deprecated: "use Abi.encode instead" .} = encodeFixed(b.data) +func encode*(b: Address): seq[byte] {.deprecated: "use Abi.encode instead" .} = encodeFixed(b.data) +func encode*[N](b: array[N, byte]): seq[byte] {.inline, deprecated: "use Abi.encode instead".} = encodeFixed(b) func encodeDynamic(v: openArray[byte]): seq[byte] = result = encode(v.len.u256) @@ -62,98 +255,20 @@ func encodeDynamic(v: openArray[byte]): seq[byte] = if pad != 0: result.setLen(result.len + 32 - pad) -func encode*(x: DynamicBytes): seq[byte] {.inline.} = +func encode*(x: DynamicBytes): seq[byte] {.inline, deprecated: "use Abi.encode instead".} = encodeDynamic(distinctBase x) -func encode*(x: seq[byte]): seq[byte] {.inline.} = +func encode*(x: seq[byte]): seq[byte] {.inline, deprecated: "use Abi.encode instead".} = encodeDynamic(x) -func encode*(x: string): seq[byte] {.inline.} = +func encode*(x: string): seq[byte] {.inline, deprecated: "use Abi.encode instead".} = encodeDynamic(x.toOpenArrayByte(0, x.high)) -func decode*(input: openArray[byte], baseOffset, offset: int, to: var seq[byte]): int = - var dataOffsetBig, dataLenBig: UInt256 - result = decode(input, baseOffset, offset, dataOffsetBig) - let dataOffset = dataOffsetBig.truncate(int) - discard decode(input, baseOffset, dataOffset, dataLenBig) - let dataLen = dataLenBig.truncate(int) - let actualDataOffset = baseOffset + dataOffset + 32 - to = input[actualDataOffset ..< actualDataOffset + dataLen] - -func decode*(input: openArray[byte], baseOffset, offset: int, to: var string): int = - var dataOffsetBig, dataLenBig: UInt256 - result = decode(input, baseOffset, offset, dataOffsetBig) - let dataOffset = dataOffsetBig.truncate(int) - discard decode(input, baseOffset, dataOffset, dataLenBig) - let dataLen = dataLenBig.truncate(int) - let actualDataOffset = baseOffset + dataOffset + 32 - to = string.fromBytes(input.toOpenArray(actualDataOffset, actualDataOffset + dataLen - 1)) - -func decode*(input: openArray[byte], baseOffset, offset: int, to: var DynamicBytes): int {.inline.} = - var s: seq[byte] - result = decode(input, baseOffset, offset, s) - # TODO: Check data len, and raise? - to = typeof(to)(move(s)) - -func decode*(input: openArray[byte], baseOffset, offset: int, obj: var object): int - -func decode*[T](input: openArray[byte], baseOffset, offset: int, to: var seq[T]): int {.inline.} = - var dataOffsetBig, dataLenBig: UInt256 - result = decode(input, baseOffset, offset, dataOffsetBig) - let dataOffset = dataOffsetBig.truncate(int) - discard decode(input, baseOffset, dataOffset, dataLenBig) - # TODO: Check data len, and raise? - let dataLen = dataLenBig.truncate(int) - to.setLen(dataLen) - let baseOffset = baseOffset + dataOffset + 32 - var offset = 0 - for i in 0 ..< dataLen: - offset += decode(input, baseOffset, offset, to[i]) - -func isDynamicObject(T: typedesc): bool - -template isDynamicType(a: typedesc): bool = - when a is seq | openArray | string | DynamicBytes: - true - elif a is object: - const r = isDynamicObject(a) - r - else: - false - -func isDynamicObject(T: typedesc): bool = - var a: T - for v in fields(a): - if isDynamicType(typeof(v)): return true - false - -func encode*(x: bool): seq[byte] = encode(x.int.u256) - -func decode*(input: openArray[byte], baseOffset, offset: int, to: var bool): int = - var i: Int256 - result = decode(input, baseOffset, offset, i) - to = not i.isZero() - -func decode*(input: openArray[byte], baseOffset, offset: int, obj: var object): int = - when isDynamicObject(typeof(obj)): - var dataOffsetBig: UInt256 - result = decode(input, baseOffset, offset, dataOffsetBig) - let dataOffset = dataOffsetBig.truncate(int) - let offset = baseOffset + dataOffset - var offset2 = 0 - for k, field in fieldPairs(obj): - let sz = decode(input, offset, offset2, field) - offset2 += sz - else: - var offset = offset - for field in fields(obj): - let sz = decode(input, baseOffset, offset, field) - offset += sz - result += sz +func encode*(x: bool): seq[byte] {.deprecated: "use Abi.encode instead" .} = encode(x.int.u256) -func encode*(x: tuple): seq[byte] +func encode*(x: tuple): seq[byte] {.deprecated: "use Abi.encode instead" .} -func encode*[T](x: openArray[T]): seq[byte] = +func encode*[T](x: openArray[T]): seq[byte] {.deprecated: "use Abi.encode instead" .} = result = encode(x.len.u256) when isDynamicType(T): result.setLen((1 + x.len) * 32) @@ -171,7 +286,7 @@ func getTupleImpl(t: NimNode): NimNode = macro typeListLen*(t: typedesc[tuple]): int = newLit(t.getTupleImpl().len) -func encode*(x: tuple): seq[byte] = +func encode*(x: tuple): seq[byte] {.deprecated: "use Abi.encode instead" .} = var offsets {.used.}: array[typeListLen(typeof(x)), int] var i = 0 for v in fields(x): @@ -188,8 +303,4 @@ func encode*(x: tuple): seq[byte] = let o = offsets[i] result[o .. o + 31] = encode(offset.u256) result &= encode(v) - inc i - -# Obsolete -func decode*(input: string, offset: int, to: var DynamicBytes): int {.inline, deprecated: "Use decode(openArray[byte], ...) instead".} = - decode(hexToSeqByte(input), 0, offset div 2, to) * 2 + inc i \ No newline at end of file From d62470d4e6a00b82212f39a1635da8f45de35368 Mon Sep 17 00:00:00 2001 From: Miran Date: Mon, 1 Sep 2025 12:40:36 +0200 Subject: [PATCH 20/27] remove macos-13 from the matrix (#221) It is no longer supported, starting from September 1st: https://github.blog/changelog/2025-07-11-upcoming-changes-to-macos-hosted-runners-macos-latest-migration-and-xcode-support-policy-updates/#macos-13-is-closing-down --- .github/workflows/ci.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index efd4d141..113e2094 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,8 +19,6 @@ jobs: cpu: amd64 - os: linux cpu: i386 - - os: macos - cpu: amd64 - os: macos cpu: arm64 - os: windows @@ -35,11 +33,6 @@ jobs: os: linux-gcc-14 # This is to use ubuntu 24 and install gcc 14. Should be removed when ubuntu-latest is 26.04 builder: ubuntu-24.04 shell: bash - - target: - os: macos - cpu: amd64 - builder: macos-13 - shell: bash - target: os: macos cpu: arm64 From c2b10b4622947536c836a5ab5399dacc74cf27e7 Mon Sep 17 00:00:00 2001 From: Chirag Parmar Date: Thu, 4 Sep 2025 12:27:59 +0530 Subject: [PATCH 21/27] fix (#223) --- web3/eth_api.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web3/eth_api.nim b/web3/eth_api.nim index 4a8064d0..e5c90714 100644 --- a/web3/eth_api.nim +++ b/web3/eth_api.nim @@ -31,7 +31,7 @@ createRpcSigsFromNim(RpcClient): proc eth_mining(): bool proc eth_hashrate(): Quantity proc eth_gasPrice(): Quantity - proc eth_blobBaseFee(): Quantity + proc eth_blobBaseFee(): UInt256 proc eth_accounts(): seq[Address] proc eth_blockNumber(): Quantity proc eth_getBalance(data: Address, blockId: BlockIdentifier): UInt256 From 141907cd958d7ee3b554ec94bc9ac7ec692e546b Mon Sep 17 00:00:00 2001 From: fryorcraken <110212804+fryorcraken@users.noreply.github.com> Date: Mon, 15 Sep 2025 16:24:58 +1000 Subject: [PATCH 22/27] fix: readValue UInt256 error (#224) * fix: readValue UInt256 error Fixing `readValue(reader`gensym50, result)' is of type 'UInt256' and has to be used (or discarded)` error by ensuring value returned by `readValue` is used. * Only one readValue function needs to be defined * ci: force arm64 target For some reason, clang assumed x86_64 and rejected nimcrypto's "-march=armv8-a+crypto" flag. --- .github/workflows/ci.yml | 6 ++++++ web3/decoding.nim | 14 +++----------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 113e2094..efa4f457 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -148,6 +148,12 @@ jobs: echo "ncpu=$ncpu" >> $GITHUB_ENV echo "MAKE_CMD=${MAKE_CMD}" >> $GITHUB_ENV + - name: Configure ARM64 build environment (macOS) + if: runner.os == 'macOS' && matrix.target.cpu == 'arm64' + run: | + # Force ARM64 architecture for Nim compilation with proper flags passed to clang + echo "NIMFLAGS=--cpu:arm64 --cc:clang --passC:\"-target arm64-apple-macos11\" --passL:\"-target arm64-apple-macos11\"" >> $GITHUB_ENV + - name: Build Nim and Nimble run: | curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_nim.sh diff --git a/web3/decoding.nim b/web3/decoding.nim index 298eb070..f9f9445f 100644 --- a/web3/decoding.nim +++ b/web3/decoding.nim @@ -325,24 +325,16 @@ proc decode*(decoder: var AbiDecoder, T: type): T {.raises: [SerializationError] decoder.finish() return value -proc readValue*[T](r: var AbiReader, value: T): T {.raises: [SerializationError]} = - try: - readValue[T](r, T) - except SerializationError as e: - raise newException(SerializationError, e.msg) - -proc readValue*[T](r: var AbiReader, _: typedesc[T]): T {.raises: [SerializationError]} = - var resultObj: T +proc readValue*[T](r: var AbiReader, value: var T) {.raises: [SerializationError]} = var decoder = AbiDecoder(input: r.getStream) type StInts = StInt | StUint when T is object and T is not StInts: - resultObj = decodeObject(decoder, T) + value = decodeObject(decoder, T) else: - resultObj = decoder.decode(T) + value = decoder.decode(T) decoder.finish() - result = resultObj # Keep the old encode functions for compatibility func decode*(input: openArray[byte], baseOffset, offset: int, to: var StUint): int {.deprecated: "use Abi.decode instead"} = From cdfe5601d2812a58e54faf53ee634452d01e5918 Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Thu, 25 Sep 2025 19:42:36 +0200 Subject: [PATCH 23/27] v0.8.0 (#226) * Updates to support the new json_serialization streaming * New ABI encoder/decoder (https://github.com/status-im/nim-web3/pull/216) --- web3.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web3.nimble b/web3.nimble index 1464afd2..986c96a7 100644 --- a/web3.nimble +++ b/web3.nimble @@ -8,7 +8,7 @@ # those terms. mode = ScriptMode.Verbose -version = "0.7.0" +version = "0.8.0" author = "Status Research & Development GmbH" description = "These are the humble beginnings of library similar to web3.[js|py]" license = "MIT or Apache License 2.0" From 5ac7da0b95ad80b57c9aa15fb4712db2e95a0abe Mon Sep 17 00:00:00 2001 From: tersec Date: Fri, 10 Oct 2025 13:27:15 +0000 Subject: [PATCH 24/27] rm unused Windows i386 CI support (#227) --- .github/workflows/ci.yml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index efa4f457..2914da62 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -75,17 +75,6 @@ jobs: chmod 755 external/bin/gcc external/bin/g++ echo '${{ github.workspace }}/external/bin' >> $GITHUB_PATH - - name: MSYS2 (Windows i386) - if: runner.os == 'Windows' && matrix.target.cpu == 'i386' - uses: msys2/setup-msys2@v2 - with: - path-type: inherit - msystem: MINGW32 - install: >- - base-devel - git - mingw-w64-i686-toolchain - - name: MSYS2 (Windows amd64) if: runner.os == 'Windows' && matrix.target.cpu == 'amd64' uses: msys2/setup-msys2@v2 From 7d85d5b1db21e0c612c5dfe2dd608dd254da82f4 Mon Sep 17 00:00:00 2001 From: Samyxandz Date: Fri, 17 Oct 2025 16:21:55 +0530 Subject: [PATCH 25/27] add focil related engine changes --- web3/engine_api.nim | 16 ++++++++++++++++ web3/engine_api_types.nim | 2 ++ 2 files changed, 18 insertions(+) diff --git a/web3/engine_api.nim b/web3/engine_api.nim index 25a4a64d..73b5126b 100644 --- a/web3/engine_api.nim +++ b/web3/engine_api.nim @@ -28,6 +28,8 @@ createRpcSigsFromNim(RpcClient): proc engine_newPayloadV2(payload: ExecutionPayloadV2): PayloadStatusV1 proc engine_newPayloadV3(payload: ExecutionPayloadV3, expectedBlobVersionedHashes: seq[VersionedHash], parentBeaconBlockRoot: Hash32): PayloadStatusV1 proc engine_newPayloadV4(payload: ExecutionPayloadV3, expectedBlobVersionedHashes: seq[VersionedHash], parentBeaconBlockRoot: Hash32, executionRequests: seq[seq[byte]]): PayloadStatusV1 + # https://github.com/jihoonsong/execution-apis/blob/focil/src/engine/experimental/eip7805.md#engine_newpayloadv5 + proc engine_newPayloadV5(payload: ExecutionPayloadV3, expectedBlobVersionedHashes: seq[VersionedHash], parentBeaconBlockRoot: Hash32, executionRequests: seq[seq[byte]], inclusionList:InclusionList ): PayloadStatusV1 proc engine_forkchoiceUpdatedV1(forkchoiceState: ForkchoiceStateV1, payloadAttributes: Opt[PayloadAttributesV1]): ForkchoiceUpdatedResponse proc engine_forkchoiceUpdatedV2(forkchoiceState: ForkchoiceStateV1, payloadAttributes: Opt[PayloadAttributesV2]): ForkchoiceUpdatedResponse proc engine_forkchoiceUpdatedV3(forkchoiceState: ForkchoiceStateV1, payloadAttributes: Opt[PayloadAttributesV3]): ForkchoiceUpdatedResponse @@ -41,6 +43,10 @@ createRpcSigsFromNim(RpcClient): proc engine_getPayloadBodiesByRangeV1(start: Quantity, count: Quantity): seq[Opt[ExecutionPayloadBodyV1]] proc engine_getBlobsV1(blob_versioned_hashes: seq[VersionedHash]): GetBlobsV1Response proc engine_getBlobsV2(blob_versioned_hashes: seq[VersionedHash]): GetBlobsV2Response + # https://github.com/jihoonsong/execution-apis/blob/focil/src/engine/experimental/eip7805.md#engine_getinclusionlistv1 + proc engine_getInclusionListV1(parentHash: Hash32): InclusionList + # https://github.com/jihoonsong/execution-apis/blob/focil/src/engine/experimental/eip7805.md#engine_updatepayloadwithinclusionlistv1 + proc engine_updatePayloadWithinInclusionListV1(payloadID: Bytes8, inclusionList: InclusionList): Bytes8 # https://github.com/ethereum/execution-apis/blob/9301c0697e4c7566f0929147112f6d91f65180f6/src/engine/common.md proc engine_exchangeCapabilities(methods: seq[string]): seq[string] @@ -157,6 +163,16 @@ template newPayload*( engine_newPayloadV4( rpcClient, payload, versionedHashes, parentBeaconBlockRoot, executionRequests) + template newPayload*( + rpcClient: RpcClient, + payload: ExecutionPayloadV3, + versionedHashes: seq[VersionedHash], + parentBeaconBlockRoot: Hash32, + executionRequests: seq[seq[byte]], + inclusionList: InclusionList): Future[PayloadStatusV1] = + engine_newPayloadV5( + rpcClient, payload, versionedHashes, parentBeaconBlockRoot, executionRequests, inclusionList) + template exchangeCapabilities*( rpcClient: RpcClient, methods: seq[string]): Future[seq[string]] = diff --git a/web3/engine_api_types.nim b/web3/engine_api_types.nim index a6ef13ae..e1d1a572 100644 --- a/web3/engine_api_types.nim +++ b/web3/engine_api_types.nim @@ -21,6 +21,8 @@ export type TypedTransaction* = distinct seq[byte] + InclusionList* = distinct seq[TypedTransaction] + # https://github.com/ethereum/execution-apis/blob/v1.0.0-beta.4/src/engine/shanghai.md#withdrawalv1 WithdrawalV1* = object index*: Quantity From 645186c0625d89248316d022011890c8ae0f36f5 Mon Sep 17 00:00:00 2001 From: Samyxandz Date: Sun, 19 Oct 2025 03:41:49 +0530 Subject: [PATCH 26/27] add Inclusionlist params --- web3/engine_api_types.nim | 5 ++++- web3/execution_types.nim | 12 ++++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/web3/engine_api_types.nim b/web3/engine_api_types.nim index e1d1a572..00102a5b 100644 --- a/web3/engine_api_types.nim +++ b/web3/engine_api_types.nim @@ -22,7 +22,7 @@ type TypedTransaction* = distinct seq[byte] InclusionList* = distinct seq[TypedTransaction] - + # https://github.com/ethereum/execution-apis/blob/v1.0.0-beta.4/src/engine/shanghai.md#withdrawalv1 WithdrawalV1* = object index*: Quantity @@ -117,6 +117,8 @@ type withdrawals*: seq[WithdrawalV1] blobGasUsed*: Quantity excessBlobGas*: Quantity + inclusionListTransactions*: seq[TypedTransaction] + SomeExecutionPayload* = ExecutionPayloadV1 | @@ -172,6 +174,7 @@ type suggestedFeeRecipient*: Address withdrawals*: seq[WithdrawalV1] parentBeaconBlockRoot*: Hash32 + inclusionListTransactions*: seq[TypedTransaction] # This is ugly, but see the comment on ExecutionPayloadV1OrV2. PayloadAttributesV1OrV2* = object diff --git a/web3/execution_types.nim b/web3/execution_types.nim index 3df02a0b..a4f390ef 100644 --- a/web3/execution_types.nim +++ b/web3/execution_types.nim @@ -36,6 +36,7 @@ type withdrawals*: Opt[seq[WithdrawalV1]] blobGasUsed*: Opt[Quantity] excessBlobGas*: Opt[Quantity] + inclusionListTransactions*: Opt[seq[TypedTransaction]] PayloadAttributes* = object timestamp*: Quantity @@ -43,6 +44,7 @@ type suggestedFeeRecipient*: Address withdrawals*: Opt[seq[WithdrawalV1]] parentBeaconBlockRoot*: Opt[Hash32] + inclusionListTransactions*: Opt[seq[TypedTransaction]] SomeOptionalPayloadAttributes* = Opt[PayloadAttributesV1] | @@ -65,7 +67,7 @@ type V5 func version*(payload: ExecutionPayload): Version = - if payload.blobGasUsed.isSome or payload.excessBlobGas.isSome: + if payload.blobGasUsed.isSome or payload.excessBlobGas.isSome or payload.inclusionListTransactions.isSome: Version.V3 elif payload.withdrawals.isSome: Version.V2 @@ -73,7 +75,7 @@ func version*(payload: ExecutionPayload): Version = Version.V1 func version*(attr: PayloadAttributes): Version = - if attr.parentBeaconBlockRoot.isSome: + if attr.parentBeaconBlockRoot.isSome or attr.inclusionListTransactions.isSome: Version.V3 elif attr.withdrawals.isSome: Version.V2 @@ -122,7 +124,8 @@ func V3*(attr: PayloadAttributes): PayloadAttributesV3 = prevRandao: attr.prevRandao, suggestedFeeRecipient: attr.suggestedFeeRecipient, withdrawals: attr.withdrawals.get(newSeq[WithdrawalV1]()), - parentBeaconBlockRoot: attr.parentBeaconBlockRoot.get + parentBeaconBlockRoot: attr.parentBeaconBlockRoot.get, + inclusionListTransactions: attr.inclusionListTransactions.get(newSeq[TypedTransaction]()) ) func V1*(attr: Opt[PayloadAttributes]): Opt[PayloadAttributesV1] = @@ -161,7 +164,8 @@ func payloadAttributes*(attr: PayloadAttributesV3): PayloadAttributes = prevRandao: attr.prevRandao, suggestedFeeRecipient: attr.suggestedFeeRecipient, withdrawals: Opt.some(attr.withdrawals), - parentBeaconBlockRoot: Opt.some(attr.parentBeaconBlockRoot) + parentBeaconBlockRoot: Opt.some(attr.parentBeaconBlockRoot), + inclusionListTransactions: Opt.some(attr.inclusionListTransactions) ) func payloadAttributes*(x: Opt[PayloadAttributesV1]): Opt[PayloadAttributes] = From 065e472d34083b13dd02d57e7b1eaeb147d6afa6 Mon Sep 17 00:00:00 2001 From: Samyxandz Date: Sun, 26 Oct 2025 22:00:58 +0530 Subject: [PATCH 27/27] fixes --- web3/engine_api_types.nim | 2 ++ 1 file changed, 2 insertions(+) diff --git a/web3/engine_api_types.nim b/web3/engine_api_types.nim index 00102a5b..1361ca9b 100644 --- a/web3/engine_api_types.nim +++ b/web3/engine_api_types.nim @@ -273,6 +273,8 @@ const engineApiInvalidPayloadAttributes* = -38003 engineApiTooLargeRequest* = -38004 engineApiUnsupportedFork* = -38005 + # https://github.com/ethereum/execution-apis/pull/609/files#diff-59590a19c9f19ab80452d1c5411f6a7206ad1d3bc2d0c5c5ba271a6a50e8d8e8R102 + engineApiUnknownParent* = -38006 template `==`*(a, b: TypedTransaction): bool = distinctBase(a) == distinctBase(b)